From 3ce9a3bf9663e83636af3cc30072f3a9dfa38692 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 27 Jan 2020 15:38:56 -0500
Subject: [PATCH 01/47] Cut over from SearchContext to ReaderContext (#51282)

With this change, we partially move the state of SearchContext to
ReaderContext. This is another step allowing us to move the state of
search to the coordinating node.

We will need several follow-ups to move the entire search state to the
coordinating node.

Relates #46523
---
 .../ParentChildInnerHitContextBuilder.java    |   8 +-
 .../action/search/SearchTransportService.java |   4 +-
 .../index/query/NestedQueryBuilder.java       |   6 +-
 .../index/search/stats/ShardSearchStats.java  |  12 +-
 .../index/shard/SearchOperationListener.java  |  57 ++--
 .../search/DefaultSearchContext.java          |  88 ++---
 .../elasticsearch/search/SearchService.java   | 306 +++++++-----------
 .../search/aggregations/AggregationPhase.java |   2 -
 .../search/aggregations/AggregatorBase.java   |   3 +-
 .../aggregations/AggregatorFactory.java       |   3 +-
 .../fetch/subphase/ExplainFetchSubPhase.java  |   2 -
 .../subphase/MatchedQueriesFetchSubPhase.java |   3 -
 .../internal/FilteredSearchContext.java       |  35 --
 .../search/internal/ReaderContext.java        | 155 +++++++++
 .../search/internal/ScrollContext.java        |   7 +
 .../search/internal/SearchContext.java        | 102 +-----
 .../search/internal/SubSearchContext.java     |  21 --
 .../search/query/QueryPhase.java              |   4 -
 .../elasticsearch/index/IndexModuleTests.java |  10 +-
 .../shard/SearchOperationListenerTests.java   |  27 +-
 .../search/DefaultSearchContextTests.java     |  40 ++-
 .../search/SearchServiceTests.java            | 149 ++++-----
 .../search/query/QueryPhaseTests.java         |  10 +-
 .../search/MockSearchService.java             |  18 +-
 .../aggregations/AggregatorTestCase.java      |   2 +-
 .../elasticsearch/test/TestSearchContext.java |  39 +--
 .../search/MockSearchServiceTests.java        |  46 +--
 .../index/engine/FrozenEngine.java            |  24 +-
 .../SecuritySearchOperationListener.java      |   5 +-
 .../SecuritySearchOperationListenerTests.java |  35 +-
 30 files changed, 541 insertions(+), 682 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java

diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
index 551e57abb62a1..626ecab0f4a34 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
@@ -158,12 +158,8 @@ public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException {
                         topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE);
                         maxScoreCollector = new MaxScoreCollector();
                     }
-                    try {
-                        for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) {
-                            intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
-                        }
-                    } finally {
-                        clearReleasables(Lifetime.COLLECTION);
+                    for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) {
+                        intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
                     }
                     TopDocs topDocs = topDocsCollector.topDocs(from(), size());
                     float maxScore = Float.NaN;
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 822557696f7e5..76aa1fa274823 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -286,13 +286,13 @@ public void writeTo(StreamOutput out) throws IOException {
     public static void registerRequestHandler(TransportService transportService, SearchService searchService) {
         transportService.registerRequestHandler(FREE_CONTEXT_SCROLL_ACTION_NAME, ThreadPool.Names.SAME, ScrollFreeContextRequest::new,
             (request, channel, task) -> {
-                boolean freed = searchService.freeContext(request.id());
+                boolean freed = searchService.freeReaderContext(request.id());
                 channel.sendResponse(new SearchFreeContextResponse(freed));
         });
         TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_SCROLL_ACTION_NAME, SearchFreeContextResponse::new);
         transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, ThreadPool.Names.SAME, SearchFreeContextRequest::new,
             (request, channel, task) -> {
-                boolean freed = searchService.freeContext(request.id());
+                boolean freed = searchService.freeReaderContext(request.id());
                 channel.sendResponse(new SearchFreeContextResponse(freed));
         });
         TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, SearchFreeContextResponse::new);
diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index fecf5c8407e98..55ec170c1e02b 100644
--- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -421,11 +421,7 @@ public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException {
                         topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE);
                         maxScoreCollector = new MaxScoreCollector();
                     }
-                    try {
-                        intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
-                    } finally {
-                        clearReleasables(Lifetime.COLLECTION);
-                    }
+                    intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
 
                     TopDocs td = topDocsCollector.topDocs(from(), size());
                     float maxScore = Float.NaN;
diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
index 5446b1ed0e372..664b631249f68 100644
--- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
+++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
@@ -24,6 +24,8 @@
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.index.shard.SearchOperationListener;
+import org.elasticsearch.search.internal.ReaderContext;
+import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.util.HashMap;
@@ -146,25 +148,25 @@ private StatsHolder groupStats(String group) {
     }
 
     @Override
-    public void onNewContext(SearchContext context) {
+    public void onNewReaderContext(ReaderContext readerContext) {
         openContexts.inc();
     }
 
     @Override
-    public void onFreeContext(SearchContext context) {
+    public void onFreeReaderContext(ReaderContext readerContext) {
         openContexts.dec();
     }
 
     @Override
-    public void onNewScrollContext(SearchContext context) {
+    public void onNewScrollContext(ScrollContext scrollContext) {
         totalStats.scrollCurrent.inc();
     }
 
     @Override
-    public void onFreeScrollContext(SearchContext context) {
+    public void onFreeScrollContext(ScrollContext scrollContext) {
         totalStats.scrollCurrent.dec();
         assert totalStats.scrollCurrent.count() >= 0;
-        totalStats.scrollMetric.inc(TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - context.getOriginNanoTime()));
+        totalStats.scrollMetric.inc(TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - scrollContext.getStartTimeInNano()));
     }
 
     static final class StatsHolder {
diff --git a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
index ede86e6ec222d..77e83edcdd18f 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
@@ -21,6 +21,8 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.search.internal.ReaderContext;
+import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.transport.TransportRequest;
 
@@ -76,34 +78,34 @@ default void onFailedFetchPhase(SearchContext searchContext) {}
     default void onFetchPhase(SearchContext searchContext, long tookInNanos) {}
 
     /**
-     * Executed when a new search context was created
-     * @param context the created context
+     * Executed when a new reader context was created
+     * @param readerContext the created context
      */
-    default void onNewContext(SearchContext context) {}
+    default void onNewReaderContext(ReaderContext readerContext) {}
 
     /**
-     * Executed when a previously created search context is freed.
+     * Executed when a previously created reader context is freed.
      * This happens either when the search execution finishes, if the
      * execution failed or if the search context as idle for and needs to be
      * cleaned up.
-     * @param context the freed search context
+     * @param readerContext the freed reader context
      */
-    default void onFreeContext(SearchContext context) {}
+    default void onFreeReaderContext(ReaderContext readerContext) {}
 
     /**
      * Executed when a new scroll search {@link SearchContext} was created
-     * @param context the created search context
+     * @param scrollContext the created search context
      */
-    default void onNewScrollContext(SearchContext context) {}
+    default void onNewScrollContext(ScrollContext scrollContext) {}
 
     /**
      * Executed when a scroll search {@link SearchContext} is freed.
      * This happens either when the scroll search execution finishes, if the
      * execution failed or if the search context as idle for and needs to be
      * cleaned up.
-     * @param context the freed search context
+     * @param scrollContext the freed search context
      */
-    default void onFreeScrollContext(SearchContext context) {}
+    default void onFreeScrollContext(ScrollContext scrollContext) {}
 
     /**
      * Executed prior to using a {@link SearchContext} that has been retrieved
@@ -114,6 +116,14 @@ default void onFreeScrollContext(SearchContext context) {}
      */
     default void validateSearchContext(SearchContext context, TransportRequest transportRequest) {}
 
+    /**
+     * Executed when a search context was freed. The implementor can implement
+     * this method to release resources used by the search context.
+     */
+    default void onFreeSearchContext(SearchContext context) {
+
+    }
+
     /**
      * A Composite listener that multiplexes calls to each of the listeners methods.
      */
@@ -193,10 +203,10 @@ public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
         }
 
         @Override
-        public void onNewContext(SearchContext context) {
+        public void onNewReaderContext(ReaderContext readerContext) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onNewContext(context);
+                    listener.onNewReaderContext(readerContext);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onNewContext listener [{}] failed", listener), e);
                 }
@@ -204,10 +214,10 @@ public void onNewContext(SearchContext context) {
         }
 
         @Override
-        public void onFreeContext(SearchContext context) {
+        public void onFreeReaderContext(ReaderContext readerContext) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onFreeContext(context);
+                    listener.onFreeReaderContext(readerContext);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onFreeContext listener [{}] failed", listener), e);
                 }
@@ -215,10 +225,10 @@ public void onFreeContext(SearchContext context) {
         }
 
         @Override
-        public void onNewScrollContext(SearchContext context) {
+        public void onNewScrollContext(ScrollContext scrollContext) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onNewScrollContext(context);
+                    listener.onNewScrollContext(scrollContext);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onNewScrollContext listener [{}] failed", listener), e);
                 }
@@ -226,10 +236,10 @@ public void onNewScrollContext(SearchContext context) {
         }
 
         @Override
-        public void onFreeScrollContext(SearchContext context) {
+        public void onFreeScrollContext(ScrollContext scrollContext) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onFreeScrollContext(context);
+                    listener.onFreeScrollContext(scrollContext);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onFreeScrollContext listener [{}] failed", listener), e);
                 }
@@ -248,5 +258,16 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
             }
             ExceptionsHelper.reThrowIfNotNull(exception);
         }
+
+        @Override
+        public void onFreeSearchContext(SearchContext context) {
+            for (SearchOperationListener listener : listeners) {
+                try {
+                    listener.onFreeSearchContext(context);
+                } catch (Exception e) {
+                    logger.warn(() -> new ParameterizedMessage("onFreeSearchContext listener [{}] failed", listener), e);
+                }
+            }
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 5ea5ed3f23e35..e74c52b70f3ec 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.WeightFactorFunction;
@@ -61,6 +60,7 @@
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
@@ -75,7 +75,6 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -83,12 +82,11 @@
 
 final class DefaultSearchContext extends SearchContext {
 
-    private final long id;
+    private final ReaderContext readerContext;
     private final ShardSearchRequest request;
     private final SearchShardTarget shardTarget;
     private final LongSupplier relativeTimeSupplier;
     private SearchType searchType;
-    private final Engine.Searcher engineSearcher;
     private final BigArrays bigArrays;
     private final IndexShard indexShard;
     private final ClusterService clusterService;
@@ -102,7 +100,6 @@ final class DefaultSearchContext extends SearchContext {
     // terminate after count
     private int terminateAfter = DEFAULT_TERMINATE_AFTER;
     private List<String> groupStats;
-    private ScrollContext scrollContext;
     private boolean explain;
     private boolean version = false; // by default, we don't return versions
     private boolean seqAndPrimaryTerm = false;
@@ -142,10 +139,6 @@ final class DefaultSearchContext extends SearchContext {
     private SearchContextAggregations aggregations;
     private SearchContextHighlight highlight;
     private SuggestionSearchContext suggest;
-    private List<RescoreContext> rescore;
-    private volatile long keepAlive;
-    private final long originNanoTime = System.nanoTime();
-    private volatile long lastAccessTime = -1;
     private Profilers profilers;
 
     private final Map<String, SearchExtBuilder> searchExtBuilders = new HashMap<>();
@@ -153,36 +146,36 @@ final class DefaultSearchContext extends SearchContext {
     private final QueryShardContext queryShardContext;
     private final FetchPhase fetchPhase;
 
-    DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget,
-                         Engine.Searcher engineSearcher, ClusterService clusterService, IndexService indexService,
-                         IndexShard indexShard, BigArrays bigArrays, LongSupplier relativeTimeSupplier, TimeValue timeout,
-                         FetchPhase fetchPhase) {
-        this.id = id;
+    DefaultSearchContext(ReaderContext readerContext, ShardSearchRequest request, SearchShardTarget shardTarget,
+                         ClusterService clusterService, IndexService indexService, IndexShard indexShard, BigArrays bigArrays,
+                         LongSupplier relativeTimeSupplier, TimeValue timeout, FetchPhase fetchPhase) {
+        this.readerContext = readerContext;
         this.request = request;
         this.fetchPhase = fetchPhase;
         this.searchType = request.searchType();
         this.shardTarget = shardTarget;
-        this.engineSearcher = engineSearcher;
         // SearchContexts use a BigArrays that can circuit break
         this.bigArrays = bigArrays.withCircuitBreaking();
-        this.dfsResult = new DfsSearchResult(id, shardTarget);
-        this.queryResult = new QuerySearchResult(id, shardTarget);
-        this.fetchResult = new FetchSearchResult(id, shardTarget);
+        this.dfsResult = new DfsSearchResult(readerContext.id(), shardTarget);
+        this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget);
+        this.fetchResult = new FetchSearchResult(readerContext.id(), shardTarget);
         this.indexShard = indexShard;
         this.indexService = indexService;
         this.clusterService = clusterService;
+        final Engine.Searcher engineSearcher = readerContext.engineSearcher();
         this.searcher = new ContextIndexSearcher(engineSearcher.getIndexReader(), engineSearcher.getSimilarity(),
             engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy());
+        searcher.setAggregatedDfs(readerContext.aggregatedDfs());
         this.relativeTimeSupplier = relativeTimeSupplier;
         this.timeout = timeout;
-        queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher,
+        queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher,
             request::nowInMillis, shardTarget.getClusterAlias());
         queryBoost = request.indexBoost();
     }
 
     @Override
     public void doClose() {
-        Releasables.close(engineSearcher);
+        readerContext.decRef();
     }
 
     /**
@@ -199,7 +192,7 @@ public void preProcess(boolean rewrite) {
         int maxResultWindow = indexService.getIndexSettings().getMaxResultWindow();
 
         if (resultWindow > maxResultWindow) {
-            if (scrollContext == null) {
+            if (scrollContext() == null) {
                 throw new IllegalArgumentException(
                         "Result window is too large, from + size must be less than or equal to: [" + maxResultWindow + "] but was ["
                                 + resultWindow + "]. See the scroll api for a more efficient way to request large data sets. "
@@ -211,12 +204,12 @@ public void preProcess(boolean rewrite) {
                             + "]. Scroll batch sizes cost as much memory as result windows so they are controlled by the ["
                             + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey() + "] index level setting.");
         }
-        if (rescore != null) {
+        if (rescore().isEmpty() == false) {
             if (sort != null) {
                 throw new IllegalArgumentException("Cannot use [sort] option in conjunction with [rescore].");
             }
             int maxWindow = indexService.getIndexSettings().getMaxRescoreWindow();
-            for (RescoreContext rescoreContext: rescore) {
+            for (RescoreContext rescoreContext: rescore()) {
                 if (rescoreContext.getWindowSize() > maxWindow) {
                     throw new IllegalArgumentException("Rescore window [" + rescoreContext.getWindowSize() + "] is too large. "
                             + "It must be less than [" + maxWindow + "]. This prevents allocating massive heaps for storing the results "
@@ -297,12 +290,12 @@ && new NestedHelper(mapperService()).mightMatchNestedDocs(query)
 
     @Override
     public long id() {
-        return this.id;
+        return readerContext.id();
     }
 
     @Override
     public String source() {
-        return engineSearcher.source();
+        return readerContext.source();
     }
 
     @Override
@@ -330,21 +323,11 @@ public float queryBoost() {
         return queryBoost;
     }
 
-    @Override
-    public long getOriginNanoTime() {
-        return originNanoTime;
-    }
-
     @Override
     public ScrollContext scrollContext() {
-        return this.scrollContext;
+        return readerContext.scrollContext();
     }
 
-    @Override
-    public SearchContext scrollContext(ScrollContext scrollContext) {
-        this.scrollContext = scrollContext;
-        return this;
-    }
 
     @Override
     public SearchContextAggregations aggregations() {
@@ -391,18 +374,7 @@ public void suggest(SuggestionSearchContext suggest) {
 
     @Override
     public List<RescoreContext> rescore() {
-        if (rescore == null) {
-            return Collections.emptyList();
-        }
-        return rescore;
-    }
-
-    @Override
-    public void addRescore(RescoreContext rescore) {
-        if (this.rescore == null) {
-            this.rescore = new ArrayList<>();
-        }
-        this.rescore.add(rescore);
+        return readerContext.rescore();
     }
 
     @Override
@@ -736,26 +708,6 @@ public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int
         return this;
     }
 
-    @Override
-    public void accessed(long accessTime) {
-        this.lastAccessTime = accessTime;
-    }
-
-    @Override
-    public long lastAccessTime() {
-        return this.lastAccessTime;
-    }
-
-    @Override
-    public long keepAlive() {
-        return this.keepAlive;
-    }
-
-    @Override
-    public void keepAlive(long keepAlive) {
-        this.keepAlive = keepAlive;
-    }
-
     @Override
     public DfsSearchResult dfsResult() {
         return dfsResult;
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 84c4903dfd838..2421baf5fb72c 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -32,9 +32,11 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -82,9 +84,8 @@
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.ScrollContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContext.Lifetime;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhase;
@@ -102,7 +103,6 @@
 import org.elasticsearch.threadpool.Scheduler.Cancellable;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Names;
-import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -118,7 +118,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.LongSupplier;
-import java.util.function.Supplier;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueHours;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
@@ -189,7 +188,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
 
     private final AtomicLong idGenerator = new AtomicLong();
 
-    private final ConcurrentMapLong<SearchContext> activeContexts = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
+    private final ConcurrentMapLong<ReaderContext> activeReaders = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
 
     private final MultiBucketConsumerService multiBucketConsumerService;
 
@@ -278,13 +277,13 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem
 
     }
 
-    protected void putContext(SearchContext context) {
-        final SearchContext previous = activeContexts.put(context.id(), context);
+    protected void putReaderContext(ReaderContext context) {
+        final ReaderContext previous = activeReaders.put(context.id(), context);
         assert previous == null;
     }
 
-    protected SearchContext removeContext(long id) {
-        return activeContexts.remove(id);
+    protected ReaderContext removeReaderContext(long id) {
+        return activeReaders.remove(id);
     }
 
     @Override
@@ -293,8 +292,8 @@ protected void doStart() {
 
     @Override
     protected void doStop() {
-        for (final SearchContext context : activeContexts.values()) {
-            freeContext(context.id());
+        for (final ReaderContext context : activeReaders.values()) {
+            freeReaderContext(context.id());
         }
     }
 
@@ -309,20 +308,14 @@ public void executeDfsPhase(ShardSearchRequest request, SearchShardTask task, Ac
     }
 
     private DfsSearchResult executeDfsPhase(ShardSearchRequest request, SearchShardTask task) throws IOException {
-        final SearchContext context = createAndPutContext(request);
-        context.incRef();
-        try {
-            context.setTask(task);
-            contextProcessing(context);
+        final ReaderContext reader = createAndPutReaderContext(request);
+        try (SearchContext context = createContext(reader, reader.request(), task, true)) {
             dfsPhase.execute(context);
-            contextProcessedSuccessfully(context);
             return context.dfsResult();
         } catch (Exception e) {
             logger.trace("Dfs phase failed", e);
-            processFailure(context, e);
+            processFailure(reader, e);
             throw e;
-        } finally {
-            cleanContext(context);
         }
     }
 
@@ -343,23 +336,18 @@ public void executeQueryPhase(ShardSearchRequest request, SearchShardTask task,
         rewriteShardRequest(request, ActionListener.map(listener, r -> executeQueryPhase(r, task)));
     }
 
-    private <T> void runAsync(long id, Supplier<T> executable, ActionListener<T> listener) {
+    private <T> void runAsync(long id, CheckedSupplier<T, IOException> executable, ActionListener<T> listener) {
         getExecutor(id).execute(ActionRunnable.supply(listener, executable::get));
     }
 
     private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchShardTask task) throws Exception {
-        final SearchContext context = createAndPutContext(request);
-        context.incRef();
-        try {
-            context.setTask(task);
+        final ReaderContext reader = createAndPutReaderContext(request);
+        try (SearchContext context = createContext(reader, reader.request(), task, true)) {
             final long afterQueryTime;
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
-                contextProcessing(context);
                 loadOrExecuteQueryPhase(request, context);
                 if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) {
-                    freeContext(context.id());
-                } else {
-                    contextProcessedSuccessfully(context);
+                    freeReaderContext(reader.id());
                 }
                 afterQueryTime = executor.success();
             }
@@ -374,10 +362,8 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchSh
                     (Exception) e.getCause() : new ElasticsearchException(e.getCause());
             }
             logger.trace("Query phase failed", e);
-            processFailure(context, e);
+            processFailure(reader, e);
             throw e;
-        } finally {
-            cleanContext(context);
         }
     }
 
@@ -386,9 +372,7 @@ private QueryFetchSearchResult executeFetchPhase(SearchContext context, long aft
             shortcutDocIdsToLoad(context);
             fetchPhase.execute(context);
             if (fetchPhaseShouldFreeContext(context)) {
-                freeContext(context.id());
-            } else {
-                contextProcessedSuccessfully(context);
+                freeReaderContext(context.id());
             }
             executor.success();
         }
@@ -399,49 +383,41 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
         runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
-            context.incRef();
-            try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
-                context.setTask(task);
-                contextProcessing(context);
-                processScroll(request, context);
+            final ReaderContext reader = findReaderContext(request.id());
+            try (SearchContext context = createContext(reader, reader.request(), task, false);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
+                processScroll(request, reader, context);
                 queryPhase.execute(context);
-                contextProcessedSuccessfully(context);
                 executor.success();
                 return new ScrollQuerySearchResult(context.queryResult(), context.shardTarget());
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
-                processFailure(context, e);
+                processFailure(reader, e);
                 throw e;
-            } finally {
-                cleanContext(context);
             }
         }, listener);
     }
 
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
         runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
-            context.setTask(task);
-            context.incRef();
-            try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
-                contextProcessing(context);
+            final ReaderContext reader = findReaderContext(request.id());
+            reader.aggregatedDfs(request.dfs());
+            try (SearchContext context = createContext(reader, reader.request(), task, true);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 context.searcher().setAggregatedDfs(request.dfs());
                 queryPhase.execute(context);
                 if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) {
                     // no hits, we can release the context since there will be no fetch phase
-                    freeContext(context.id());
-                } else {
-                    contextProcessedSuccessfully(context);
+                    freeReaderContext(reader.id());
                 }
                 executor.success();
                 return context.queryResult();
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
-                processFailure(context, e);
+                processFailure(reader, e);
                 throw e;
-            } finally {
-                cleanContext(context);
             }
         }, listener);
     }
@@ -457,11 +433,7 @@ private boolean fetchPhaseShouldFreeContext(SearchContext context) {
     }
 
     final Executor getExecutor(long id) {
-        SearchContext context = activeContexts.get(id);
-        if (context == null) {
-            throw new SearchContextMissingException(id);
-        }
-        return getExecutor(context.indexShard());
+        return getExecutor(findReaderContext(id).indexShard());
     }
 
     private Executor getExecutor(IndexShard indexShard) {
@@ -472,33 +444,28 @@ private Executor getExecutor(IndexShard indexShard) {
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
         runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
-            context.setTask(task);
-            context.incRef();
-            try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)){
-                contextProcessing(context);
-                processScroll(request, context);
+            final ReaderContext reader = findReaderContext(request.id());
+            try (SearchContext context = createContext(reader, reader.request(), task, false);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
+                processScroll(request, reader, context);
                 queryPhase.execute(context);
                 final long afterQueryTime = executor.success();
                 QueryFetchSearchResult fetchSearchResult = executeFetchPhase(context, afterQueryTime);
                 return new ScrollQueryFetchSearchResult(fetchSearchResult, context.shardTarget());
             } catch (Exception e) {
                 logger.trace("Fetch phase failed", e);
-                processFailure(context, e);
+                processFailure(reader, e);
                 throw e;
-            } finally {
-                cleanContext(context);
             }
         }, listener);
     }
 
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
         runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
-            context.incRef();
-            try {
-                context.setTask(task);
-                contextProcessing(context);
+            final ReaderContext reader = findReaderContext(request.id());
+            try (SearchContext context = createContext(reader, reader.request(), task, false)) {
+                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 if (request.lastEmittedDoc() != null) {
                     context.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
                 }
@@ -506,89 +473,74 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
                 try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, System.nanoTime())) {
                     fetchPhase.execute(context);
                     if (fetchPhaseShouldFreeContext(context)) {
-                        freeContext(request.id());
-                    } else {
-                        contextProcessedSuccessfully(context);
+                        freeReaderContext(request.id());
                     }
                     executor.success();
                 }
                 return context.fetchResult();
             } catch (Exception e) {
                 logger.trace("Fetch phase failed", e);
-                processFailure(context, e);
+                processFailure(reader, e);
                 throw e;
-            } finally {
-                cleanContext(context);
             }
         }, listener);
     }
 
-    private SearchContext findContext(long id, TransportRequest request) throws SearchContextMissingException {
-        SearchContext context = activeContexts.get(id);
-        if (context == null) {
+    private ReaderContext findReaderContext(long id) throws SearchContextMissingException {
+        final ReaderContext reader = activeReaders.get(id);
+        if (reader == null) {
             throw new SearchContextMissingException(id);
         }
-
-        SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
-        try {
-            operationListener.validateSearchContext(context, request);
-            return context;
-        } catch (Exception e) {
-            processFailure(context, e);
-            throw e;
-        }
+        return reader;
     }
 
-    final SearchContext createAndPutContext(ShardSearchRequest request) throws IOException {
+    final ReaderContext createAndPutReaderContext(ShardSearchRequest request) {
         if (request.scroll() != null && openScrollContexts.get() >= maxOpenScrollContext) {
             throw new ElasticsearchException(
                 "Trying to create too many scroll contexts. Must be less than or equal to: [" +
                     maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
                     + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
         }
-
-        SearchContext context = createContext(request);
-        onNewContext(context);
-        boolean success = false;
+        final IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().getId());
+        final SearchOperationListener searchOperationListener = indexShard.getSearchOperationListener();
+        Engine.Searcher engineSearcher = indexShard.acquireSearcher("search");
+        ReaderContext readerContext = null;
         try {
-            putContext(context);
-            success = true;
-            return context;
-        } finally {
-            if (success == false) {
-                freeContext(context.id());
-            }
-        }
-    }
-
-    private void onNewContext(SearchContext context) {
-        boolean success = false;
-        try {
-            if (context.scrollContext() != null) {
+            readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, engineSearcher, request);
+            engineSearcher = null;
+            final ReaderContext finalReaderContext = readerContext;
+             searchOperationListener.onNewReaderContext(finalReaderContext);
+            if (finalReaderContext.scrollContext() != null) {
+                searchOperationListener.onNewScrollContext(finalReaderContext.scrollContext());
                 openScrollContexts.incrementAndGet();
-                context.indexShard().getSearchOperationListener().onNewScrollContext(context);
             }
-            context.indexShard().getSearchOperationListener().onNewContext(context);
-            success = true;
-        } finally {
-            // currently, the concrete listener is CompositeListener, which swallows exceptions, but here we anyway try to do the
-            // right thing by closing and notifying onFreeXXX in case one of the listeners fails with an exception in the future.
-            if (success == false) {
-                try (context) {
-                    onFreeContext(context);
+            readerContext.addOnClose(() -> {
+                try {
+                    if (finalReaderContext.scrollContext() != null) {
+                        openScrollContexts.decrementAndGet();
+                        searchOperationListener.onFreeScrollContext(finalReaderContext.scrollContext());
+                    }
+                } finally {
+                     searchOperationListener.onFreeReaderContext(finalReaderContext);
                 }
-            }
+            });
+            putReaderContext(finalReaderContext);
+            readerContext = null;
+            return finalReaderContext;
+        } finally {
+            Releasables.close(engineSearcher, readerContext);
         }
     }
 
-    final SearchContext createContext(ShardSearchRequest request) throws IOException {
-        final DefaultSearchContext context = createSearchContext(request, defaultSearchTimeout);
+    final SearchContext createContext(ReaderContext reader, ShardSearchRequest request, SearchShardTask task,
+                                      boolean includeAggregations) throws IOException {
+        final DefaultSearchContext context = createSearchContext(reader, request, defaultSearchTimeout);
         try {
+            context.setTask(task);
             if (request.scroll() != null) {
-                context.scrollContext(new ScrollContext());
                 context.scrollContext().scroll = request.scroll();
             }
-            parseSource(context, request.source());
+            parseSource(reader, context, request.source(), includeAggregations);
 
             // if the from and size are still not set, default them
             if (context.from() == -1) {
@@ -608,8 +560,12 @@ final SearchContext createContext(ShardSearchRequest request) throws IOException
             if (request.scroll() != null && request.scroll().keepAlive() != null) {
                 keepAlive = request.scroll().keepAlive().millis();
             }
-            contextScrollKeepAlive(context, keepAlive);
+            contextScrollKeepAlive(reader, keepAlive);
             context.lowLevelCancellation(lowLevelCancellation);
+            // Disable timeout while searching and mark the reader as accessed when releasing the context.
+            context.addReleasable(() -> reader.accessed(threadPool.relativeTimeInMillis()));
+            reader.accessed(-1L);
+            context.addReleasable(() -> reader.indexShard().getSearchOperationListener().onFreeSearchContext(context));
         } catch (Exception e) {
             context.close();
             throw e;
@@ -619,28 +575,34 @@ final SearchContext createContext(ShardSearchRequest request) throws IOException
     }
 
     public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout) throws IOException {
-        return createSearchContext(request, timeout, true, "search");
+        IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().getId());
+        Engine.Searcher engineSearcher = indexShard.acquireSearcher("search");
+        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, engineSearcher, request)) {
+            engineSearcher = null; // transfer ownership to readerContext
+            return createSearchContext(readerContext, request, timeout);
+        } finally {
+            Releasables.close(engineSearcher);
+        }
     }
 
-    private DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout,
-                                                     boolean assertAsyncActions, String source)
+    private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSearchRequest request, TimeValue timeout)
             throws IOException {
         IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
         IndexShard indexShard = indexService.getShard(request.shardId().getId());
         SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(),
                 indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE);
-        Engine.Searcher searcher = indexShard.acquireSearcher(source);
 
         boolean success = false;
         DefaultSearchContext searchContext = null;
+        reader.incRef();
         try {
-            searchContext = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget,
-                searcher, clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase);
+            searchContext = new DefaultSearchContext(reader, request, shardTarget,
+                clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase);
             // we clone the query shard context here just for rewriting otherwise we
             // might end up with incorrect state since we are using now() or script services
             // during rewrite and normalized / evaluate templates etc.
             QueryShardContext context = new QueryShardContext(searchContext.getQueryShardContext());
-            Rewriteable.rewrite(request.getRewriteable(), context, assertAsyncActions);
+            Rewriteable.rewrite(request.getRewriteable(), context, true);
             assert searchContext.getQueryShardContext().isCacheable();
             success = true;
         } finally {
@@ -649,7 +611,7 @@ private DefaultSearchContext createSearchContext(ShardSearchRequest request, Tim
                 if (searchContext == null) {
                     // we handle the case where the DefaultSearchContext constructor throws an exception since we would otherwise
                     // leak a searcher and this can have severe implications (unable to obtain shard lock exceptions).
-                    IOUtils.closeWhileHandlingException(searcher);
+                    IOUtils.closeWhileHandlingException(reader::decRef);
                 }
             }
         }
@@ -658,72 +620,41 @@ private DefaultSearchContext createSearchContext(ShardSearchRequest request, Tim
 
     private void freeAllContextForIndex(Index index) {
         assert index != null;
-        for (SearchContext ctx : activeContexts.values()) {
+        for (ReaderContext ctx : activeReaders.values()) {
             if (index.equals(ctx.indexShard().shardId().getIndex())) {
-                freeContext(ctx.id());
+                freeReaderContext(ctx.id());
             }
         }
     }
 
 
-    public boolean freeContext(long id) {
-        try (SearchContext context = removeContext(id)) {
-            if (context != null) {
-                onFreeContext(context);
-                return true;
-            }
-            return false;
+    public boolean freeReaderContext(long id) {
+        try (ReaderContext context = removeReaderContext(id)) {
+            return context != null;
         }
     }
 
-    private void onFreeContext(SearchContext context) {
-        assert context.refCount() > 0 : " refCount must be > 0: " + context.refCount();
-        assert activeContexts.containsKey(context.id()) == false;
-        context.indexShard().getSearchOperationListener().onFreeContext(context);
-        if (context.scrollContext() != null) {
-            openScrollContexts.decrementAndGet();
-            context.indexShard().getSearchOperationListener().onFreeScrollContext(context);
-        }
-    }
 
     public void freeAllScrollContexts() {
-        for (SearchContext searchContext : activeContexts.values()) {
-            if (searchContext.scrollContext() != null) {
-                freeContext(searchContext.id());
+        for (ReaderContext readerContext : activeReaders.values()) {
+            if (readerContext.scrollContext() != null) {
+                freeReaderContext(readerContext.id());
             }
         }
     }
 
-    private void contextScrollKeepAlive(SearchContext context, long keepAlive) {
+    private void contextScrollKeepAlive(ReaderContext readerContext, long keepAlive) {
         if (keepAlive > maxKeepAlive) {
             throw new IllegalArgumentException(
                 "Keep alive for scroll (" + TimeValue.timeValueMillis(keepAlive) + ") is too large. " +
                     "It must be less than (" + TimeValue.timeValueMillis(maxKeepAlive) + "). " +
                     "This limit can be set by changing the [" + MAX_KEEPALIVE_SETTING.getKey() + "] cluster level setting.");
         }
-        context.keepAlive(keepAlive);
-    }
-
-    private void contextProcessing(SearchContext context) {
-        // disable timeout while executing a search
-        context.accessed(-1);
-    }
-
-    private void contextProcessedSuccessfully(SearchContext context) {
-        context.accessed(threadPool.relativeTimeInMillis());
-    }
-
-    private void cleanContext(SearchContext context) {
-        try {
-            context.clearReleasables(Lifetime.PHASE);
-            context.setTask(null);
-        } finally {
-            context.decRef();
-        }
+        readerContext.keepAlive(keepAlive);
     }
 
-    private void processFailure(SearchContext context, Exception e) {
-        freeContext(context.id());
+    private void processFailure(ReaderContext context, Exception e) {
+        freeReaderContext(context.id());
         try {
             if (Lucene.isCorruptionException(e)) {
                 context.indexShard().failShard("search execution corruption failure", e);
@@ -734,7 +665,8 @@ private void processFailure(SearchContext context, Exception e) {
         }
     }
 
-    private void parseSource(DefaultSearchContext context, SearchSourceBuilder source) throws SearchException {
+    private void parseSource(ReaderContext reader, DefaultSearchContext context,
+                             SearchSourceBuilder source, boolean includeAggregations) throws SearchException {
         // nothing to parse...
         if (source == null) {
             return;
@@ -790,7 +722,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
             context.timeout(source.timeout());
         }
         context.terminateAfter(source.terminateAfter());
-        if (source.aggregations() != null) {
+        if (source.aggregations() != null && includeAggregations) {
             try {
                 AggregatorFactories factories = source.aggregations().build(queryShardContext, null);
                 context.aggregations(new SearchContextAggregations(factories, multiBucketConsumerService.create()));
@@ -805,10 +737,10 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
                 throw new SearchException(shardTarget, "failed to create SuggestionSearchContext", e);
             }
         }
-        if (source.rescores() != null) {
+        if (source.rescores() != null && reader.rescore().isEmpty()) {
             try {
                 for (RescorerBuilder<?> rescore : source.rescores()) {
-                    context.addRescore(rescore.buildContext(queryShardContext));
+                    reader.addRescore(rescore.buildContext(queryShardContext));
                 }
             } catch (IOException e) {
                 throw new SearchException(shardTarget, "failed to create RescoreSearchContext", e);
@@ -966,13 +898,13 @@ private void shortcutDocIdsToLoad(SearchContext context) {
         context.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length);
     }
 
-    private void processScroll(InternalScrollSearchRequest request, SearchContext context) {
+    private void processScroll(InternalScrollSearchRequest request, ReaderContext reader, SearchContext context) {
         // process scroll
         context.from(context.from() + context.size());
         context.scrollContext().scroll = request.scroll();
         // update the context keep alive based on the new scroll value
         if (request.scroll() != null && request.scroll().keepAlive() != null) {
-            contextScrollKeepAlive(context, request.scroll().keepAlive().millis());
+            contextScrollKeepAlive(reader, request.scroll().keepAlive().millis());
         }
     }
 
@@ -981,7 +913,7 @@ private void processScroll(InternalScrollSearchRequest request, SearchContext co
      * SearchService
      */
     public int getActiveContexts() {
-        return this.activeContexts.size();
+        return this.activeReaders.size();
     }
 
     public ResponseCollectorService getResponseCollectorService() {
@@ -992,7 +924,7 @@ class Reaper implements Runnable {
         @Override
         public void run() {
             final long time = threadPool.relativeTimeInMillis();
-            for (SearchContext context : activeContexts.values()) {
+            for (ReaderContext context : activeReaders.values()) {
                 // Use the same value for both checks since lastAccessTime can
                 // be modified by another thread between checks!
                 final long lastAccessTime = context.lastAccessTime();
@@ -1002,7 +934,7 @@ public void run() {
                 if ((time - lastAccessTime > context.keepAlive())) {
                     logger.debug("freeing search context [{}], time [{}], lastAccessTime [{}], keepAlive [{}]", context.id(), time,
                         lastAccessTime, context.keepAlive());
-                    freeContext(context.id());
+                    freeReaderContext(context.id());
                 }
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
index c150f99cfe196..fca5b14e34be3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
@@ -117,8 +117,6 @@ public void execute(SearchContext context) {
                 context.searcher().search(query, collector);
             } catch (Exception e) {
                 throw new QueryPhaseExecutionException(context.shardTarget(), "Failed to execute global aggregators", e);
-            } finally {
-                context.clearReleasables(SearchContext.Lifetime.COLLECTION);
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
index ea89700521798..43e38c731bbe1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContext.Lifetime;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
 
 import java.io.IOException;
@@ -76,7 +75,7 @@ protected AggregatorBase(String name, AggregatorFactories factories, SearchConte
         this.breakerService = context.bigArrays().breakerService();
         assert factories != null : "sub-factories provided to BucketAggregator must not be null, use AggragatorFactories.EMPTY instead";
         this.subAggregators = factories.createSubAggregators(context, this);
-        context.addReleasable(this, Lifetime.PHASE);
+        context.addReleasable(this);
         final SearchShardTarget shardTarget = context.shardTarget();
         // Register a safeguard to highlight any invalid construction logic (call to this constructor without subsequent preCollection call)
         collectableSubAggregators = new BucketCollector() {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
index 970ef725f027d..5451285a379c8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContext.Lifetime;
 
 import java.io.IOException;
 import java.util.List;
@@ -50,7 +49,7 @@ public static final class MultiBucketAggregatorWrapper extends Aggregator {
             this.parent = parent;
             this.factory = factory;
             this.first = first;
-            context.addReleasable(this, Lifetime.PHASE);
+            context.addReleasable(this);
             aggregators = bigArrays.newObjectArray(1);
             aggregators.set(0, first);
             collectors = bigArrays.newObjectArray(1);
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainFetchSubPhase.java
index 3a7d5c96b8179..345a4c5a0a7ad 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainFetchSubPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainFetchSubPhase.java
@@ -48,8 +48,6 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
         } catch (IOException e) {
             throw new FetchPhaseExecutionException(context.shardTarget(),
                 "Failed to explain doc [" + hitContext.hit().getId() + "]", e);
-        } finally {
-            context.clearReleasables(SearchContext.Lifetime.COLLECTION);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java
index c2f6980781dba..319bd40cfbd70 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContext.Lifetime;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -92,8 +91,6 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) {
             }
         } catch (IOException e) {
             throw ExceptionsHelper.convertToElastic(e);
-        } finally {
-            context.clearReleasables(Lifetime.COLLECTION);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index 8c04954a4efcf..ab605062de50a 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -140,21 +140,11 @@ public float queryBoost() {
         return in.queryBoost();
     }
 
-    @Override
-    public long getOriginNanoTime() {
-        return in.getOriginNanoTime();
-    }
-
     @Override
     public ScrollContext scrollContext() {
         return in.scrollContext();
     }
 
-    @Override
-    public SearchContext scrollContext(ScrollContext scroll) {
-        return in.scrollContext(scroll);
-    }
-
     @Override
     public SearchContextAggregations aggregations() {
         return in.aggregations();
@@ -195,11 +185,6 @@ public List<RescoreContext> rescore() {
         return in.rescore();
     }
 
-    @Override
-    public void addRescore(RescoreContext rescore) {
-        in.addRescore(rescore);
-    }
-
     @Override
     public boolean hasScriptFields() {
         return in.hasScriptFields();
@@ -451,26 +436,6 @@ public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int
         return in.docIdsToLoad(docIdsToLoad, docsIdsToLoadFrom, docsIdsToLoadSize);
     }
 
-    @Override
-    public void accessed(long accessTime) {
-        in.accessed(accessTime);
-    }
-
-    @Override
-    public long lastAccessTime() {
-        return in.lastAccessTime();
-    }
-
-    @Override
-    public long keepAlive() {
-        return in.keepAlive();
-    }
-
-    @Override
-    public void keepAlive(long keepAlive) {
-        in.keepAlive(keepAlive);
-    }
-
     @Override
     public SearchLookup lookup() {
         return in.lookup();
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
new file mode 100644
index 0000000000000..7c9dfe14dc9f8
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.search.rescore.RescoreContext;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Holds a reference to a point in time {@link Engine.Searcher} that will be used to construct {@link SearchContext}.
+ * This class also implements {@link org.elasticsearch.common.util.concurrent.RefCounted} since in some situations like
+ * in {@link org.elasticsearch.search.SearchService} a SearchContext can be closed concurrently due to independent events
+ * ie. when an index gets removed. To prevent accessing closed IndexReader / IndexSearcher instances the SearchContext
+ * can be guarded by a reference count and fail if it's been closed by an external event.
+ *
+ * For reference why we use RefCounted here see https://github.com/elastic/elasticsearch/pull/20095.
+ */
+public class ReaderContext extends AbstractRefCounted implements Releasable {
+    private final long id;
+    private final IndexShard indexShard;
+    private final Engine.Searcher engineSearcher;
+    private final AtomicBoolean closed = new AtomicBoolean(false);
+
+    private volatile long keepAlive;
+    private volatile long lastAccessTime = -1L;
+
+    // BWC
+    private final ShardSearchRequest request;
+    private final ScrollContext scrollContext;
+    private AggregatedDfs aggregatedDfs;
+    private List<RescoreContext> rescore;
+
+    private final List<Releasable> onCloses = new CopyOnWriteArrayList<>();
+
+    public ReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher, ShardSearchRequest request) {
+        super("reader_context");
+        this.id = id;
+        this.indexShard = indexShard;
+        this.engineSearcher = engineSearcher;
+        this.request = request;
+        if (request.scroll() != null) {
+            this.scrollContext = new ScrollContext();
+        } else {
+            this.scrollContext = null;
+        }
+    }
+
+    @Override
+    public final void close() {
+        if (closed.compareAndSet(false, true)) {
+            decRef();
+        } else {
+            assert false : "ReaderContext was closed already";
+        }
+    }
+
+    @Override
+    protected void closeInternal() {
+        Releasables.close(Releasables.wrap(onCloses), engineSearcher);
+    }
+
+    public void addOnClose(Releasable releasable) {
+        onCloses.add(releasable);
+    }
+
+    public long id() {
+        return id;
+    }
+
+    public IndexShard indexShard() {
+        return indexShard;
+    }
+
+    public Engine.Searcher engineSearcher() {
+        return engineSearcher;
+    }
+
+    public String source() {
+        return engineSearcher.source();
+    }
+
+    public ShardSearchRequest request() {
+        return request;
+    }
+
+    public ScrollContext scrollContext() {
+        return scrollContext;
+    }
+
+    public AggregatedDfs aggregatedDfs() {
+        return aggregatedDfs;
+    }
+
+    public void aggregatedDfs(AggregatedDfs aggregatedDfs) {
+        this.aggregatedDfs = aggregatedDfs;
+    }
+
+    public void accessed(long accessTime) {
+        this.lastAccessTime = accessTime;
+    }
+
+    public long lastAccessTime() {
+        return this.lastAccessTime;
+    }
+
+    public long keepAlive() {
+        return this.keepAlive;
+    }
+
+    public void keepAlive(long keepAlive) {
+        this.keepAlive = keepAlive;
+    }
+
+    public List<RescoreContext> rescore() {
+        if (rescore == null) {
+            return Collections.emptyList();
+        } else {
+            return Collections.unmodifiableList(rescore);
+        }
+    }
+
+    public void addRescore(RescoreContext rescore) {
+        if (this.rescore == null) {
+            this.rescore = new ArrayList<>();
+        }
+        this.rescore.add(rescore);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java b/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
index 41d7680a780b0..440408646d127 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
@@ -36,6 +36,8 @@ public final class ScrollContext {
     public ScoreDoc lastEmittedDoc;
     public Scroll scroll;
 
+    private final long startTimeInNano = System.nanoTime();
+
     /**
      * Returns the object or <code>null</code> if the given key does not have a
      * value in the context
@@ -54,4 +56,9 @@ public void putInContext(String key, Object value) {
         }
         context.put(key, value);
     }
+
+
+    public long getStartTimeInNano() {
+        return startTimeInNano;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index c0477ece69aa0..d0eddac1d86c6 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -29,9 +29,6 @@
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
-import org.elasticsearch.common.util.concurrent.RefCounted;
-import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -61,36 +58,29 @@
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
-import java.util.ArrayList;
-import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * This class encapsulates the state needed to execute a search. It holds a reference to the
  * shards point in time snapshot (IndexReader / ContextIndexSearcher) and allows passing on
  * state from one query / fetch phase to another.
- *
- * This class also implements {@link RefCounted} since in some situations like in {@link org.elasticsearch.search.SearchService}
- * a SearchContext can be closed concurrently due to independent events ie. when an index gets removed. To prevent accessing closed
- * IndexReader / IndexSearcher instances the SearchContext can be guarded by a reference count and fail if it's been closed by
- * an external event.
  */
-// For reference why we use RefCounted here see #20095
-public abstract class SearchContext extends AbstractRefCounted implements Releasable {
+public abstract class SearchContext implements Releasable {
 
     public static final int DEFAULT_TERMINATE_AFTER = 0;
     public static final int TRACK_TOTAL_HITS_ACCURATE = Integer.MAX_VALUE;
     public static final int TRACK_TOTAL_HITS_DISABLED = -1;
     public static final int DEFAULT_TRACK_TOTAL_HITS_UP_TO = 10000;
 
-    private Map<Lifetime, List<Releasable>> clearables = null;
+    private final List<Releasable> releasables = new CopyOnWriteArrayList<>();
     private final AtomicBoolean closed = new AtomicBoolean(false);
     private InnerHitsContext innerHitsContext;
 
     protected SearchContext() {
-        super("search_context");
+
     }
 
     public abstract void setTask(SearchShardTask task);
@@ -101,25 +91,15 @@ protected SearchContext() {
 
     @Override
     public final void close() {
-        if (closed.compareAndSet(false, true)) { // prevent double closing
-            decRef();
-        }
-    }
-
-    @Override
-    protected final void closeInternal() {
-        try {
-            clearReleasables(Lifetime.CONTEXT);
-        } finally {
-            doClose();
+        if (closed.compareAndSet(false, true)) {
+            try {
+                Releasables.close(releasables);
+            } finally {
+                doClose();
+            }
         }
     }
 
-    @Override
-    protected void alreadyClosed() {
-        throw new IllegalStateException("search context is already closed can't increment refCount current count [" + refCount() + "]");
-    }
-
     protected abstract void doClose();
 
     /**
@@ -146,12 +126,8 @@ protected void alreadyClosed() {
 
     public abstract float queryBoost();
 
-    public abstract long getOriginNanoTime();
-
     public abstract ScrollContext scrollContext();
 
-    public abstract SearchContext scrollContext(ScrollContext scroll);
-
     public abstract SearchContextAggregations aggregations();
 
     public abstract SearchContext aggregations(SearchContextAggregations aggregations);
@@ -180,8 +156,6 @@ public InnerHitsContext innerHits() {
      */
     public abstract List<RescoreContext> rescore();
 
-    public abstract void addRescore(RescoreContext rescore);
-
     public abstract boolean hasScriptFields();
 
     public abstract ScriptFieldsContext scriptFields();
@@ -322,14 +296,6 @@ public InnerHitsContext innerHits() {
 
     public abstract SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int docsIdsToLoadSize);
 
-    public abstract void accessed(long accessTime);
-
-    public abstract long lastAccessTime();
-
-    public abstract long keepAlive();
-
-    public abstract void keepAlive(long keepAlive);
-
     public SearchLookup lookup() {
         return getQueryShardContext().lookup();
     }
@@ -347,38 +313,14 @@ public SearchLookup lookup() {
      */
     public abstract Profilers getProfilers();
 
+
     /**
-     * Schedule the release of a resource. The time when {@link Releasable#close()} will be called on this object
-     * is function of the provided {@link Lifetime}.
+     * Adds a releasable that will be freed when this context is closed.
      */
-    public void addReleasable(Releasable releasable, Lifetime lifetime) {
-        if (clearables == null) {
-            clearables = new EnumMap<>(Lifetime.class);
-        }
-        List<Releasable> releasables = clearables.get(lifetime);
-        if (releasables == null) {
-            releasables = new ArrayList<>();
-            clearables.put(lifetime, releasables);
-        }
+    public void addReleasable(Releasable releasable) {
         releasables.add(releasable);
     }
 
-    public void clearReleasables(Lifetime lifetime) {
-        if (clearables != null) {
-            List<List<Releasable>>releasables = new ArrayList<>();
-            for (Lifetime lc : Lifetime.values()) {
-                if (lc.compareTo(lifetime) > 0) {
-                    break;
-                }
-                List<Releasable> remove = clearables.remove(lc);
-                if (remove != null) {
-                    releasables.add(remove);
-                }
-            }
-            Releasables.close(Iterables.flatten(releasables));
-        }
-    }
-
     /**
      * @return true if the request contains only suggest
      */
@@ -403,24 +345,6 @@ public final boolean hasOnlySuggest() {
     /** Return a view of the additional query collectors that should be run for this context. */
     public abstract Map<Class<?>, Collector> queryCollectors();
 
-    /**
-     * The life time of an object that is used during search execution.
-     */
-    public enum Lifetime {
-        /**
-         * This life time is for objects that only live during collection time.
-         */
-        COLLECTION,
-        /**
-         * This life time is for objects that need to live until the end of the current search phase.
-         */
-        PHASE,
-        /**
-         * This life time is for objects that need to live until the search context they are attached to is destroyed.
-         */
-        CONTEXT
-    }
-
     public abstract QueryShardContext getQueryShardContext();
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
index e918aed603750..8ce36550f7b94 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
 import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
@@ -85,11 +84,6 @@ public Query buildFilteredQuery(Query query) {
         throw new UnsupportedOperationException("this context should be read only");
     }
 
-    @Override
-    public SearchContext scrollContext(ScrollContext scrollContext) {
-        throw new UnsupportedOperationException("Not supported");
-    }
-
     @Override
     public SearchContext aggregations(SearchContextAggregations aggregations) {
         throw new UnsupportedOperationException("Not supported");
@@ -110,11 +104,6 @@ public void suggest(SuggestionSearchContext suggest) {
         throw new UnsupportedOperationException("Not supported");
     }
 
-    @Override
-    public void addRescore(RescoreContext rescore) {
-        throw new UnsupportedOperationException("Not supported");
-    }
-
     @Override
     public boolean hasScriptFields() {
         return scriptFields != null;
@@ -332,16 +321,6 @@ public CollapseContext collapse() {
         return null;
     }
 
-    @Override
-    public void accessed(long accessTime) {
-        throw new UnsupportedOperationException("Not supported");
-    }
-
-    @Override
-    public void keepAlive(long keepAlive) {
-        throw new UnsupportedOperationException("Not supported");
-    }
-
     @Override
     public QuerySearchResult queryResult() {
         return querySearchResult;
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index ba5ae725cab6f..afcd7355f155f 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -343,8 +343,6 @@ private static boolean searchWithCollector(SearchContext searchContext, ContextI
                 throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded");
             }
             queryResult.searchTimedOut(true);
-        } finally {
-            searchContext.clearReleasables(SearchContext.Lifetime.COLLECTION);
         }
         if (searchContext.terminateAfter() != SearchContext.DEFAULT_TERMINATE_AFTER && queryResult.terminatedEarly() == null) {
             queryResult.terminatedEarly(false);
@@ -399,8 +397,6 @@ private static boolean searchWithCollectorManager(SearchContext searchContext, C
                 throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded");
             }
             searchContext.queryResult().searchTimedOut(true);
-        } finally {
-            searchContext.clearReleasables(SearchContext.Lifetime.COLLECTION);
         }
         return false; // no rescoring when sorting by field
     }
diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 2c11b26580f3d..18b9b5f4f6d58 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -78,11 +78,10 @@
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.test.engine.MockEngineFactory;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -103,6 +102,7 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.Mockito.mock;
 
 public class IndexModuleTests extends ESTestCase {
     private Index index;
@@ -272,9 +272,8 @@ public void testAddSearchOperationListener() throws IOException {
         IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         AtomicBoolean executed = new AtomicBoolean(false);
         SearchOperationListener listener = new SearchOperationListener() {
-
             @Override
-            public void onNewContext(SearchContext context) {
+            public void onNewReaderContext(ReaderContext readerContext) {
                 executed.set(true);
             }
         };
@@ -287,9 +286,8 @@ public void onNewContext(SearchContext context) {
         assertEquals(2, indexService.getSearchOperationListener().size());
         assertEquals(SearchSlowLog.class, indexService.getSearchOperationListener().get(0).getClass());
         assertSame(listener, indexService.getSearchOperationListener().get(1));
-
         for (SearchOperationListener l : indexService.getSearchOperationListener()) {
-            l.onNewContext(new TestSearchContext(null));
+            l.onNewReaderContext(mock(ReaderContext.class));
         }
         assertTrue(executed.get());
         indexService.close("simon says", false);
diff --git a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
index 28bab8da0fdfb..34c528dfeda6a 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.index.shard;
 
+import org.elasticsearch.search.internal.ReaderContext;
+import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
@@ -33,6 +35,7 @@
 
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.mock;
 
 public class SearchOperationListenerTests extends ESTestCase {
 
@@ -90,26 +93,26 @@ public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
             }
 
             @Override
-            public void onNewContext(SearchContext context) {
-                assertNotNull(context);
+            public void onNewReaderContext(ReaderContext readerContext) {
+                assertNotNull(readerContext);
                 newContext.incrementAndGet();
             }
 
             @Override
-            public void onFreeContext(SearchContext context) {
-                assertNotNull(context);
+            public void onFreeReaderContext(ReaderContext readerContext) {
+                assertNotNull(readerContext);
                 freeContext.incrementAndGet();
             }
 
             @Override
-            public void onNewScrollContext(SearchContext context) {
-                assertNotNull(context);
+            public void onNewScrollContext(ScrollContext scrollContext) {
+                assertNotNull(scrollContext);
                 newScrollContext.incrementAndGet();
             }
 
             @Override
-            public void onFreeScrollContext(SearchContext context) {
-                assertNotNull(context);
+            public void onFreeScrollContext(ScrollContext scrollContext) {
+                assertNotNull(scrollContext);
                 freeScrollContext.incrementAndGet();
             }
 
@@ -216,7 +219,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onNewContext(ctx);
+        compositeListener.onNewReaderContext(mock(ReaderContext.class));
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -229,7 +232,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onNewScrollContext(ctx);
+        compositeListener.onNewScrollContext(new ScrollContext());
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -242,7 +245,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onFreeContext(ctx);
+        compositeListener.onFreeReaderContext(mock(ReaderContext.class));
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -255,7 +258,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onFreeScrollContext(ctx);
+        compositeListener.onFreeScrollContext(new ScrollContext());
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 560f052664604..198daf0f41a11 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -50,7 +50,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.ScrollContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.slice.SliceBuilder;
@@ -111,26 +111,33 @@ public void testPreProcess() throws Exception {
 
         try (Directory dir = newDirectory();
              RandomIndexWriter w = new RandomIndexWriter(random(), dir);
-             IndexReader reader = w.getReader();
-             Engine.Searcher searcher = new Engine.Searcher("test", reader,
-                 IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
-                 IndexSearcher.getDefaultQueryCachingPolicy(), reader)) {
+             IndexReader reader = w.getReader()) {
+
+            final Engine.Searcher engineSearcher = new Engine.Searcher("test", w.getReader(),
+                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
+                IndexSearcher.getDefaultQueryCachingPolicy(), reader);
 
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
-            DefaultSearchContext context1 = new DefaultSearchContext(1L, shardSearchRequest, target, searcher, null, indexService,
-                indexShard, bigArrays, null, timeout, null);
-            context1.from(300);
+            ReaderContext readerWithoutScroll = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            DefaultSearchContext contextWithoutScroll = new DefaultSearchContext(readerWithoutScroll, shardSearchRequest, target, null,
+                indexService, indexShard, bigArrays, null, timeout, null);
+            contextWithoutScroll.from(300);
+            contextWithoutScroll.close();
 
             // resultWindow greater than maxResultWindow and scrollContext is null
-            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
+            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> contextWithoutScroll.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Result window is too large, from + size must be less than or equal to:"
                 + " [" + maxResultWindow + "] but was [310]. See the scroll api for a more efficient way to request large data sets. "
                 + "This limit can be set by changing the [" + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey()
                 + "] index level setting."));
 
             // resultWindow greater than maxResultWindow and scrollContext isn't null
-            context1.scrollContext(new ScrollContext());
+            when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMillis(randomInt(1000))));
+            ReaderContext readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            DefaultSearchContext context1 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null, indexService,
+                indexShard, bigArrays, null, timeout, null);
+            context1.from(300);
             exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Batch size is too large, size must be less than or equal to: ["
                 + maxResultWindow + "] but was [310]. Scroll batch sizes cost as much memory as result windows so they are "
@@ -144,7 +151,7 @@ public void testPreProcess() throws Exception {
 
             RescoreContext rescoreContext = mock(RescoreContext.class);
             when(rescoreContext.getWindowSize()).thenReturn(500);
-            context1.addRescore(rescoreContext);
+            readerContext.addRescore(rescoreContext);
 
             exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Cannot use [sort] option in conjunction with [rescore]."));
@@ -158,8 +165,10 @@ public void testPreProcess() throws Exception {
                 + "to be rescored. This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey()
                 + "] index level setting."));
 
+            readerContext.close();
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
             // rescore is null but sliceBuilder is not null
-            DefaultSearchContext context2 = new DefaultSearchContext(2L, shardSearchRequest, target, searcher,
+            DefaultSearchContext context2 = new DefaultSearchContext(readerContext, shardSearchRequest, target,
                 null, indexService, indexShard, bigArrays, null, timeout, null);
 
             SliceBuilder sliceBuilder = mock(SliceBuilder.class);
@@ -176,7 +185,7 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.getAliasFilter()).thenReturn(AliasFilter.EMPTY);
             when(shardSearchRequest.indexBoost()).thenReturn(AbstractQueryBuilder.DEFAULT_BOOST);
 
-            DefaultSearchContext context3 = new DefaultSearchContext(3L, shardSearchRequest, target, searcher, null,
+            DefaultSearchContext context3 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
                 indexService, indexShard, bigArrays, null, timeout, null);
             ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery();
             context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false);
@@ -186,7 +195,9 @@ public void testPreProcess() throws Exception {
             when(queryShardContext.fieldMapper(anyString())).thenReturn(mock(MappedFieldType.class));
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
-            DefaultSearchContext context4 = new DefaultSearchContext(4L, shardSearchRequest, target, searcher, null,
+            readerContext.close();
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            DefaultSearchContext context4 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
                 indexService, indexShard, bigArrays, null, timeout, null);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
             Query query1 = context4.query();
@@ -194,6 +205,7 @@ public void testPreProcess() throws Exception {
             Query query2 = context4.query();
             assertTrue(query1 instanceof MatchNoDocsQuery || query2 instanceof MatchNoDocsQuery);
 
+            readerContext.close();
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 5691530dd32fe..c47ccefca3217 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -47,6 +47,7 @@
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
@@ -75,6 +76,7 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.suggest.SuggestBuilder;
@@ -167,8 +169,8 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
         public void onIndexModule(IndexModule indexModule) {
             indexModule.addSearchOperationListener(new SearchOperationListener() {
                 @Override
-                public void onNewContext(SearchContext context) {
-                    if ("throttled_threadpool_index".equals(context.indexShard().shardId().getIndex().getName())) {
+                public void onNewReaderContext(ReaderContext readerContext) {
+                    if ("throttled_threadpool_index".equals(readerContext.indexShard().shardId().getIndex().getName())) {
                         assertThat(Thread.currentThread().getName(), startsWith("elasticsearch[node_s_0][search_throttled]"));
                     } else {
                         assertThat(Thread.currentThread().getName(), startsWith("elasticsearch[node_s_0][search]"));
@@ -322,7 +324,7 @@ public void onFailure(Exception e) {
                         service.executeFetchPhase(req, new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()), listener);
                         listener.get();
                         if (useScroll) {
-                            service.freeContext(searchPhaseResult.getRequestId());
+                            service.freeReaderContext(searchPhaseResult.getRequestId());
                         }
                     } catch (ExecutionException ex) {
                         assertThat(ex.getCause(), instanceOf(RuntimeException.class));
@@ -331,7 +333,7 @@ public void onFailure(Exception e) {
                 } catch (AlreadyClosedException ex) {
                     throw ex;
                 } catch (IllegalStateException ex) {
-                    assertEquals("search context is already closed can't increment refCount current count [0]", ex.getMessage());
+                    assertEquals("reader_context is already closed can't increment refCount current count [0]", ex.getMessage());
                 } catch (SearchContextMissingException ex) {
                     // that's fine
                 }
@@ -357,42 +359,34 @@ public void testTimeout() throws IOException {
         final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
         final IndexShard indexShard = indexService.getShard(0);
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
-        final SearchContext contextWithDefaultTimeout = service.createContext(
-                new ShardSearchRequest(
-                        OriginalIndices.NONE,
-                        searchRequest,
-                        indexShard.shardId(),
-                        1,
-                        new AliasFilter(null, Strings.EMPTY_ARRAY),
-                        1.0f, -1, null, null)
-        );
-        try {
+        final ShardSearchRequest requestWithDefaultTimeout = new ShardSearchRequest(
+            OriginalIndices.NONE,
+            searchRequest,
+            indexShard.shardId(),
+            1,
+            new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1.0f, -1, null, null);
+
+        try (ReaderContext reader = createReaderContext(indexShard, requestWithDefaultTimeout);
+             SearchContext contextWithDefaultTimeout = service.createContext(reader, requestWithDefaultTimeout, null, randomBoolean())) {
             // the search context should inherit the default timeout
             assertThat(contextWithDefaultTimeout.timeout(), equalTo(TimeValue.timeValueSeconds(5)));
-        } finally {
-            contextWithDefaultTimeout.decRef();
-            service.freeContext(contextWithDefaultTimeout.id());
         }
 
         final long seconds = randomIntBetween(6, 10);
         searchRequest.source(new SearchSourceBuilder().timeout(TimeValue.timeValueSeconds(seconds)));
-        final SearchContext context = service.createContext(
-                new ShardSearchRequest(
-                        OriginalIndices.NONE,
-                        searchRequest,
-                        indexShard.shardId(),
-                        1,
-                        new AliasFilter(null, Strings.EMPTY_ARRAY),
-                        1.0f, -1, null, null)
-        );
-        try {
+        final ShardSearchRequest requestWithCustomTimeout = new ShardSearchRequest(
+            OriginalIndices.NONE,
+            searchRequest,
+            indexShard.shardId(),
+            1,
+            new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1.0f, -1, null, null);
+        try (ReaderContext reader = createReaderContext(indexShard, requestWithCustomTimeout);
+             SearchContext context = service.createContext(reader, requestWithCustomTimeout, null, randomBoolean())) {
             // the search context should inherit the query timeout
             assertThat(context.timeout(), equalTo(TimeValue.timeValueSeconds(seconds)));
-        } finally {
-            context.decRef();
-            service.freeContext(context.id());
         }
-
     }
 
     /**
@@ -412,19 +406,20 @@ public void testMaxDocvalueFieldsSearch() throws IOException {
         for (int i = 0; i < indexService.getIndexSettings().getMaxDocvalueFields(); i++) {
             searchSourceBuilder.docValueField("field" + i);
         }
-        try (SearchContext context = service.createContext(
-            new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
-                new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null))
-        ) {
+        final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
+            new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
+        try (ReaderContext reader = createReaderContext(indexShard, request);
+             SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
             assertNotNull(context);
-            searchSourceBuilder.docValueField("one_field_too_much");
+        }
+        searchSourceBuilder.docValueField("one_field_too_much");
+        try (ReaderContext reader = createReaderContext(indexShard, request)) {
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                    () -> service.createContext(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
-                        new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null)));
+                () -> service.createContext(reader, request, null, randomBoolean()));
             assertEquals(
-                    "Trying to retrieve too many docvalue_fields. Must be less than or equal to: [100] but was [101]. "
-                            + "This limit can be set by changing the [index.max_docvalue_fields_search] index level setting.",
-                    ex.getMessage());
+                "Trying to retrieve too many docvalue_fields. Must be less than or equal to: [100] but was [101]. "
+                    + "This limit can be set by changing the [index.max_docvalue_fields_search] index level setting.",
+                ex.getMessage());
         }
     }
 
@@ -447,20 +442,22 @@ public void testMaxScriptFieldsSearch() throws IOException {
             searchSourceBuilder.scriptField("field" + i,
                     new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
         }
-        try (SearchContext context = service.createContext(new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
-                indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY),
-                1.0f, -1, null, null))) {
-            assertNotNull(context);
+        final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
+            indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
+
+        try(ReaderContext reader = createReaderContext(indexShard, request)) {
+            try (SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
+                assertNotNull(context);
+            }
             searchSourceBuilder.scriptField("anotherScriptField",
-                    new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
+                new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                    () -> service.createContext(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
-                        new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null)));
+                () -> service.createContext(reader, request, null, randomBoolean()));
             assertEquals(
-                    "Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxScriptFields + "] but was ["
-                            + (maxScriptFields + 1)
-                            + "]. This limit can be set by changing the [index.max_script_fields] index level setting.",
-                    ex.getMessage());
+                "Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxScriptFields + "] but was ["
+                    + (maxScriptFields + 1)
+                    + "]. This limit can be set by changing the [index.max_script_fields] index level setting.",
+                ex.getMessage());
         }
     }
 
@@ -477,10 +474,12 @@ public void testIgnoreScriptfieldIfSizeZero() throws IOException {
         searchSourceBuilder.scriptField("field" + 0,
                 new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
         searchSourceBuilder.size(0);
-        try (SearchContext context = service.createContext(new ShardSearchRequest(OriginalIndices.NONE,
+        final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE,
             searchRequest, indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY),
-                1.0f, -1, null, null))) {
-                assertEquals(0, context.scriptFields().fields().size());
+            1.0f, -1, null, null);
+        try (ReaderContext reader = createReaderContext(indexShard, request);
+             SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
+            assertEquals(0, context.scriptFields().fields().size());
         }
     }
 
@@ -514,7 +513,7 @@ public void testMaxOpenScrollContexts() throws RuntimeException {
         }
 
         ElasticsearchException ex = expectThrows(ElasticsearchException.class,
-            () -> service.createAndPutContext(new ShardScrollRequestTest(indexShard.shardId())));
+            () -> service.createAndPutReaderContext(new ShardScrollRequestTest(indexShard.shardId())));
         assertEquals(
             "Trying to create too many scroll contexts. Must be less than or equal to: [" +
                 SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY) + "]. " +
@@ -626,10 +625,11 @@ public void testCanMatch() throws IOException {
         assertEquals(numWrapReader, numWrapInvocations.get());
 
         // make sure that the wrapper is called when the context is actually created
-        service.createContext(new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
-            indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY),
-            1f, -1, null, null)).close();
-        assertEquals(numWrapReader+1, numWrapInvocations.get());
+        try (ReaderContext reader = createReaderContext(indexShard, new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
+            indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null));
+             SearchContext context = service.createContext(reader, reader.request(), null, randomBoolean())) {
+        }
+        assertEquals(numWrapReader + 1, numWrapInvocations.get());
     }
 
     public void testCanRewriteToMatchNone() {
@@ -744,18 +744,23 @@ public void testCreateSearchContextFailure() throws IOException {
         final IndexService indexService = createIndex(index);
         final SearchService service = getInstanceFromNode(SearchService.class);
         final ShardId shardId = new ShardId(indexService.index(), 0);
-
-        NullPointerException e = expectThrows(NullPointerException.class,
-            () -> service.createContext(
-                new ShardSearchRequest(shardId, 0, null) {
-                    @Override
-                    public SearchType searchType() {
-                        // induce an artificial NPE
-                        throw new NullPointerException("expected");
-                    }
-                }
-            ));
-        assertEquals("expected", e.getMessage());
+        final ShardSearchRequest request = new ShardSearchRequest(shardId, 0, null) {
+            @Override
+            public SearchType searchType() {
+                // induce an artificial NPE
+                throw new NullPointerException("expected");
+            }
+        };
+        try (ReaderContext reader = createReaderContext(indexService.getShard(shardId.id()), request)) {
+            NullPointerException e = expectThrows(NullPointerException.class,
+                () -> service.createContext(reader, request, null, randomBoolean()));
+            assertEquals("expected", e.getMessage());
+        }
         assertEquals("should have 2 store refs (IndexService + InternalEngine)", 2, indexService.getShard(0).store().refCount());
     }
+
+    private ReaderContext createReaderContext(IndexShard shard, ShardSearchRequest request) {
+        Engine.Searcher searcher = shard.acquireSearcher("test");
+        return new ReaderContext(randomNonNegativeLong(), shard, searcher, request);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
index 4c33177e268f9..b2d4314b3a0a5 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
@@ -316,13 +316,12 @@ public void testInOrderScrollOptimization() throws Exception {
         }
         w.close();
         IndexReader reader = DirectoryReader.open(dir);
-        TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader));
-        context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
         ScrollContext scrollContext = new ScrollContext();
+        TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader), scrollContext);
+        context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
         scrollContext.lastEmittedDoc = null;
         scrollContext.maxScore = Float.NaN;
         scrollContext.totalHits = null;
-        context.scrollContext(scrollContext);
         context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()));
         int size = randomIntBetween(2, 5);
         context.setSize(size);
@@ -545,13 +544,12 @@ public void testIndexSortScrollOptimization() throws Exception {
         // search sort is a prefix of the index sort
         searchSortAndFormats.add(new SortAndFormats(new Sort(indexSort.getSort()[0]), new DocValueFormat[]{DocValueFormat.RAW}));
         for (SortAndFormats searchSortAndFormat : searchSortAndFormats) {
-            TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader));
-            context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
             ScrollContext scrollContext = new ScrollContext();
+            TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader), scrollContext);
+            context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
             scrollContext.lastEmittedDoc = null;
             scrollContext.maxScore = Float.NaN;
             scrollContext.totalHits = null;
-            context.scrollContext(scrollContext);
             context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()));
             context.setSize(10);
             context.sort(searchSortAndFormat);
diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
index c1ebb1213495b..554f47814a8ab 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.fetch.FetchPhase;
-import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.HashMap;
@@ -39,11 +39,11 @@ public class MockSearchService extends SearchService {
      */
     public static class TestPlugin extends Plugin {}
 
-    private static final Map<SearchContext, Throwable> ACTIVE_SEARCH_CONTEXTS = new ConcurrentHashMap<>();
+    private static final Map<ReaderContext, Throwable> ACTIVE_SEARCH_CONTEXTS = new ConcurrentHashMap<>();
 
     /** Throw an {@link AssertionError} if there are still in-flight contexts. */
     public static void assertNoInFlightContext() {
-        final Map<SearchContext, Throwable> copy = new HashMap<>(ACTIVE_SEARCH_CONTEXTS);
+        final Map<ReaderContext, Throwable> copy = new HashMap<>(ACTIVE_SEARCH_CONTEXTS);
         if (copy.isEmpty() == false) {
             throw new AssertionError(
                     "There are still [" + copy.size()
@@ -55,14 +55,14 @@ public static void assertNoInFlightContext() {
     /**
      * Add an active search context to the list of tracked contexts. Package private for testing.
      */
-    static void addActiveContext(SearchContext context) {
+    static void addActiveContext(ReaderContext context) {
         ACTIVE_SEARCH_CONTEXTS.put(context, new RuntimeException(context.toString()));
     }
 
     /**
      * Clear an active search context from the list of tracked contexts. Package private for testing.
      */
-    static void removeActiveContext(SearchContext context) {
+    static void removeActiveContext(ReaderContext context) {
         ACTIVE_SEARCH_CONTEXTS.remove(context);
     }
 
@@ -73,14 +73,14 @@ public MockSearchService(ClusterService clusterService,
     }
 
     @Override
-    protected void putContext(SearchContext context) {
+    protected void putReaderContext(ReaderContext context) {
         addActiveContext(context);
-        super.putContext(context);
+        super.putReaderContext(context);
     }
 
     @Override
-    protected SearchContext removeContext(long id) {
-        final SearchContext removed = super.removeContext(id);
+    protected ReaderContext removeReaderContext(long id) {
+        final ReaderContext removed = super.removeReaderContext(id);
         if (removed != null) {
             removeActiveContext(removed);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index 7c02fb2481f4c..705912da3a8f5 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -248,7 +248,7 @@ public boolean shouldCache(Query query) {
              * close their sub-aggregations. This is fairly similar to what the production code does. */
             releasables.add((Releasable) invocation.getArguments()[0]);
             return null;
-        }).when(searchContext).addReleasable(anyObject(), anyObject());
+        }).when(searchContext).addReleasable(anyObject());
         return searchContext;
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index 4edb9adacd13b..fc9a8fe7efa62 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -110,12 +110,18 @@ public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexSh
     }
 
     public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexShard, ContextIndexSearcher searcher) {
+        this(queryShardContext, indexShard, searcher, null);
+    }
+
+    public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexShard,
+                             ContextIndexSearcher searcher, ScrollContext scrollContext) {
         this.bigArrays = null;
         this.indexService = null;
         this.fixedBitSetFilterCache = null;
         this.indexShard = indexShard;
         this.queryShardContext = queryShardContext;
         this.searcher = searcher;
+        this.scrollContext = scrollContext;
     }
 
     public void setSearcher(ContextIndexSearcher searcher) {
@@ -166,22 +172,11 @@ public float queryBoost() {
         return 0;
     }
 
-    @Override
-    public long getOriginNanoTime() {
-        return originNanoTime;
-    }
-
     @Override
     public ScrollContext scrollContext() {
         return scrollContext;
     }
 
-    @Override
-    public SearchContext scrollContext(ScrollContext scrollContext) {
-        this.scrollContext = scrollContext;
-        return this;
-    }
-
     @Override
     public SearchContextAggregations aggregations() {
         return aggregations;
@@ -226,10 +221,6 @@ public List<RescoreContext> rescore() {
         return Collections.emptyList();
     }
 
-    @Override
-    public void addRescore(RescoreContext rescore) {
-    }
-
     @Override
     public boolean hasScriptFields() {
         return false;
@@ -536,24 +527,6 @@ public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int
         return null;
     }
 
-    @Override
-    public void accessed(long accessTime) {
-    }
-
-    @Override
-    public long lastAccessTime() {
-        return 0;
-    }
-
-    @Override
-    public long keepAlive() {
-        return 0;
-    }
-
-    @Override
-    public void keepAlive(long keepAlive) {
-    }
-
     @Override
     public DfsSearchResult dfsResult() {
         return null;
diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
index 8a8842487f14a..b49f1fb0acaf5 100644
--- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
@@ -19,59 +19,25 @@
 
 package org.elasticsearch.search;
 
-import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.TestSearchContext;
+
+import static org.mockito.Mockito.mock;
 
 public class MockSearchServiceTests extends ESTestCase {
-    public static final IndexMetaData EMPTY_INDEX_METADATA = IndexMetaData.builder("")
-        .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
-        .numberOfShards(1).numberOfReplicas(0).build();
 
     public void testAssertNoInFlightContext() {
-        final long nowInMillis = randomNonNegativeLong();
-        SearchContext s = new TestSearchContext(new QueryShardContext(0,
-            new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null,
-            xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null)) {
-
-            @Override
-            public SearchShardTarget shardTarget() {
-                return new SearchShardTarget("node", new ShardId("idx", "ignored", 0), null, OriginalIndices.NONE);
-            }
-
-            @Override
-            public SearchType searchType() {
-                return SearchType.DEFAULT;
-            }
-
-            @Override
-            public Query query() {
-                return Queries.newMatchAllQuery();
-            }
-        };
-        MockSearchService.addActiveContext(s);
+        ReaderContext reader = mock(ReaderContext.class);
+        MockSearchService.addActiveContext(reader);
         try {
             Throwable e = expectThrows(AssertionError.class, () -> MockSearchService.assertNoInFlightContext());
             assertEquals("There are still [1] in-flight contexts. The first one's creation site is listed as the cause of this exception.",
                     e.getMessage());
             e = e.getCause();
-            // The next line with throw an exception if the date looks wrong
-            assertEquals("[node][idx][0] query=[*:*]", e.getMessage());
             assertEquals(MockSearchService.class.getName(), e.getStackTrace()[0].getClassName());
             assertEquals(MockSearchServiceTests.class.getName(), e.getStackTrace()[1].getClassName());
         } finally {
-            MockSearchService.removeActiveContext(s);
+            MockSearchService.removeActiveContext(reader);
         }
     }
 }
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index 5a22857494557..e78e937a1f142 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.shard.SearchOperationListener;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.transport.TransportRequest;
 
@@ -285,27 +286,34 @@ public void validateSearchContext(SearchContext context, TransportRequest transp
                 } catch (IOException e) {
                     throw new UncheckedIOException(e);
                 }
-                // also register a release resource in this case if we have multiple roundtrips like in DFS
-                registerRelease(context, lazyDirectoryReader);
             }
         }
 
-        private void registerRelease(SearchContext context, LazyDirectoryReader lazyDirectoryReader) {
-            context.addReleasable(() -> {
+        @Override
+        public void onFreeSearchContext(SearchContext context) {
+            DirectoryReader dirReader = context.searcher().getDirectoryReader();
+            LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
+            if (lazyDirectoryReader != null) {
                 try {
                     lazyDirectoryReader.release();
                 } catch (IOException e) {
                     throw new UncheckedIOException(e);
                 }
-            }, SearchContext.Lifetime.PHASE);
+            }
         }
 
         @Override
-        public void onNewContext(SearchContext context) {
-            DirectoryReader dirReader = context.searcher().getDirectoryReader();
+        public void onNewReaderContext(ReaderContext readerContext) {
+            DirectoryReader dirReader = readerContext.engineSearcher().getDirectoryReader();
             LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
             if (lazyDirectoryReader != null) {
-                registerRelease(context, lazyDirectoryReader);
+                readerContext.addOnClose(() -> {
+                    try {
+                        lazyDirectoryReader.release();
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
+                    }
+                });
             }
         }
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
index 5e0c2945caadc..aef1652b42fc5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
@@ -46,10 +46,9 @@ public SecuritySearchOperationListener(ThreadContext threadContext, XPackLicense
      * Adds the {@link Authentication} to the {@link ScrollContext}
      */
     @Override
-    public void onNewScrollContext(SearchContext searchContext) {
+    public void onNewScrollContext(ScrollContext scrollContext) {
         if (licenseState.isAuthAllowed()) {
-            searchContext.scrollContext().putInContext(AuthenticationField.AUTHENTICATION_KEY,
-                    Authentication.getAuthentication(threadContext));
+            scrollContext.putInContext(AuthenticationField.AUTHENTICATION_KEY, Authentication.getAuthentication(threadContext));
         }
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index 5b73d6d212fc5..05f0ed7f0c42f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -49,18 +49,19 @@ public void testUnlicensed() {
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         AuditTrailService auditTrailService = mock(AuditTrailService.class);
         SearchContext searchContext = mock(SearchContext.class);
-        when(searchContext.scrollContext()).thenReturn(new ScrollContext());
+        ScrollContext scrollContext = new ScrollContext();
+        when(searchContext.scrollContext()).thenReturn(scrollContext);
 
         SecuritySearchOperationListener listener = new SecuritySearchOperationListener(threadContext, licenseState, auditTrailService);
-        listener.onNewScrollContext(searchContext);
+        listener.onNewScrollContext(scrollContext);
         listener.validateSearchContext(searchContext, Empty.INSTANCE);
         verify(licenseState, times(2)).isAuthAllowed();
         verifyZeroInteractions(auditTrailService, searchContext);
     }
 
     public void testOnNewContextSetsAuthentication() throws Exception {
-        TestScrollSearchContext testSearchContext = new TestScrollSearchContext();
-        testSearchContext.scrollContext(new ScrollContext());
+        ScrollContext scrollContext = new ScrollContext();
+        TestSearchContext testSearchContext = new TestSearchContext(null, null, null, scrollContext);
         final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(2L));
         testSearchContext.scrollContext().scroll = scroll;
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
@@ -71,7 +72,7 @@ public void testOnNewContextSetsAuthentication() throws Exception {
         authentication.writeToContext(threadContext);
 
         SecuritySearchOperationListener listener = new SecuritySearchOperationListener(threadContext, licenseState, auditTrailService);
-        listener.onNewScrollContext(testSearchContext);
+        listener.onNewScrollContext(scrollContext);
 
         Authentication contextAuth = testSearchContext.scrollContext().getFromContext(AuthenticationField.AUTHENTICATION_KEY);
         assertEquals(authentication, contextAuth);
@@ -82,8 +83,8 @@ public void testOnNewContextSetsAuthentication() throws Exception {
     }
 
     public void testValidateSearchContext() throws Exception {
-        TestScrollSearchContext testSearchContext = new TestScrollSearchContext();
-        testSearchContext.scrollContext(new ScrollContext());
+        ScrollContext scrollContext = new ScrollContext();
+        TestSearchContext testSearchContext = new TestSearchContext(null, null, null, scrollContext);
         testSearchContext.scrollContext().putInContext(AuthenticationField.AUTHENTICATION_KEY,
                 new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null));
         testSearchContext.scrollContext().scroll = new Scroll(TimeValue.timeValueSeconds(2L));
@@ -235,24 +236,4 @@ public void testEnsuredAuthenticatedUserIsSame() {
         verify(auditTrail).accessDenied(eq(auditId), eq(runAsDiffType), eq(action), eq(request),
             authzInfoRoles(original.getUser().roles()));
     }
-
-    static class TestScrollSearchContext extends TestSearchContext {
-
-        private ScrollContext scrollContext;
-
-        TestScrollSearchContext() {
-            super(null);
-        }
-
-        @Override
-        public ScrollContext scrollContext() {
-            return scrollContext;
-        }
-
-        @Override
-        public SearchContext scrollContext(ScrollContext scrollContext) {
-            this.scrollContext = scrollContext;
-            return this;
-        }
-    }
 }

From e6a5becf77b0d490f73a8b4f7f1283830f2d91ce Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sat, 15 Feb 2020 21:17:36 -0500
Subject: [PATCH 02/47] Revert "Cut over from SearchContext to ReaderContext
 (#51282)"

This reverts commit 3ce9a3bf9663e83636af3cc30072f3a9dfa38692.
---
 .../ParentChildInnerHitContextBuilder.java    |   8 +-
 .../action/search/SearchTransportService.java |   4 +-
 .../index/query/NestedQueryBuilder.java       |   6 +-
 .../index/search/stats/ShardSearchStats.java  |  12 +-
 .../index/shard/SearchOperationListener.java  |  57 ++--
 .../search/DefaultSearchContext.java          |  88 +++--
 .../elasticsearch/search/SearchService.java   | 306 +++++++++++-------
 .../search/aggregations/AggregationPhase.java |   2 +
 .../search/aggregations/AggregatorBase.java   |   3 +-
 .../aggregations/AggregatorFactory.java       |   3 +-
 .../fetch/subphase/ExplainFetchSubPhase.java  |   2 +
 .../subphase/MatchedQueriesFetchSubPhase.java |   3 +
 .../internal/FilteredSearchContext.java       |  35 ++
 .../search/internal/ReaderContext.java        | 155 ---------
 .../search/internal/ScrollContext.java        |   7 -
 .../search/internal/SearchContext.java        | 102 +++++-
 .../search/internal/SubSearchContext.java     |  21 ++
 .../search/query/QueryPhase.java              |   4 +
 .../elasticsearch/index/IndexModuleTests.java |  10 +-
 .../shard/SearchOperationListenerTests.java   |  27 +-
 .../search/DefaultSearchContextTests.java     |  40 +--
 .../search/SearchServiceTests.java            | 149 +++++----
 .../search/query/QueryPhaseTests.java         |  10 +-
 .../search/MockSearchService.java             |  18 +-
 .../aggregations/AggregatorTestCase.java      |   2 +-
 .../elasticsearch/test/TestSearchContext.java |  39 ++-
 .../search/MockSearchServiceTests.java        |  46 ++-
 .../index/engine/FrozenEngine.java            |  24 +-
 .../SecuritySearchOperationListener.java      |   5 +-
 .../SecuritySearchOperationListenerTests.java |  35 +-
 30 files changed, 682 insertions(+), 541 deletions(-)
 delete mode 100644 server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java

diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
index 626ecab0f4a34..551e57abb62a1 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
@@ -158,8 +158,12 @@ public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException {
                         topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE);
                         maxScoreCollector = new MaxScoreCollector();
                     }
-                    for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) {
-                        intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
+                    try {
+                        for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) {
+                            intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
+                        }
+                    } finally {
+                        clearReleasables(Lifetime.COLLECTION);
                     }
                     TopDocs topDocs = topDocsCollector.topDocs(from(), size());
                     float maxScore = Float.NaN;
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 53d8d266ea795..16e8c17688906 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -285,13 +285,13 @@ public void writeTo(StreamOutput out) throws IOException {
     public static void registerRequestHandler(TransportService transportService, SearchService searchService) {
         transportService.registerRequestHandler(FREE_CONTEXT_SCROLL_ACTION_NAME, ThreadPool.Names.SAME, ScrollFreeContextRequest::new,
             (request, channel, task) -> {
-                boolean freed = searchService.freeReaderContext(request.id());
+                boolean freed = searchService.freeContext(request.id());
                 channel.sendResponse(new SearchFreeContextResponse(freed));
         });
         TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_SCROLL_ACTION_NAME, SearchFreeContextResponse::new);
         transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, ThreadPool.Names.SAME, SearchFreeContextRequest::new,
             (request, channel, task) -> {
-                boolean freed = searchService.freeReaderContext(request.id());
+                boolean freed = searchService.freeContext(request.id());
                 channel.sendResponse(new SearchFreeContextResponse(freed));
         });
         TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, SearchFreeContextResponse::new);
diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index 55ec170c1e02b..fecf5c8407e98 100644
--- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -421,7 +421,11 @@ public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException {
                         topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE);
                         maxScoreCollector = new MaxScoreCollector();
                     }
-                    intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
+                    try {
+                        intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
+                    } finally {
+                        clearReleasables(Lifetime.COLLECTION);
+                    }
 
                     TopDocs td = topDocsCollector.topDocs(from(), size());
                     float maxScore = Float.NaN;
diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
index 664b631249f68..5446b1ed0e372 100644
--- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
+++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
@@ -24,8 +24,6 @@
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.index.shard.SearchOperationListener;
-import org.elasticsearch.search.internal.ReaderContext;
-import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.util.HashMap;
@@ -148,25 +146,25 @@ private StatsHolder groupStats(String group) {
     }
 
     @Override
-    public void onNewReaderContext(ReaderContext readerContext) {
+    public void onNewContext(SearchContext context) {
         openContexts.inc();
     }
 
     @Override
-    public void onFreeReaderContext(ReaderContext readerContext) {
+    public void onFreeContext(SearchContext context) {
         openContexts.dec();
     }
 
     @Override
-    public void onNewScrollContext(ScrollContext scrollContext) {
+    public void onNewScrollContext(SearchContext context) {
         totalStats.scrollCurrent.inc();
     }
 
     @Override
-    public void onFreeScrollContext(ScrollContext scrollContext) {
+    public void onFreeScrollContext(SearchContext context) {
         totalStats.scrollCurrent.dec();
         assert totalStats.scrollCurrent.count() >= 0;
-        totalStats.scrollMetric.inc(TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - scrollContext.getStartTimeInNano()));
+        totalStats.scrollMetric.inc(TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - context.getOriginNanoTime()));
     }
 
     static final class StatsHolder {
diff --git a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
index 77e83edcdd18f..ede86e6ec222d 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
@@ -21,8 +21,6 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.search.internal.ReaderContext;
-import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.transport.TransportRequest;
 
@@ -78,34 +76,34 @@ default void onFailedFetchPhase(SearchContext searchContext) {}
     default void onFetchPhase(SearchContext searchContext, long tookInNanos) {}
 
     /**
-     * Executed when a new reader context was created
-     * @param readerContext the created context
+     * Executed when a new search context was created
+     * @param context the created context
      */
-    default void onNewReaderContext(ReaderContext readerContext) {}
+    default void onNewContext(SearchContext context) {}
 
     /**
-     * Executed when a previously created reader context is freed.
+     * Executed when a previously created search context is freed.
      * This happens either when the search execution finishes, if the
      * execution failed or if the search context as idle for and needs to be
      * cleaned up.
-     * @param readerContext the freed reader context
+     * @param context the freed search context
      */
-    default void onFreeReaderContext(ReaderContext readerContext) {}
+    default void onFreeContext(SearchContext context) {}
 
     /**
      * Executed when a new scroll search {@link SearchContext} was created
-     * @param scrollContext the created search context
+     * @param context the created search context
      */
-    default void onNewScrollContext(ScrollContext scrollContext) {}
+    default void onNewScrollContext(SearchContext context) {}
 
     /**
      * Executed when a scroll search {@link SearchContext} is freed.
      * This happens either when the scroll search execution finishes, if the
      * execution failed or if the search context as idle for and needs to be
      * cleaned up.
-     * @param scrollContext the freed search context
+     * @param context the freed search context
      */
-    default void onFreeScrollContext(ScrollContext scrollContext) {}
+    default void onFreeScrollContext(SearchContext context) {}
 
     /**
      * Executed prior to using a {@link SearchContext} that has been retrieved
@@ -116,14 +114,6 @@ default void onFreeScrollContext(ScrollContext scrollContext) {}
      */
     default void validateSearchContext(SearchContext context, TransportRequest transportRequest) {}
 
-    /**
-     * Executed when a search context was freed. The implementor can implement
-     * this method to release resources used by the search context.
-     */
-    default void onFreeSearchContext(SearchContext context) {
-
-    }
-
     /**
      * A Composite listener that multiplexes calls to each of the listeners methods.
      */
@@ -203,10 +193,10 @@ public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
         }
 
         @Override
-        public void onNewReaderContext(ReaderContext readerContext) {
+        public void onNewContext(SearchContext context) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onNewReaderContext(readerContext);
+                    listener.onNewContext(context);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onNewContext listener [{}] failed", listener), e);
                 }
@@ -214,10 +204,10 @@ public void onNewReaderContext(ReaderContext readerContext) {
         }
 
         @Override
-        public void onFreeReaderContext(ReaderContext readerContext) {
+        public void onFreeContext(SearchContext context) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onFreeReaderContext(readerContext);
+                    listener.onFreeContext(context);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onFreeContext listener [{}] failed", listener), e);
                 }
@@ -225,10 +215,10 @@ public void onFreeReaderContext(ReaderContext readerContext) {
         }
 
         @Override
-        public void onNewScrollContext(ScrollContext scrollContext) {
+        public void onNewScrollContext(SearchContext context) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onNewScrollContext(scrollContext);
+                    listener.onNewScrollContext(context);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onNewScrollContext listener [{}] failed", listener), e);
                 }
@@ -236,10 +226,10 @@ public void onNewScrollContext(ScrollContext scrollContext) {
         }
 
         @Override
-        public void onFreeScrollContext(ScrollContext scrollContext) {
+        public void onFreeScrollContext(SearchContext context) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onFreeScrollContext(scrollContext);
+                    listener.onFreeScrollContext(context);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onFreeScrollContext listener [{}] failed", listener), e);
                 }
@@ -258,16 +248,5 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
             }
             ExceptionsHelper.reThrowIfNotNull(exception);
         }
-
-        @Override
-        public void onFreeSearchContext(SearchContext context) {
-            for (SearchOperationListener listener : listeners) {
-                try {
-                    listener.onFreeSearchContext(context);
-                } catch (Exception e) {
-                    logger.warn(() -> new ParameterizedMessage("onFreeSearchContext listener [{}] failed", listener), e);
-                }
-            }
-        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index e74c52b70f3ec..5ea5ed3f23e35 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.WeightFactorFunction;
@@ -60,7 +61,6 @@
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
-import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
@@ -75,6 +75,7 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -82,11 +83,12 @@
 
 final class DefaultSearchContext extends SearchContext {
 
-    private final ReaderContext readerContext;
+    private final long id;
     private final ShardSearchRequest request;
     private final SearchShardTarget shardTarget;
     private final LongSupplier relativeTimeSupplier;
     private SearchType searchType;
+    private final Engine.Searcher engineSearcher;
     private final BigArrays bigArrays;
     private final IndexShard indexShard;
     private final ClusterService clusterService;
@@ -100,6 +102,7 @@ final class DefaultSearchContext extends SearchContext {
     // terminate after count
     private int terminateAfter = DEFAULT_TERMINATE_AFTER;
     private List<String> groupStats;
+    private ScrollContext scrollContext;
     private boolean explain;
     private boolean version = false; // by default, we don't return versions
     private boolean seqAndPrimaryTerm = false;
@@ -139,6 +142,10 @@ final class DefaultSearchContext extends SearchContext {
     private SearchContextAggregations aggregations;
     private SearchContextHighlight highlight;
     private SuggestionSearchContext suggest;
+    private List<RescoreContext> rescore;
+    private volatile long keepAlive;
+    private final long originNanoTime = System.nanoTime();
+    private volatile long lastAccessTime = -1;
     private Profilers profilers;
 
     private final Map<String, SearchExtBuilder> searchExtBuilders = new HashMap<>();
@@ -146,36 +153,36 @@ final class DefaultSearchContext extends SearchContext {
     private final QueryShardContext queryShardContext;
     private final FetchPhase fetchPhase;
 
-    DefaultSearchContext(ReaderContext readerContext, ShardSearchRequest request, SearchShardTarget shardTarget,
-                         ClusterService clusterService, IndexService indexService, IndexShard indexShard, BigArrays bigArrays,
-                         LongSupplier relativeTimeSupplier, TimeValue timeout, FetchPhase fetchPhase) {
-        this.readerContext = readerContext;
+    DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget,
+                         Engine.Searcher engineSearcher, ClusterService clusterService, IndexService indexService,
+                         IndexShard indexShard, BigArrays bigArrays, LongSupplier relativeTimeSupplier, TimeValue timeout,
+                         FetchPhase fetchPhase) {
+        this.id = id;
         this.request = request;
         this.fetchPhase = fetchPhase;
         this.searchType = request.searchType();
         this.shardTarget = shardTarget;
+        this.engineSearcher = engineSearcher;
         // SearchContexts use a BigArrays that can circuit break
         this.bigArrays = bigArrays.withCircuitBreaking();
-        this.dfsResult = new DfsSearchResult(readerContext.id(), shardTarget);
-        this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget);
-        this.fetchResult = new FetchSearchResult(readerContext.id(), shardTarget);
+        this.dfsResult = new DfsSearchResult(id, shardTarget);
+        this.queryResult = new QuerySearchResult(id, shardTarget);
+        this.fetchResult = new FetchSearchResult(id, shardTarget);
         this.indexShard = indexShard;
         this.indexService = indexService;
         this.clusterService = clusterService;
-        final Engine.Searcher engineSearcher = readerContext.engineSearcher();
         this.searcher = new ContextIndexSearcher(engineSearcher.getIndexReader(), engineSearcher.getSimilarity(),
             engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy());
-        searcher.setAggregatedDfs(readerContext.aggregatedDfs());
         this.relativeTimeSupplier = relativeTimeSupplier;
         this.timeout = timeout;
-        queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher,
+        queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher,
             request::nowInMillis, shardTarget.getClusterAlias());
         queryBoost = request.indexBoost();
     }
 
     @Override
     public void doClose() {
-        readerContext.decRef();
+        Releasables.close(engineSearcher);
     }
 
     /**
@@ -192,7 +199,7 @@ public void preProcess(boolean rewrite) {
         int maxResultWindow = indexService.getIndexSettings().getMaxResultWindow();
 
         if (resultWindow > maxResultWindow) {
-            if (scrollContext() == null) {
+            if (scrollContext == null) {
                 throw new IllegalArgumentException(
                         "Result window is too large, from + size must be less than or equal to: [" + maxResultWindow + "] but was ["
                                 + resultWindow + "]. See the scroll api for a more efficient way to request large data sets. "
@@ -204,12 +211,12 @@ public void preProcess(boolean rewrite) {
                             + "]. Scroll batch sizes cost as much memory as result windows so they are controlled by the ["
                             + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey() + "] index level setting.");
         }
-        if (rescore().isEmpty() == false) {
+        if (rescore != null) {
             if (sort != null) {
                 throw new IllegalArgumentException("Cannot use [sort] option in conjunction with [rescore].");
             }
             int maxWindow = indexService.getIndexSettings().getMaxRescoreWindow();
-            for (RescoreContext rescoreContext: rescore()) {
+            for (RescoreContext rescoreContext: rescore) {
                 if (rescoreContext.getWindowSize() > maxWindow) {
                     throw new IllegalArgumentException("Rescore window [" + rescoreContext.getWindowSize() + "] is too large. "
                             + "It must be less than [" + maxWindow + "]. This prevents allocating massive heaps for storing the results "
@@ -290,12 +297,12 @@ && new NestedHelper(mapperService()).mightMatchNestedDocs(query)
 
     @Override
     public long id() {
-        return readerContext.id();
+        return this.id;
     }
 
     @Override
     public String source() {
-        return readerContext.source();
+        return engineSearcher.source();
     }
 
     @Override
@@ -323,11 +330,21 @@ public float queryBoost() {
         return queryBoost;
     }
 
+    @Override
+    public long getOriginNanoTime() {
+        return originNanoTime;
+    }
+
     @Override
     public ScrollContext scrollContext() {
-        return readerContext.scrollContext();
+        return this.scrollContext;
     }
 
+    @Override
+    public SearchContext scrollContext(ScrollContext scrollContext) {
+        this.scrollContext = scrollContext;
+        return this;
+    }
 
     @Override
     public SearchContextAggregations aggregations() {
@@ -374,7 +391,18 @@ public void suggest(SuggestionSearchContext suggest) {
 
     @Override
     public List<RescoreContext> rescore() {
-        return readerContext.rescore();
+        if (rescore == null) {
+            return Collections.emptyList();
+        }
+        return rescore;
+    }
+
+    @Override
+    public void addRescore(RescoreContext rescore) {
+        if (this.rescore == null) {
+            this.rescore = new ArrayList<>();
+        }
+        this.rescore.add(rescore);
     }
 
     @Override
@@ -708,6 +736,26 @@ public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int
         return this;
     }
 
+    @Override
+    public void accessed(long accessTime) {
+        this.lastAccessTime = accessTime;
+    }
+
+    @Override
+    public long lastAccessTime() {
+        return this.lastAccessTime;
+    }
+
+    @Override
+    public long keepAlive() {
+        return this.keepAlive;
+    }
+
+    @Override
+    public void keepAlive(long keepAlive) {
+        this.keepAlive = keepAlive;
+    }
+
     @Override
     public DfsSearchResult dfsResult() {
         return dfsResult;
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 84d47729f45b5..f65ccf8d6638d 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -32,12 +32,10 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -86,8 +84,9 @@
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.ReaderContext;
+import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContext.Lifetime;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhase;
@@ -105,6 +104,7 @@
 import org.elasticsearch.threadpool.Scheduler.Cancellable;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Names;
+import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -120,6 +120,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.LongSupplier;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueHours;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
@@ -190,7 +191,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
 
     private final AtomicLong idGenerator = new AtomicLong();
 
-    private final ConcurrentMapLong<ReaderContext> activeReaders = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
+    private final ConcurrentMapLong<SearchContext> activeContexts = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
 
     private final MultiBucketConsumerService multiBucketConsumerService;
 
@@ -280,13 +281,13 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem
 
     }
 
-    protected void putReaderContext(ReaderContext context) {
-        final ReaderContext previous = activeReaders.put(context.id(), context);
+    protected void putContext(SearchContext context) {
+        final SearchContext previous = activeContexts.put(context.id(), context);
         assert previous == null;
     }
 
-    protected ReaderContext removeReaderContext(long id) {
-        return activeReaders.remove(id);
+    protected SearchContext removeContext(long id) {
+        return activeContexts.remove(id);
     }
 
     @Override
@@ -295,8 +296,8 @@ protected void doStart() {
 
     @Override
     protected void doStop() {
-        for (final ReaderContext context : activeReaders.values()) {
-            freeReaderContext(context.id());
+        for (final SearchContext context : activeContexts.values()) {
+            freeContext(context.id());
         }
     }
 
@@ -311,14 +312,20 @@ public void executeDfsPhase(ShardSearchRequest request, SearchShardTask task, Ac
     }
 
     private DfsSearchResult executeDfsPhase(ShardSearchRequest request, SearchShardTask task) throws IOException {
-        final ReaderContext reader = createAndPutReaderContext(request);
-        try (SearchContext context = createContext(reader, reader.request(), task, true)) {
+        final SearchContext context = createAndPutContext(request);
+        context.incRef();
+        try {
+            context.setTask(task);
+            contextProcessing(context);
             dfsPhase.execute(context);
+            contextProcessedSuccessfully(context);
             return context.dfsResult();
         } catch (Exception e) {
             logger.trace("Dfs phase failed", e);
-            processFailure(reader, e);
+            processFailure(context, e);
             throw e;
+        } finally {
+            cleanContext(context);
         }
     }
 
@@ -339,18 +346,23 @@ public void executeQueryPhase(ShardSearchRequest request, SearchShardTask task,
         rewriteShardRequest(request, ActionListener.map(listener, r -> executeQueryPhase(r, task)));
     }
 
-    private <T> void runAsync(long id, CheckedSupplier<T, IOException> executable, ActionListener<T> listener) {
+    private <T> void runAsync(long id, Supplier<T> executable, ActionListener<T> listener) {
         getExecutor(id).execute(ActionRunnable.supply(listener, executable::get));
     }
 
     private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchShardTask task) throws Exception {
-        final ReaderContext reader = createAndPutReaderContext(request);
-        try (SearchContext context = createContext(reader, reader.request(), task, true)) {
+        final SearchContext context = createAndPutContext(request);
+        context.incRef();
+        try {
+            context.setTask(task);
             final long afterQueryTime;
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                contextProcessing(context);
                 loadOrExecuteQueryPhase(request, context);
                 if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) {
-                    freeReaderContext(reader.id());
+                    freeContext(context.id());
+                } else {
+                    contextProcessedSuccessfully(context);
                 }
                 afterQueryTime = executor.success();
             }
@@ -365,8 +377,10 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchSh
                     (Exception) e.getCause() : new ElasticsearchException(e.getCause());
             }
             logger.trace("Query phase failed", e);
-            processFailure(reader, e);
+            processFailure(context, e);
             throw e;
+        } finally {
+            cleanContext(context);
         }
     }
 
@@ -375,7 +389,9 @@ private QueryFetchSearchResult executeFetchPhase(SearchContext context, long aft
             shortcutDocIdsToLoad(context);
             fetchPhase.execute(context);
             if (fetchPhaseShouldFreeContext(context)) {
-                freeReaderContext(context.id());
+                freeContext(context.id());
+            } else {
+                contextProcessedSuccessfully(context);
             }
             executor.success();
         }
@@ -386,41 +402,49 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
         runAsync(request.id(), () -> {
-            final ReaderContext reader = findReaderContext(request.id());
-            try (SearchContext context = createContext(reader, reader.request(), task, false);
-                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
-                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
-                processScroll(request, reader, context);
+            final SearchContext context = findContext(request.id(), request);
+            context.incRef();
+            try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                context.setTask(task);
+                contextProcessing(context);
+                processScroll(request, context);
                 queryPhase.execute(context);
+                contextProcessedSuccessfully(context);
                 executor.success();
                 return new ScrollQuerySearchResult(context.queryResult(), context.shardTarget());
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
-                processFailure(reader, e);
+                processFailure(context, e);
                 throw e;
+            } finally {
+                cleanContext(context);
             }
         }, listener);
     }
 
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
         runAsync(request.id(), () -> {
-            final ReaderContext reader = findReaderContext(request.id());
-            reader.aggregatedDfs(request.dfs());
-            try (SearchContext context = createContext(reader, reader.request(), task, true);
-                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
-                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
+            final SearchContext context = findContext(request.id(), request);
+            context.setTask(task);
+            context.incRef();
+            try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                contextProcessing(context);
                 context.searcher().setAggregatedDfs(request.dfs());
                 queryPhase.execute(context);
                 if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) {
                     // no hits, we can release the context since there will be no fetch phase
-                    freeReaderContext(reader.id());
+                    freeContext(context.id());
+                } else {
+                    contextProcessedSuccessfully(context);
                 }
                 executor.success();
                 return context.queryResult();
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
-                processFailure(reader, e);
+                processFailure(context, e);
                 throw e;
+            } finally {
+                cleanContext(context);
             }
         }, listener);
     }
@@ -436,7 +460,11 @@ private boolean fetchPhaseShouldFreeContext(SearchContext context) {
     }
 
     final Executor getExecutor(long id) {
-        return getExecutor(findReaderContext(id).indexShard());
+        SearchContext context = activeContexts.get(id);
+        if (context == null) {
+            throw new SearchContextMissingException(id);
+        }
+        return getExecutor(context.indexShard());
     }
 
     private Executor getExecutor(IndexShard indexShard) {
@@ -447,28 +475,33 @@ private Executor getExecutor(IndexShard indexShard) {
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
         runAsync(request.id(), () -> {
-            final ReaderContext reader = findReaderContext(request.id());
-            try (SearchContext context = createContext(reader, reader.request(), task, false);
-                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
-                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
-                processScroll(request, reader, context);
+            final SearchContext context = findContext(request.id(), request);
+            context.setTask(task);
+            context.incRef();
+            try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)){
+                contextProcessing(context);
+                processScroll(request, context);
                 queryPhase.execute(context);
                 final long afterQueryTime = executor.success();
                 QueryFetchSearchResult fetchSearchResult = executeFetchPhase(context, afterQueryTime);
                 return new ScrollQueryFetchSearchResult(fetchSearchResult, context.shardTarget());
             } catch (Exception e) {
                 logger.trace("Fetch phase failed", e);
-                processFailure(reader, e);
+                processFailure(context, e);
                 throw e;
+            } finally {
+                cleanContext(context);
             }
         }, listener);
     }
 
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
         runAsync(request.id(), () -> {
-            final ReaderContext reader = findReaderContext(request.id());
-            try (SearchContext context = createContext(reader, reader.request(), task, false)) {
-                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
+            final SearchContext context = findContext(request.id(), request);
+            context.incRef();
+            try {
+                context.setTask(task);
+                contextProcessing(context);
                 if (request.lastEmittedDoc() != null) {
                     context.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
                 }
@@ -476,74 +509,89 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
                 try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, System.nanoTime())) {
                     fetchPhase.execute(context);
                     if (fetchPhaseShouldFreeContext(context)) {
-                        freeReaderContext(request.id());
+                        freeContext(request.id());
+                    } else {
+                        contextProcessedSuccessfully(context);
                     }
                     executor.success();
                 }
                 return context.fetchResult();
             } catch (Exception e) {
                 logger.trace("Fetch phase failed", e);
-                processFailure(reader, e);
+                processFailure(context, e);
                 throw e;
+            } finally {
+                cleanContext(context);
             }
         }, listener);
     }
 
-    private ReaderContext findReaderContext(long id) throws SearchContextMissingException {
-        final ReaderContext reader = activeReaders.get(id);
-        if (reader == null) {
+    private SearchContext findContext(long id, TransportRequest request) throws SearchContextMissingException {
+        SearchContext context = activeContexts.get(id);
+        if (context == null) {
             throw new SearchContextMissingException(id);
         }
-        return reader;
+
+        SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
+        try {
+            operationListener.validateSearchContext(context, request);
+            return context;
+        } catch (Exception e) {
+            processFailure(context, e);
+            throw e;
+        }
     }
 
-    final ReaderContext createAndPutReaderContext(ShardSearchRequest request) {
+    final SearchContext createAndPutContext(ShardSearchRequest request) throws IOException {
         if (request.scroll() != null && openScrollContexts.get() >= maxOpenScrollContext) {
             throw new ElasticsearchException(
                 "Trying to create too many scroll contexts. Must be less than or equal to: [" +
                     maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
                     + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
         }
-        final IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().getId());
-        final SearchOperationListener searchOperationListener = indexShard.getSearchOperationListener();
-        Engine.Searcher engineSearcher = indexShard.acquireSearcher("search");
-        ReaderContext readerContext = null;
+
+        SearchContext context = createContext(request);
+        onNewContext(context);
+        boolean success = false;
         try {
-            readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, engineSearcher, request);
-            engineSearcher = null;
-            final ReaderContext finalReaderContext = readerContext;
-             searchOperationListener.onNewReaderContext(finalReaderContext);
-            if (finalReaderContext.scrollContext() != null) {
-                searchOperationListener.onNewScrollContext(finalReaderContext.scrollContext());
+            putContext(context);
+            success = true;
+            return context;
+        } finally {
+            if (success == false) {
+                freeContext(context.id());
+            }
+        }
+    }
+
+    private void onNewContext(SearchContext context) {
+        boolean success = false;
+        try {
+            if (context.scrollContext() != null) {
                 openScrollContexts.incrementAndGet();
+                context.indexShard().getSearchOperationListener().onNewScrollContext(context);
             }
-            readerContext.addOnClose(() -> {
-                try {
-                    if (finalReaderContext.scrollContext() != null) {
-                        openScrollContexts.decrementAndGet();
-                        searchOperationListener.onFreeScrollContext(finalReaderContext.scrollContext());
-                    }
-                } finally {
-                     searchOperationListener.onFreeReaderContext(finalReaderContext);
-                }
-            });
-            putReaderContext(finalReaderContext);
-            readerContext = null;
-            return finalReaderContext;
+            context.indexShard().getSearchOperationListener().onNewContext(context);
+            success = true;
         } finally {
-            Releasables.close(engineSearcher, readerContext);
+            // currently, the concrete listener is CompositeListener, which swallows exceptions, but here we anyway try to do the
+            // right thing by closing and notifying onFreeXXX in case one of the listeners fails with an exception in the future.
+            if (success == false) {
+                try (context) {
+                    onFreeContext(context);
+                }
+            }
         }
     }
 
-    final SearchContext createContext(ReaderContext reader, ShardSearchRequest request, SearchShardTask task,
-                                      boolean includeAggregations) throws IOException {
-        final DefaultSearchContext context = createSearchContext(reader, request, defaultSearchTimeout);
+    final SearchContext createContext(ShardSearchRequest request) throws IOException {
+        final DefaultSearchContext context = createSearchContext(request, defaultSearchTimeout);
         try {
-            context.setTask(task);
             if (request.scroll() != null) {
+                context.scrollContext(new ScrollContext());
                 context.scrollContext().scroll = request.scroll();
             }
-            parseSource(reader, context, request.source(), includeAggregations);
+            parseSource(context, request.source());
 
             // if the from and size are still not set, default them
             if (context.from() == -1) {
@@ -563,12 +611,8 @@ final SearchContext createContext(ReaderContext reader, ShardSearchRequest reque
             if (request.scroll() != null && request.scroll().keepAlive() != null) {
                 keepAlive = request.scroll().keepAlive().millis();
             }
-            contextScrollKeepAlive(reader, keepAlive);
+            contextScrollKeepAlive(context, keepAlive);
             context.lowLevelCancellation(lowLevelCancellation);
-            // Disable timeout while searching and mark the reader as accessed when releasing the context.
-            context.addReleasable(() -> reader.accessed(threadPool.relativeTimeInMillis()));
-            reader.accessed(-1L);
-            context.addReleasable(() -> reader.indexShard().getSearchOperationListener().onFreeSearchContext(context));
         } catch (Exception e) {
             context.close();
             throw e;
@@ -578,34 +622,28 @@ final SearchContext createContext(ReaderContext reader, ShardSearchRequest reque
     }
 
     public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout) throws IOException {
-        IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().getId());
-        Engine.Searcher engineSearcher = indexShard.acquireSearcher("search");
-        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, engineSearcher, request)) {
-            engineSearcher = null; // transfer ownership to readerContext
-            return createSearchContext(readerContext, request, timeout);
-        } finally {
-            Releasables.close(engineSearcher);
-        }
+        return createSearchContext(request, timeout, true, "search");
     }
 
-    private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSearchRequest request, TimeValue timeout)
+    private DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout,
+                                                     boolean assertAsyncActions, String source)
             throws IOException {
         IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
         IndexShard indexShard = indexService.getShard(request.shardId().getId());
         SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(),
                 indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE);
+        Engine.Searcher searcher = indexShard.acquireSearcher(source);
 
         boolean success = false;
         DefaultSearchContext searchContext = null;
-        reader.incRef();
         try {
-            searchContext = new DefaultSearchContext(reader, request, shardTarget,
-                clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase);
+            searchContext = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget,
+                searcher, clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase);
             // we clone the query shard context here just for rewriting otherwise we
             // might end up with incorrect state since we are using now() or script services
             // during rewrite and normalized / evaluate templates etc.
             QueryShardContext context = new QueryShardContext(searchContext.getQueryShardContext());
-            Rewriteable.rewrite(request.getRewriteable(), context, true);
+            Rewriteable.rewrite(request.getRewriteable(), context, assertAsyncActions);
             assert searchContext.getQueryShardContext().isCacheable();
             success = true;
         } finally {
@@ -614,7 +652,7 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
                 if (searchContext == null) {
                     // we handle the case where the DefaultSearchContext constructor throws an exception since we would otherwise
                     // leak a searcher and this can have severe implications (unable to obtain shard lock exceptions).
-                    IOUtils.closeWhileHandlingException(reader::decRef);
+                    IOUtils.closeWhileHandlingException(searcher);
                 }
             }
         }
@@ -623,41 +661,72 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
 
     private void freeAllContextForIndex(Index index) {
         assert index != null;
-        for (ReaderContext ctx : activeReaders.values()) {
+        for (SearchContext ctx : activeContexts.values()) {
             if (index.equals(ctx.indexShard().shardId().getIndex())) {
-                freeReaderContext(ctx.id());
+                freeContext(ctx.id());
             }
         }
     }
 
 
-    public boolean freeReaderContext(long id) {
-        try (ReaderContext context = removeReaderContext(id)) {
-            return context != null;
+    public boolean freeContext(long id) {
+        try (SearchContext context = removeContext(id)) {
+            if (context != null) {
+                onFreeContext(context);
+                return true;
+            }
+            return false;
         }
     }
 
+    private void onFreeContext(SearchContext context) {
+        assert context.refCount() > 0 : " refCount must be > 0: " + context.refCount();
+        assert activeContexts.containsKey(context.id()) == false;
+        context.indexShard().getSearchOperationListener().onFreeContext(context);
+        if (context.scrollContext() != null) {
+            openScrollContexts.decrementAndGet();
+            context.indexShard().getSearchOperationListener().onFreeScrollContext(context);
+        }
+    }
 
     public void freeAllScrollContexts() {
-        for (ReaderContext readerContext : activeReaders.values()) {
-            if (readerContext.scrollContext() != null) {
-                freeReaderContext(readerContext.id());
+        for (SearchContext searchContext : activeContexts.values()) {
+            if (searchContext.scrollContext() != null) {
+                freeContext(searchContext.id());
             }
         }
     }
 
-    private void contextScrollKeepAlive(ReaderContext readerContext, long keepAlive) {
+    private void contextScrollKeepAlive(SearchContext context, long keepAlive) {
         if (keepAlive > maxKeepAlive) {
             throw new IllegalArgumentException(
                 "Keep alive for scroll (" + TimeValue.timeValueMillis(keepAlive) + ") is too large. " +
                     "It must be less than (" + TimeValue.timeValueMillis(maxKeepAlive) + "). " +
                     "This limit can be set by changing the [" + MAX_KEEPALIVE_SETTING.getKey() + "] cluster level setting.");
         }
-        readerContext.keepAlive(keepAlive);
+        context.keepAlive(keepAlive);
+    }
+
+    private void contextProcessing(SearchContext context) {
+        // disable timeout while executing a search
+        context.accessed(-1);
+    }
+
+    private void contextProcessedSuccessfully(SearchContext context) {
+        context.accessed(threadPool.relativeTimeInMillis());
+    }
+
+    private void cleanContext(SearchContext context) {
+        try {
+            context.clearReleasables(Lifetime.PHASE);
+            context.setTask(null);
+        } finally {
+            context.decRef();
+        }
     }
 
-    private void processFailure(ReaderContext context, Exception e) {
-        freeReaderContext(context.id());
+    private void processFailure(SearchContext context, Exception e) {
+        freeContext(context.id());
         try {
             if (Lucene.isCorruptionException(e)) {
                 context.indexShard().failShard("search execution corruption failure", e);
@@ -668,8 +737,7 @@ private void processFailure(ReaderContext context, Exception e) {
         }
     }
 
-    private void parseSource(ReaderContext reader, DefaultSearchContext context,
-                             SearchSourceBuilder source, boolean includeAggregations) throws SearchException {
+    private void parseSource(DefaultSearchContext context, SearchSourceBuilder source) throws SearchException {
         // nothing to parse...
         if (source == null) {
             return;
@@ -725,7 +793,7 @@ private void parseSource(ReaderContext reader, DefaultSearchContext context,
             context.timeout(source.timeout());
         }
         context.terminateAfter(source.terminateAfter());
-        if (source.aggregations() != null && includeAggregations) {
+        if (source.aggregations() != null) {
             try {
                 AggregatorFactories factories = source.aggregations().build(queryShardContext, null);
                 context.aggregations(new SearchContextAggregations(factories, multiBucketConsumerService.create()));
@@ -740,10 +808,10 @@ private void parseSource(ReaderContext reader, DefaultSearchContext context,
                 throw new SearchException(shardTarget, "failed to create SuggestionSearchContext", e);
             }
         }
-        if (source.rescores() != null && reader.rescore().isEmpty()) {
+        if (source.rescores() != null) {
             try {
                 for (RescorerBuilder<?> rescore : source.rescores()) {
-                    reader.addRescore(rescore.buildContext(queryShardContext));
+                    context.addRescore(rescore.buildContext(queryShardContext));
                 }
             } catch (IOException e) {
                 throw new SearchException(shardTarget, "failed to create RescoreSearchContext", e);
@@ -901,13 +969,13 @@ private void shortcutDocIdsToLoad(SearchContext context) {
         context.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length);
     }
 
-    private void processScroll(InternalScrollSearchRequest request, ReaderContext reader, SearchContext context) {
+    private void processScroll(InternalScrollSearchRequest request, SearchContext context) {
         // process scroll
         context.from(context.from() + context.size());
         context.scrollContext().scroll = request.scroll();
         // update the context keep alive based on the new scroll value
         if (request.scroll() != null && request.scroll().keepAlive() != null) {
-            contextScrollKeepAlive(reader, request.scroll().keepAlive().millis());
+            contextScrollKeepAlive(context, request.scroll().keepAlive().millis());
         }
     }
 
@@ -916,7 +984,7 @@ private void processScroll(InternalScrollSearchRequest request, ReaderContext re
      * SearchService
      */
     public int getActiveContexts() {
-        return this.activeReaders.size();
+        return this.activeContexts.size();
     }
 
     public ResponseCollectorService getResponseCollectorService() {
@@ -927,7 +995,7 @@ class Reaper implements Runnable {
         @Override
         public void run() {
             final long time = threadPool.relativeTimeInMillis();
-            for (ReaderContext context : activeReaders.values()) {
+            for (SearchContext context : activeContexts.values()) {
                 // Use the same value for both checks since lastAccessTime can
                 // be modified by another thread between checks!
                 final long lastAccessTime = context.lastAccessTime();
@@ -937,7 +1005,7 @@ public void run() {
                 if ((time - lastAccessTime > context.keepAlive())) {
                     logger.debug("freeing search context [{}], time [{}], lastAccessTime [{}], keepAlive [{}]", context.id(), time,
                         lastAccessTime, context.keepAlive());
-                    freeReaderContext(context.id());
+                    freeContext(context.id());
                 }
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
index fca5b14e34be3..c150f99cfe196 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
@@ -117,6 +117,8 @@ public void execute(SearchContext context) {
                 context.searcher().search(query, collector);
             } catch (Exception e) {
                 throw new QueryPhaseExecutionException(context.shardTarget(), "Failed to execute global aggregators", e);
+            } finally {
+                context.clearReleasables(SearchContext.Lifetime.COLLECTION);
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
index 43e38c731bbe1..ea89700521798 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContext.Lifetime;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
 
 import java.io.IOException;
@@ -75,7 +76,7 @@ protected AggregatorBase(String name, AggregatorFactories factories, SearchConte
         this.breakerService = context.bigArrays().breakerService();
         assert factories != null : "sub-factories provided to BucketAggregator must not be null, use AggragatorFactories.EMPTY instead";
         this.subAggregators = factories.createSubAggregators(context, this);
-        context.addReleasable(this);
+        context.addReleasable(this, Lifetime.PHASE);
         final SearchShardTarget shardTarget = context.shardTarget();
         // Register a safeguard to highlight any invalid construction logic (call to this constructor without subsequent preCollection call)
         collectableSubAggregators = new BucketCollector() {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
index 5451285a379c8..970ef725f027d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContext.Lifetime;
 
 import java.io.IOException;
 import java.util.List;
@@ -49,7 +50,7 @@ public static final class MultiBucketAggregatorWrapper extends Aggregator {
             this.parent = parent;
             this.factory = factory;
             this.first = first;
-            context.addReleasable(this);
+            context.addReleasable(this, Lifetime.PHASE);
             aggregators = bigArrays.newObjectArray(1);
             aggregators.set(0, first);
             collectors = bigArrays.newObjectArray(1);
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainFetchSubPhase.java
index 345a4c5a0a7ad..3a7d5c96b8179 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainFetchSubPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainFetchSubPhase.java
@@ -48,6 +48,8 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
         } catch (IOException e) {
             throw new FetchPhaseExecutionException(context.shardTarget(),
                 "Failed to explain doc [" + hitContext.hit().getId() + "]", e);
+        } finally {
+            context.clearReleasables(SearchContext.Lifetime.COLLECTION);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java
index 319bd40cfbd70..c2f6980781dba 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesFetchSubPhase.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContext.Lifetime;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -91,6 +92,8 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) {
             }
         } catch (IOException e) {
             throw ExceptionsHelper.convertToElastic(e);
+        } finally {
+            context.clearReleasables(Lifetime.COLLECTION);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index ab605062de50a..8c04954a4efcf 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -140,11 +140,21 @@ public float queryBoost() {
         return in.queryBoost();
     }
 
+    @Override
+    public long getOriginNanoTime() {
+        return in.getOriginNanoTime();
+    }
+
     @Override
     public ScrollContext scrollContext() {
         return in.scrollContext();
     }
 
+    @Override
+    public SearchContext scrollContext(ScrollContext scroll) {
+        return in.scrollContext(scroll);
+    }
+
     @Override
     public SearchContextAggregations aggregations() {
         return in.aggregations();
@@ -185,6 +195,11 @@ public List<RescoreContext> rescore() {
         return in.rescore();
     }
 
+    @Override
+    public void addRescore(RescoreContext rescore) {
+        in.addRescore(rescore);
+    }
+
     @Override
     public boolean hasScriptFields() {
         return in.hasScriptFields();
@@ -436,6 +451,26 @@ public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int
         return in.docIdsToLoad(docIdsToLoad, docsIdsToLoadFrom, docsIdsToLoadSize);
     }
 
+    @Override
+    public void accessed(long accessTime) {
+        in.accessed(accessTime);
+    }
+
+    @Override
+    public long lastAccessTime() {
+        return in.lastAccessTime();
+    }
+
+    @Override
+    public long keepAlive() {
+        return in.keepAlive();
+    }
+
+    @Override
+    public void keepAlive(long keepAlive) {
+        in.keepAlive(keepAlive);
+    }
+
     @Override
     public SearchLookup lookup() {
         return in.lookup();
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
deleted file mode 100644
index 7c9dfe14dc9f8..0000000000000
--- a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.internal;
-
-import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.common.lease.Releasables;
-import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.search.dfs.AggregatedDfs;
-import org.elasticsearch.search.rescore.RescoreContext;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Holds a reference to a point in time {@link Engine.Searcher} that will be used to construct {@link SearchContext}.
- * This class also implements {@link org.elasticsearch.common.util.concurrent.RefCounted} since in some situations like
- * in {@link org.elasticsearch.search.SearchService} a SearchContext can be closed concurrently due to independent events
- * ie. when an index gets removed. To prevent accessing closed IndexReader / IndexSearcher instances the SearchContext
- * can be guarded by a reference count and fail if it's been closed by an external event.
- *
- * For reference why we use RefCounted here see https://github.com/elastic/elasticsearch/pull/20095.
- */
-public class ReaderContext extends AbstractRefCounted implements Releasable {
-    private final long id;
-    private final IndexShard indexShard;
-    private final Engine.Searcher engineSearcher;
-    private final AtomicBoolean closed = new AtomicBoolean(false);
-
-    private volatile long keepAlive;
-    private volatile long lastAccessTime = -1L;
-
-    // BWC
-    private final ShardSearchRequest request;
-    private final ScrollContext scrollContext;
-    private AggregatedDfs aggregatedDfs;
-    private List<RescoreContext> rescore;
-
-    private final List<Releasable> onCloses = new CopyOnWriteArrayList<>();
-
-    public ReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher, ShardSearchRequest request) {
-        super("reader_context");
-        this.id = id;
-        this.indexShard = indexShard;
-        this.engineSearcher = engineSearcher;
-        this.request = request;
-        if (request.scroll() != null) {
-            this.scrollContext = new ScrollContext();
-        } else {
-            this.scrollContext = null;
-        }
-    }
-
-    @Override
-    public final void close() {
-        if (closed.compareAndSet(false, true)) {
-            decRef();
-        } else {
-            assert false : "ReaderContext was closed already";
-        }
-    }
-
-    @Override
-    protected void closeInternal() {
-        Releasables.close(Releasables.wrap(onCloses), engineSearcher);
-    }
-
-    public void addOnClose(Releasable releasable) {
-        onCloses.add(releasable);
-    }
-
-    public long id() {
-        return id;
-    }
-
-    public IndexShard indexShard() {
-        return indexShard;
-    }
-
-    public Engine.Searcher engineSearcher() {
-        return engineSearcher;
-    }
-
-    public String source() {
-        return engineSearcher.source();
-    }
-
-    public ShardSearchRequest request() {
-        return request;
-    }
-
-    public ScrollContext scrollContext() {
-        return scrollContext;
-    }
-
-    public AggregatedDfs aggregatedDfs() {
-        return aggregatedDfs;
-    }
-
-    public void aggregatedDfs(AggregatedDfs aggregatedDfs) {
-        this.aggregatedDfs = aggregatedDfs;
-    }
-
-    public void accessed(long accessTime) {
-        this.lastAccessTime = accessTime;
-    }
-
-    public long lastAccessTime() {
-        return this.lastAccessTime;
-    }
-
-    public long keepAlive() {
-        return this.keepAlive;
-    }
-
-    public void keepAlive(long keepAlive) {
-        this.keepAlive = keepAlive;
-    }
-
-    public List<RescoreContext> rescore() {
-        if (rescore == null) {
-            return Collections.emptyList();
-        } else {
-            return Collections.unmodifiableList(rescore);
-        }
-    }
-
-    public void addRescore(RescoreContext rescore) {
-        if (this.rescore == null) {
-            this.rescore = new ArrayList<>();
-        }
-        this.rescore.add(rescore);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java b/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
index 440408646d127..41d7680a780b0 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
@@ -36,8 +36,6 @@ public final class ScrollContext {
     public ScoreDoc lastEmittedDoc;
     public Scroll scroll;
 
-    private final long startTimeInNano = System.nanoTime();
-
     /**
      * Returns the object or <code>null</code> if the given key does not have a
      * value in the context
@@ -56,9 +54,4 @@ public void putInContext(String key, Object value) {
         }
         context.put(key, value);
     }
-
-
-    public long getStartTimeInNano() {
-        return startTimeInNano;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index d0eddac1d86c6..c0477ece69aa0 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -29,6 +29,9 @@
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
+import org.elasticsearch.common.util.concurrent.RefCounted;
+import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -58,29 +61,36 @@
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
+import java.util.ArrayList;
+import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * This class encapsulates the state needed to execute a search. It holds a reference to the
  * shards point in time snapshot (IndexReader / ContextIndexSearcher) and allows passing on
  * state from one query / fetch phase to another.
+ *
+ * This class also implements {@link RefCounted} since in some situations like in {@link org.elasticsearch.search.SearchService}
+ * a SearchContext can be closed concurrently due to independent events ie. when an index gets removed. To prevent accessing closed
+ * IndexReader / IndexSearcher instances the SearchContext can be guarded by a reference count and fail if it's been closed by
+ * an external event.
  */
-public abstract class SearchContext implements Releasable {
+// For reference why we use RefCounted here see #20095
+public abstract class SearchContext extends AbstractRefCounted implements Releasable {
 
     public static final int DEFAULT_TERMINATE_AFTER = 0;
     public static final int TRACK_TOTAL_HITS_ACCURATE = Integer.MAX_VALUE;
     public static final int TRACK_TOTAL_HITS_DISABLED = -1;
     public static final int DEFAULT_TRACK_TOTAL_HITS_UP_TO = 10000;
 
-    private final List<Releasable> releasables = new CopyOnWriteArrayList<>();
+    private Map<Lifetime, List<Releasable>> clearables = null;
     private final AtomicBoolean closed = new AtomicBoolean(false);
     private InnerHitsContext innerHitsContext;
 
     protected SearchContext() {
-
+        super("search_context");
     }
 
     public abstract void setTask(SearchShardTask task);
@@ -91,15 +101,25 @@ protected SearchContext() {
 
     @Override
     public final void close() {
-        if (closed.compareAndSet(false, true)) {
-            try {
-                Releasables.close(releasables);
-            } finally {
-                doClose();
-            }
+        if (closed.compareAndSet(false, true)) { // prevent double closing
+            decRef();
+        }
+    }
+
+    @Override
+    protected final void closeInternal() {
+        try {
+            clearReleasables(Lifetime.CONTEXT);
+        } finally {
+            doClose();
         }
     }
 
+    @Override
+    protected void alreadyClosed() {
+        throw new IllegalStateException("search context is already closed can't increment refCount current count [" + refCount() + "]");
+    }
+
     protected abstract void doClose();
 
     /**
@@ -126,8 +146,12 @@ public final void close() {
 
     public abstract float queryBoost();
 
+    public abstract long getOriginNanoTime();
+
     public abstract ScrollContext scrollContext();
 
+    public abstract SearchContext scrollContext(ScrollContext scroll);
+
     public abstract SearchContextAggregations aggregations();
 
     public abstract SearchContext aggregations(SearchContextAggregations aggregations);
@@ -156,6 +180,8 @@ public InnerHitsContext innerHits() {
      */
     public abstract List<RescoreContext> rescore();
 
+    public abstract void addRescore(RescoreContext rescore);
+
     public abstract boolean hasScriptFields();
 
     public abstract ScriptFieldsContext scriptFields();
@@ -296,6 +322,14 @@ public InnerHitsContext innerHits() {
 
     public abstract SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int docsIdsToLoadSize);
 
+    public abstract void accessed(long accessTime);
+
+    public abstract long lastAccessTime();
+
+    public abstract long keepAlive();
+
+    public abstract void keepAlive(long keepAlive);
+
     public SearchLookup lookup() {
         return getQueryShardContext().lookup();
     }
@@ -313,14 +347,38 @@ public SearchLookup lookup() {
      */
     public abstract Profilers getProfilers();
 
-
     /**
-     * Adds a releasable that will be freed when this context is closed.
+     * Schedule the release of a resource. The time when {@link Releasable#close()} will be called on this object
+     * is function of the provided {@link Lifetime}.
      */
-    public void addReleasable(Releasable releasable) {
+    public void addReleasable(Releasable releasable, Lifetime lifetime) {
+        if (clearables == null) {
+            clearables = new EnumMap<>(Lifetime.class);
+        }
+        List<Releasable> releasables = clearables.get(lifetime);
+        if (releasables == null) {
+            releasables = new ArrayList<>();
+            clearables.put(lifetime, releasables);
+        }
         releasables.add(releasable);
     }
 
+    public void clearReleasables(Lifetime lifetime) {
+        if (clearables != null) {
+            List<List<Releasable>>releasables = new ArrayList<>();
+            for (Lifetime lc : Lifetime.values()) {
+                if (lc.compareTo(lifetime) > 0) {
+                    break;
+                }
+                List<Releasable> remove = clearables.remove(lc);
+                if (remove != null) {
+                    releasables.add(remove);
+                }
+            }
+            Releasables.close(Iterables.flatten(releasables));
+        }
+    }
+
     /**
      * @return true if the request contains only suggest
      */
@@ -345,6 +403,24 @@ public final boolean hasOnlySuggest() {
     /** Return a view of the additional query collectors that should be run for this context. */
     public abstract Map<Class<?>, Collector> queryCollectors();
 
+    /**
+     * The life time of an object that is used during search execution.
+     */
+    public enum Lifetime {
+        /**
+         * This life time is for objects that only live during collection time.
+         */
+        COLLECTION,
+        /**
+         * This life time is for objects that need to live until the end of the current search phase.
+         */
+        PHASE,
+        /**
+         * This life time is for objects that need to live until the search context they are attached to is destroyed.
+         */
+        CONTEXT
+    }
+
     public abstract QueryShardContext getQueryShardContext();
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
index 8ce36550f7b94..e918aed603750 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
 import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
@@ -84,6 +85,11 @@ public Query buildFilteredQuery(Query query) {
         throw new UnsupportedOperationException("this context should be read only");
     }
 
+    @Override
+    public SearchContext scrollContext(ScrollContext scrollContext) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
     @Override
     public SearchContext aggregations(SearchContextAggregations aggregations) {
         throw new UnsupportedOperationException("Not supported");
@@ -104,6 +110,11 @@ public void suggest(SuggestionSearchContext suggest) {
         throw new UnsupportedOperationException("Not supported");
     }
 
+    @Override
+    public void addRescore(RescoreContext rescore) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
     @Override
     public boolean hasScriptFields() {
         return scriptFields != null;
@@ -321,6 +332,16 @@ public CollapseContext collapse() {
         return null;
     }
 
+    @Override
+    public void accessed(long accessTime) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
+    @Override
+    public void keepAlive(long keepAlive) {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
     @Override
     public QuerySearchResult queryResult() {
         return querySearchResult;
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 01636b2d7f1c3..e58cf5ad8f7c6 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -339,6 +339,8 @@ private static boolean searchWithCollector(SearchContext searchContext, ContextI
                 throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded");
             }
             queryResult.searchTimedOut(true);
+        } finally {
+            searchContext.clearReleasables(SearchContext.Lifetime.COLLECTION);
         }
         if (searchContext.terminateAfter() != SearchContext.DEFAULT_TERMINATE_AFTER && queryResult.terminatedEarly() == null) {
             queryResult.terminatedEarly(false);
@@ -393,6 +395,8 @@ private static boolean searchWithCollectorManager(SearchContext searchContext, C
                 throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded");
             }
             searchContext.queryResult().searchTimedOut(true);
+        } finally {
+            searchContext.clearReleasables(SearchContext.Lifetime.COLLECTION);
         }
         return false; // no rescoring when sorting by field
     }
diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 18b9b5f4f6d58..2c11b26580f3d 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -78,10 +78,11 @@
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.internal.ReaderContext;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.test.engine.MockEngineFactory;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -102,7 +103,6 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.mockito.Mockito.mock;
 
 public class IndexModuleTests extends ESTestCase {
     private Index index;
@@ -272,8 +272,9 @@ public void testAddSearchOperationListener() throws IOException {
         IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         AtomicBoolean executed = new AtomicBoolean(false);
         SearchOperationListener listener = new SearchOperationListener() {
+
             @Override
-            public void onNewReaderContext(ReaderContext readerContext) {
+            public void onNewContext(SearchContext context) {
                 executed.set(true);
             }
         };
@@ -286,8 +287,9 @@ public void onNewReaderContext(ReaderContext readerContext) {
         assertEquals(2, indexService.getSearchOperationListener().size());
         assertEquals(SearchSlowLog.class, indexService.getSearchOperationListener().get(0).getClass());
         assertSame(listener, indexService.getSearchOperationListener().get(1));
+
         for (SearchOperationListener l : indexService.getSearchOperationListener()) {
-            l.onNewReaderContext(mock(ReaderContext.class));
+            l.onNewContext(new TestSearchContext(null));
         }
         assertTrue(executed.get());
         indexService.close("simon says", false);
diff --git a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
index 34c528dfeda6a..28bab8da0fdfb 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.index.shard;
 
-import org.elasticsearch.search.internal.ReaderContext;
-import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
@@ -35,7 +33,6 @@
 
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.Mockito.mock;
 
 public class SearchOperationListenerTests extends ESTestCase {
 
@@ -93,26 +90,26 @@ public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
             }
 
             @Override
-            public void onNewReaderContext(ReaderContext readerContext) {
-                assertNotNull(readerContext);
+            public void onNewContext(SearchContext context) {
+                assertNotNull(context);
                 newContext.incrementAndGet();
             }
 
             @Override
-            public void onFreeReaderContext(ReaderContext readerContext) {
-                assertNotNull(readerContext);
+            public void onFreeContext(SearchContext context) {
+                assertNotNull(context);
                 freeContext.incrementAndGet();
             }
 
             @Override
-            public void onNewScrollContext(ScrollContext scrollContext) {
-                assertNotNull(scrollContext);
+            public void onNewScrollContext(SearchContext context) {
+                assertNotNull(context);
                 newScrollContext.incrementAndGet();
             }
 
             @Override
-            public void onFreeScrollContext(ScrollContext scrollContext) {
-                assertNotNull(scrollContext);
+            public void onFreeScrollContext(SearchContext context) {
+                assertNotNull(context);
                 freeScrollContext.incrementAndGet();
             }
 
@@ -219,7 +216,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onNewReaderContext(mock(ReaderContext.class));
+        compositeListener.onNewContext(ctx);
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -232,7 +229,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onNewScrollContext(new ScrollContext());
+        compositeListener.onNewScrollContext(ctx);
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -245,7 +242,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onFreeReaderContext(mock(ReaderContext.class));
+        compositeListener.onFreeContext(ctx);
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -258,7 +255,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onFreeScrollContext(new ScrollContext());
+        compositeListener.onFreeScrollContext(ctx);
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 198daf0f41a11..560f052664604 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -50,7 +50,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.ReaderContext;
+import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.slice.SliceBuilder;
@@ -111,33 +111,26 @@ public void testPreProcess() throws Exception {
 
         try (Directory dir = newDirectory();
              RandomIndexWriter w = new RandomIndexWriter(random(), dir);
-             IndexReader reader = w.getReader()) {
-
-            final Engine.Searcher engineSearcher = new Engine.Searcher("test", w.getReader(),
-                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
-                IndexSearcher.getDefaultQueryCachingPolicy(), reader);
+             IndexReader reader = w.getReader();
+             Engine.Searcher searcher = new Engine.Searcher("test", reader,
+                 IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
+                 IndexSearcher.getDefaultQueryCachingPolicy(), reader)) {
 
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
-            ReaderContext readerWithoutScroll = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
-            DefaultSearchContext contextWithoutScroll = new DefaultSearchContext(readerWithoutScroll, shardSearchRequest, target, null,
-                indexService, indexShard, bigArrays, null, timeout, null);
-            contextWithoutScroll.from(300);
-            contextWithoutScroll.close();
+            DefaultSearchContext context1 = new DefaultSearchContext(1L, shardSearchRequest, target, searcher, null, indexService,
+                indexShard, bigArrays, null, timeout, null);
+            context1.from(300);
 
             // resultWindow greater than maxResultWindow and scrollContext is null
-            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> contextWithoutScroll.preProcess(false));
+            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Result window is too large, from + size must be less than or equal to:"
                 + " [" + maxResultWindow + "] but was [310]. See the scroll api for a more efficient way to request large data sets. "
                 + "This limit can be set by changing the [" + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey()
                 + "] index level setting."));
 
             // resultWindow greater than maxResultWindow and scrollContext isn't null
-            when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMillis(randomInt(1000))));
-            ReaderContext readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
-            DefaultSearchContext context1 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null, indexService,
-                indexShard, bigArrays, null, timeout, null);
-            context1.from(300);
+            context1.scrollContext(new ScrollContext());
             exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Batch size is too large, size must be less than or equal to: ["
                 + maxResultWindow + "] but was [310]. Scroll batch sizes cost as much memory as result windows so they are "
@@ -151,7 +144,7 @@ public void testPreProcess() throws Exception {
 
             RescoreContext rescoreContext = mock(RescoreContext.class);
             when(rescoreContext.getWindowSize()).thenReturn(500);
-            readerContext.addRescore(rescoreContext);
+            context1.addRescore(rescoreContext);
 
             exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Cannot use [sort] option in conjunction with [rescore]."));
@@ -165,10 +158,8 @@ public void testPreProcess() throws Exception {
                 + "to be rescored. This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey()
                 + "] index level setting."));
 
-            readerContext.close();
-            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
             // rescore is null but sliceBuilder is not null
-            DefaultSearchContext context2 = new DefaultSearchContext(readerContext, shardSearchRequest, target,
+            DefaultSearchContext context2 = new DefaultSearchContext(2L, shardSearchRequest, target, searcher,
                 null, indexService, indexShard, bigArrays, null, timeout, null);
 
             SliceBuilder sliceBuilder = mock(SliceBuilder.class);
@@ -185,7 +176,7 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.getAliasFilter()).thenReturn(AliasFilter.EMPTY);
             when(shardSearchRequest.indexBoost()).thenReturn(AbstractQueryBuilder.DEFAULT_BOOST);
 
-            DefaultSearchContext context3 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
+            DefaultSearchContext context3 = new DefaultSearchContext(3L, shardSearchRequest, target, searcher, null,
                 indexService, indexShard, bigArrays, null, timeout, null);
             ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery();
             context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false);
@@ -195,9 +186,7 @@ public void testPreProcess() throws Exception {
             when(queryShardContext.fieldMapper(anyString())).thenReturn(mock(MappedFieldType.class));
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
-            readerContext.close();
-            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
-            DefaultSearchContext context4 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
+            DefaultSearchContext context4 = new DefaultSearchContext(4L, shardSearchRequest, target, searcher, null,
                 indexService, indexShard, bigArrays, null, timeout, null);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
             Query query1 = context4.query();
@@ -205,7 +194,6 @@ public void testPreProcess() throws Exception {
             Query query2 = context4.query();
             assertTrue(query1 instanceof MatchNoDocsQuery || query2 instanceof MatchNoDocsQuery);
 
-            readerContext.close();
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index c47ccefca3217..5691530dd32fe 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -47,7 +47,6 @@
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
@@ -76,7 +75,6 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.suggest.SuggestBuilder;
@@ -169,8 +167,8 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
         public void onIndexModule(IndexModule indexModule) {
             indexModule.addSearchOperationListener(new SearchOperationListener() {
                 @Override
-                public void onNewReaderContext(ReaderContext readerContext) {
-                    if ("throttled_threadpool_index".equals(readerContext.indexShard().shardId().getIndex().getName())) {
+                public void onNewContext(SearchContext context) {
+                    if ("throttled_threadpool_index".equals(context.indexShard().shardId().getIndex().getName())) {
                         assertThat(Thread.currentThread().getName(), startsWith("elasticsearch[node_s_0][search_throttled]"));
                     } else {
                         assertThat(Thread.currentThread().getName(), startsWith("elasticsearch[node_s_0][search]"));
@@ -324,7 +322,7 @@ public void onFailure(Exception e) {
                         service.executeFetchPhase(req, new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()), listener);
                         listener.get();
                         if (useScroll) {
-                            service.freeReaderContext(searchPhaseResult.getRequestId());
+                            service.freeContext(searchPhaseResult.getRequestId());
                         }
                     } catch (ExecutionException ex) {
                         assertThat(ex.getCause(), instanceOf(RuntimeException.class));
@@ -333,7 +331,7 @@ public void onFailure(Exception e) {
                 } catch (AlreadyClosedException ex) {
                     throw ex;
                 } catch (IllegalStateException ex) {
-                    assertEquals("reader_context is already closed can't increment refCount current count [0]", ex.getMessage());
+                    assertEquals("search context is already closed can't increment refCount current count [0]", ex.getMessage());
                 } catch (SearchContextMissingException ex) {
                     // that's fine
                 }
@@ -359,34 +357,42 @@ public void testTimeout() throws IOException {
         final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
         final IndexShard indexShard = indexService.getShard(0);
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
-        final ShardSearchRequest requestWithDefaultTimeout = new ShardSearchRequest(
-            OriginalIndices.NONE,
-            searchRequest,
-            indexShard.shardId(),
-            1,
-            new AliasFilter(null, Strings.EMPTY_ARRAY),
-            1.0f, -1, null, null);
-
-        try (ReaderContext reader = createReaderContext(indexShard, requestWithDefaultTimeout);
-             SearchContext contextWithDefaultTimeout = service.createContext(reader, requestWithDefaultTimeout, null, randomBoolean())) {
+        final SearchContext contextWithDefaultTimeout = service.createContext(
+                new ShardSearchRequest(
+                        OriginalIndices.NONE,
+                        searchRequest,
+                        indexShard.shardId(),
+                        1,
+                        new AliasFilter(null, Strings.EMPTY_ARRAY),
+                        1.0f, -1, null, null)
+        );
+        try {
             // the search context should inherit the default timeout
             assertThat(contextWithDefaultTimeout.timeout(), equalTo(TimeValue.timeValueSeconds(5)));
+        } finally {
+            contextWithDefaultTimeout.decRef();
+            service.freeContext(contextWithDefaultTimeout.id());
         }
 
         final long seconds = randomIntBetween(6, 10);
         searchRequest.source(new SearchSourceBuilder().timeout(TimeValue.timeValueSeconds(seconds)));
-        final ShardSearchRequest requestWithCustomTimeout = new ShardSearchRequest(
-            OriginalIndices.NONE,
-            searchRequest,
-            indexShard.shardId(),
-            1,
-            new AliasFilter(null, Strings.EMPTY_ARRAY),
-            1.0f, -1, null, null);
-        try (ReaderContext reader = createReaderContext(indexShard, requestWithCustomTimeout);
-             SearchContext context = service.createContext(reader, requestWithCustomTimeout, null, randomBoolean())) {
+        final SearchContext context = service.createContext(
+                new ShardSearchRequest(
+                        OriginalIndices.NONE,
+                        searchRequest,
+                        indexShard.shardId(),
+                        1,
+                        new AliasFilter(null, Strings.EMPTY_ARRAY),
+                        1.0f, -1, null, null)
+        );
+        try {
             // the search context should inherit the query timeout
             assertThat(context.timeout(), equalTo(TimeValue.timeValueSeconds(seconds)));
+        } finally {
+            context.decRef();
+            service.freeContext(context.id());
         }
+
     }
 
     /**
@@ -406,20 +412,19 @@ public void testMaxDocvalueFieldsSearch() throws IOException {
         for (int i = 0; i < indexService.getIndexSettings().getMaxDocvalueFields(); i++) {
             searchSourceBuilder.docValueField("field" + i);
         }
-        final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
-            new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
-        try (ReaderContext reader = createReaderContext(indexShard, request);
-             SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
+        try (SearchContext context = service.createContext(
+            new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
+                new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null))
+        ) {
             assertNotNull(context);
-        }
-        searchSourceBuilder.docValueField("one_field_too_much");
-        try (ReaderContext reader = createReaderContext(indexShard, request)) {
+            searchSourceBuilder.docValueField("one_field_too_much");
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> service.createContext(reader, request, null, randomBoolean()));
+                    () -> service.createContext(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
+                        new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null)));
             assertEquals(
-                "Trying to retrieve too many docvalue_fields. Must be less than or equal to: [100] but was [101]. "
-                    + "This limit can be set by changing the [index.max_docvalue_fields_search] index level setting.",
-                ex.getMessage());
+                    "Trying to retrieve too many docvalue_fields. Must be less than or equal to: [100] but was [101]. "
+                            + "This limit can be set by changing the [index.max_docvalue_fields_search] index level setting.",
+                    ex.getMessage());
         }
     }
 
@@ -442,22 +447,20 @@ public void testMaxScriptFieldsSearch() throws IOException {
             searchSourceBuilder.scriptField("field" + i,
                     new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
         }
-        final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
-            indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
-
-        try(ReaderContext reader = createReaderContext(indexShard, request)) {
-            try (SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
-                assertNotNull(context);
-            }
+        try (SearchContext context = service.createContext(new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
+                indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY),
+                1.0f, -1, null, null))) {
+            assertNotNull(context);
             searchSourceBuilder.scriptField("anotherScriptField",
-                new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
+                    new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> service.createContext(reader, request, null, randomBoolean()));
+                    () -> service.createContext(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
+                        new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null)));
             assertEquals(
-                "Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxScriptFields + "] but was ["
-                    + (maxScriptFields + 1)
-                    + "]. This limit can be set by changing the [index.max_script_fields] index level setting.",
-                ex.getMessage());
+                    "Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxScriptFields + "] but was ["
+                            + (maxScriptFields + 1)
+                            + "]. This limit can be set by changing the [index.max_script_fields] index level setting.",
+                    ex.getMessage());
         }
     }
 
@@ -474,12 +477,10 @@ public void testIgnoreScriptfieldIfSizeZero() throws IOException {
         searchSourceBuilder.scriptField("field" + 0,
                 new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
         searchSourceBuilder.size(0);
-        final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE,
+        try (SearchContext context = service.createContext(new ShardSearchRequest(OriginalIndices.NONE,
             searchRequest, indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY),
-            1.0f, -1, null, null);
-        try (ReaderContext reader = createReaderContext(indexShard, request);
-             SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
-            assertEquals(0, context.scriptFields().fields().size());
+                1.0f, -1, null, null))) {
+                assertEquals(0, context.scriptFields().fields().size());
         }
     }
 
@@ -513,7 +514,7 @@ public void testMaxOpenScrollContexts() throws RuntimeException {
         }
 
         ElasticsearchException ex = expectThrows(ElasticsearchException.class,
-            () -> service.createAndPutReaderContext(new ShardScrollRequestTest(indexShard.shardId())));
+            () -> service.createAndPutContext(new ShardScrollRequestTest(indexShard.shardId())));
         assertEquals(
             "Trying to create too many scroll contexts. Must be less than or equal to: [" +
                 SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY) + "]. " +
@@ -625,11 +626,10 @@ public void testCanMatch() throws IOException {
         assertEquals(numWrapReader, numWrapInvocations.get());
 
         // make sure that the wrapper is called when the context is actually created
-        try (ReaderContext reader = createReaderContext(indexShard, new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
-            indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null));
-             SearchContext context = service.createContext(reader, reader.request(), null, randomBoolean())) {
-        }
-        assertEquals(numWrapReader + 1, numWrapInvocations.get());
+        service.createContext(new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
+            indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1f, -1, null, null)).close();
+        assertEquals(numWrapReader+1, numWrapInvocations.get());
     }
 
     public void testCanRewriteToMatchNone() {
@@ -744,23 +744,18 @@ public void testCreateSearchContextFailure() throws IOException {
         final IndexService indexService = createIndex(index);
         final SearchService service = getInstanceFromNode(SearchService.class);
         final ShardId shardId = new ShardId(indexService.index(), 0);
-        final ShardSearchRequest request = new ShardSearchRequest(shardId, 0, null) {
-            @Override
-            public SearchType searchType() {
-                // induce an artificial NPE
-                throw new NullPointerException("expected");
-            }
-        };
-        try (ReaderContext reader = createReaderContext(indexService.getShard(shardId.id()), request)) {
-            NullPointerException e = expectThrows(NullPointerException.class,
-                () -> service.createContext(reader, request, null, randomBoolean()));
-            assertEquals("expected", e.getMessage());
-        }
-        assertEquals("should have 2 store refs (IndexService + InternalEngine)", 2, indexService.getShard(0).store().refCount());
-    }
 
-    private ReaderContext createReaderContext(IndexShard shard, ShardSearchRequest request) {
-        Engine.Searcher searcher = shard.acquireSearcher("test");
-        return new ReaderContext(randomNonNegativeLong(), shard, searcher, request);
+        NullPointerException e = expectThrows(NullPointerException.class,
+            () -> service.createContext(
+                new ShardSearchRequest(shardId, 0, null) {
+                    @Override
+                    public SearchType searchType() {
+                        // induce an artificial NPE
+                        throw new NullPointerException("expected");
+                    }
+                }
+            ));
+        assertEquals("expected", e.getMessage());
+        assertEquals("should have 2 store refs (IndexService + InternalEngine)", 2, indexService.getShard(0).store().refCount());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
index b2d4314b3a0a5..4c33177e268f9 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
@@ -316,12 +316,13 @@ public void testInOrderScrollOptimization() throws Exception {
         }
         w.close();
         IndexReader reader = DirectoryReader.open(dir);
-        ScrollContext scrollContext = new ScrollContext();
-        TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader), scrollContext);
+        TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader));
         context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
+        ScrollContext scrollContext = new ScrollContext();
         scrollContext.lastEmittedDoc = null;
         scrollContext.maxScore = Float.NaN;
         scrollContext.totalHits = null;
+        context.scrollContext(scrollContext);
         context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()));
         int size = randomIntBetween(2, 5);
         context.setSize(size);
@@ -544,12 +545,13 @@ public void testIndexSortScrollOptimization() throws Exception {
         // search sort is a prefix of the index sort
         searchSortAndFormats.add(new SortAndFormats(new Sort(indexSort.getSort()[0]), new DocValueFormat[]{DocValueFormat.RAW}));
         for (SortAndFormats searchSortAndFormat : searchSortAndFormats) {
-            ScrollContext scrollContext = new ScrollContext();
-            TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader), scrollContext);
+            TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader));
             context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
+            ScrollContext scrollContext = new ScrollContext();
             scrollContext.lastEmittedDoc = null;
             scrollContext.maxScore = Float.NaN;
             scrollContext.totalHits = null;
+            context.scrollContext(scrollContext);
             context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()));
             context.setSize(10);
             context.sort(searchSortAndFormat);
diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
index 1a5c3d7815440..9653f3b66a515 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.fetch.FetchPhase;
-import org.elasticsearch.search.internal.ReaderContext;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.HashMap;
@@ -40,11 +40,11 @@ public class MockSearchService extends SearchService {
      */
     public static class TestPlugin extends Plugin {}
 
-    private static final Map<ReaderContext, Throwable> ACTIVE_SEARCH_CONTEXTS = new ConcurrentHashMap<>();
+    private static final Map<SearchContext, Throwable> ACTIVE_SEARCH_CONTEXTS = new ConcurrentHashMap<>();
 
     /** Throw an {@link AssertionError} if there are still in-flight contexts. */
     public static void assertNoInFlightContext() {
-        final Map<ReaderContext, Throwable> copy = new HashMap<>(ACTIVE_SEARCH_CONTEXTS);
+        final Map<SearchContext, Throwable> copy = new HashMap<>(ACTIVE_SEARCH_CONTEXTS);
         if (copy.isEmpty() == false) {
             throw new AssertionError(
                     "There are still [" + copy.size()
@@ -56,14 +56,14 @@ public static void assertNoInFlightContext() {
     /**
      * Add an active search context to the list of tracked contexts. Package private for testing.
      */
-    static void addActiveContext(ReaderContext context) {
+    static void addActiveContext(SearchContext context) {
         ACTIVE_SEARCH_CONTEXTS.put(context, new RuntimeException(context.toString()));
     }
 
     /**
      * Clear an active search context from the list of tracked contexts. Package private for testing.
      */
-    static void removeActiveContext(ReaderContext context) {
+    static void removeActiveContext(SearchContext context) {
         ACTIVE_SEARCH_CONTEXTS.remove(context);
     }
 
@@ -74,14 +74,14 @@ public MockSearchService(ClusterService clusterService,
     }
 
     @Override
-    protected void putReaderContext(ReaderContext context) {
+    protected void putContext(SearchContext context) {
         addActiveContext(context);
-        super.putReaderContext(context);
+        super.putContext(context);
     }
 
     @Override
-    protected ReaderContext removeReaderContext(long id) {
-        final ReaderContext removed = super.removeReaderContext(id);
+    protected SearchContext removeContext(long id) {
+        final SearchContext removed = super.removeContext(id);
         if (removed != null) {
             removeActiveContext(removed);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index cb461009b7a0d..c671b9c109017 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -249,7 +249,7 @@ public boolean shouldCache(Query query) {
              * close their sub-aggregations. This is fairly similar to what the production code does. */
             releasables.add((Releasable) invocation.getArguments()[0]);
             return null;
-        }).when(searchContext).addReleasable(anyObject());
+        }).when(searchContext).addReleasable(anyObject(), anyObject());
         return searchContext;
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index fc9a8fe7efa62..4edb9adacd13b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -110,18 +110,12 @@ public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexSh
     }
 
     public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexShard, ContextIndexSearcher searcher) {
-        this(queryShardContext, indexShard, searcher, null);
-    }
-
-    public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexShard,
-                             ContextIndexSearcher searcher, ScrollContext scrollContext) {
         this.bigArrays = null;
         this.indexService = null;
         this.fixedBitSetFilterCache = null;
         this.indexShard = indexShard;
         this.queryShardContext = queryShardContext;
         this.searcher = searcher;
-        this.scrollContext = scrollContext;
     }
 
     public void setSearcher(ContextIndexSearcher searcher) {
@@ -172,11 +166,22 @@ public float queryBoost() {
         return 0;
     }
 
+    @Override
+    public long getOriginNanoTime() {
+        return originNanoTime;
+    }
+
     @Override
     public ScrollContext scrollContext() {
         return scrollContext;
     }
 
+    @Override
+    public SearchContext scrollContext(ScrollContext scrollContext) {
+        this.scrollContext = scrollContext;
+        return this;
+    }
+
     @Override
     public SearchContextAggregations aggregations() {
         return aggregations;
@@ -221,6 +226,10 @@ public List<RescoreContext> rescore() {
         return Collections.emptyList();
     }
 
+    @Override
+    public void addRescore(RescoreContext rescore) {
+    }
+
     @Override
     public boolean hasScriptFields() {
         return false;
@@ -527,6 +536,24 @@ public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int
         return null;
     }
 
+    @Override
+    public void accessed(long accessTime) {
+    }
+
+    @Override
+    public long lastAccessTime() {
+        return 0;
+    }
+
+    @Override
+    public long keepAlive() {
+        return 0;
+    }
+
+    @Override
+    public void keepAlive(long keepAlive) {
+    }
+
     @Override
     public DfsSearchResult dfsResult() {
         return null;
diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
index b49f1fb0acaf5..8a8842487f14a 100644
--- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
@@ -19,25 +19,59 @@
 
 package org.elasticsearch.search;
 
-import org.elasticsearch.search.internal.ReaderContext;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
-
-import static org.mockito.Mockito.mock;
+import org.elasticsearch.test.TestSearchContext;
 
 public class MockSearchServiceTests extends ESTestCase {
+    public static final IndexMetaData EMPTY_INDEX_METADATA = IndexMetaData.builder("")
+        .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+        .numberOfShards(1).numberOfReplicas(0).build();
 
     public void testAssertNoInFlightContext() {
-        ReaderContext reader = mock(ReaderContext.class);
-        MockSearchService.addActiveContext(reader);
+        final long nowInMillis = randomNonNegativeLong();
+        SearchContext s = new TestSearchContext(new QueryShardContext(0,
+            new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null,
+            xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null)) {
+
+            @Override
+            public SearchShardTarget shardTarget() {
+                return new SearchShardTarget("node", new ShardId("idx", "ignored", 0), null, OriginalIndices.NONE);
+            }
+
+            @Override
+            public SearchType searchType() {
+                return SearchType.DEFAULT;
+            }
+
+            @Override
+            public Query query() {
+                return Queries.newMatchAllQuery();
+            }
+        };
+        MockSearchService.addActiveContext(s);
         try {
             Throwable e = expectThrows(AssertionError.class, () -> MockSearchService.assertNoInFlightContext());
             assertEquals("There are still [1] in-flight contexts. The first one's creation site is listed as the cause of this exception.",
                     e.getMessage());
             e = e.getCause();
+            // The next line with throw an exception if the date looks wrong
+            assertEquals("[node][idx][0] query=[*:*]", e.getMessage());
             assertEquals(MockSearchService.class.getName(), e.getStackTrace()[0].getClassName());
             assertEquals(MockSearchServiceTests.class.getName(), e.getStackTrace()[1].getClassName());
         } finally {
-            MockSearchService.removeActiveContext(reader);
+            MockSearchService.removeActiveContext(s);
         }
     }
 }
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index e78e937a1f142..5a22857494557 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.shard.SearchOperationListener;
-import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.transport.TransportRequest;
 
@@ -286,34 +285,27 @@ public void validateSearchContext(SearchContext context, TransportRequest transp
                 } catch (IOException e) {
                     throw new UncheckedIOException(e);
                 }
+                // also register a release resource in this case if we have multiple roundtrips like in DFS
+                registerRelease(context, lazyDirectoryReader);
             }
         }
 
-        @Override
-        public void onFreeSearchContext(SearchContext context) {
-            DirectoryReader dirReader = context.searcher().getDirectoryReader();
-            LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
-            if (lazyDirectoryReader != null) {
+        private void registerRelease(SearchContext context, LazyDirectoryReader lazyDirectoryReader) {
+            context.addReleasable(() -> {
                 try {
                     lazyDirectoryReader.release();
                 } catch (IOException e) {
                     throw new UncheckedIOException(e);
                 }
-            }
+            }, SearchContext.Lifetime.PHASE);
         }
 
         @Override
-        public void onNewReaderContext(ReaderContext readerContext) {
-            DirectoryReader dirReader = readerContext.engineSearcher().getDirectoryReader();
+        public void onNewContext(SearchContext context) {
+            DirectoryReader dirReader = context.searcher().getDirectoryReader();
             LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
             if (lazyDirectoryReader != null) {
-                readerContext.addOnClose(() -> {
-                    try {
-                        lazyDirectoryReader.release();
-                    } catch (IOException e) {
-                        throw new UncheckedIOException(e);
-                    }
-                });
+                registerRelease(context, lazyDirectoryReader);
             }
         }
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
index aef1652b42fc5..5e0c2945caadc 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
@@ -46,9 +46,10 @@ public SecuritySearchOperationListener(ThreadContext threadContext, XPackLicense
      * Adds the {@link Authentication} to the {@link ScrollContext}
      */
     @Override
-    public void onNewScrollContext(ScrollContext scrollContext) {
+    public void onNewScrollContext(SearchContext searchContext) {
         if (licenseState.isAuthAllowed()) {
-            scrollContext.putInContext(AuthenticationField.AUTHENTICATION_KEY, Authentication.getAuthentication(threadContext));
+            searchContext.scrollContext().putInContext(AuthenticationField.AUTHENTICATION_KEY,
+                    Authentication.getAuthentication(threadContext));
         }
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index 05f0ed7f0c42f..5b73d6d212fc5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -49,19 +49,18 @@ public void testUnlicensed() {
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         AuditTrailService auditTrailService = mock(AuditTrailService.class);
         SearchContext searchContext = mock(SearchContext.class);
-        ScrollContext scrollContext = new ScrollContext();
-        when(searchContext.scrollContext()).thenReturn(scrollContext);
+        when(searchContext.scrollContext()).thenReturn(new ScrollContext());
 
         SecuritySearchOperationListener listener = new SecuritySearchOperationListener(threadContext, licenseState, auditTrailService);
-        listener.onNewScrollContext(scrollContext);
+        listener.onNewScrollContext(searchContext);
         listener.validateSearchContext(searchContext, Empty.INSTANCE);
         verify(licenseState, times(2)).isAuthAllowed();
         verifyZeroInteractions(auditTrailService, searchContext);
     }
 
     public void testOnNewContextSetsAuthentication() throws Exception {
-        ScrollContext scrollContext = new ScrollContext();
-        TestSearchContext testSearchContext = new TestSearchContext(null, null, null, scrollContext);
+        TestScrollSearchContext testSearchContext = new TestScrollSearchContext();
+        testSearchContext.scrollContext(new ScrollContext());
         final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(2L));
         testSearchContext.scrollContext().scroll = scroll;
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
@@ -72,7 +71,7 @@ public void testOnNewContextSetsAuthentication() throws Exception {
         authentication.writeToContext(threadContext);
 
         SecuritySearchOperationListener listener = new SecuritySearchOperationListener(threadContext, licenseState, auditTrailService);
-        listener.onNewScrollContext(scrollContext);
+        listener.onNewScrollContext(testSearchContext);
 
         Authentication contextAuth = testSearchContext.scrollContext().getFromContext(AuthenticationField.AUTHENTICATION_KEY);
         assertEquals(authentication, contextAuth);
@@ -83,8 +82,8 @@ public void testOnNewContextSetsAuthentication() throws Exception {
     }
 
     public void testValidateSearchContext() throws Exception {
-        ScrollContext scrollContext = new ScrollContext();
-        TestSearchContext testSearchContext = new TestSearchContext(null, null, null, scrollContext);
+        TestScrollSearchContext testSearchContext = new TestScrollSearchContext();
+        testSearchContext.scrollContext(new ScrollContext());
         testSearchContext.scrollContext().putInContext(AuthenticationField.AUTHENTICATION_KEY,
                 new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null));
         testSearchContext.scrollContext().scroll = new Scroll(TimeValue.timeValueSeconds(2L));
@@ -236,4 +235,24 @@ public void testEnsuredAuthenticatedUserIsSame() {
         verify(auditTrail).accessDenied(eq(auditId), eq(runAsDiffType), eq(action), eq(request),
             authzInfoRoles(original.getUser().roles()));
     }
+
+    static class TestScrollSearchContext extends TestSearchContext {
+
+        private ScrollContext scrollContext;
+
+        TestScrollSearchContext() {
+            super(null);
+        }
+
+        @Override
+        public ScrollContext scrollContext() {
+            return scrollContext;
+        }
+
+        @Override
+        public SearchContext scrollContext(ScrollContext scrollContext) {
+            this.scrollContext = scrollContext;
+            return this;
+        }
+    }
 }

From fb7386ab7355cc487d6502915b71a603f9d216b9 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 27 Jan 2020 15:38:56 -0500
Subject: [PATCH 03/47] Cut over from SearchContext to ReaderContext (#51282)

With this change, we partially move the state of SearchContext to
ReaderContext. This is another step allowing us to move the state of
search to the coordinating node.

We will need several follow-ups to move the entire search state to the
coordinating node.

Relates #46523
---
 .../ParentChildInnerHitContextBuilder.java    |   8 +-
 .../action/search/SearchTransportService.java |   4 +-
 .../index/query/NestedQueryBuilder.java       |   6 +-
 .../index/search/stats/ShardSearchStats.java  |  12 +-
 .../index/shard/SearchOperationListener.java  |  57 ++-
 .../search/DefaultSearchContext.java          |  88 ++---
 .../elasticsearch/search/SearchService.java   | 329 +++++++-----------
 .../search/aggregations/AggregationPhase.java |   2 -
 .../search/aggregations/AggregatorBase.java   |   3 +-
 .../aggregations/AggregatorFactory.java       |   3 +-
 .../search/fetch/subphase/ExplainPhase.java   |   2 -
 .../fetch/subphase/MatchedQueriesPhase.java   |   3 -
 .../internal/FilteredSearchContext.java       |  35 --
 .../search/internal/ReaderContext.java        | 155 +++++++++
 .../search/internal/ScrollContext.java        |   7 +
 .../search/internal/SearchContext.java        | 102 +-----
 .../search/internal/SubSearchContext.java     |  21 --
 .../search/query/QueryPhase.java              |   4 -
 .../elasticsearch/index/IndexModuleTests.java |  10 +-
 .../shard/SearchOperationListenerTests.java   |  27 +-
 .../search/DefaultSearchContextTests.java     |  40 ++-
 .../search/SearchServiceTests.java            | 143 ++++----
 .../search/query/QueryPhaseTests.java         |  10 +-
 .../search/MockSearchService.java             |  18 +-
 .../aggregations/AggregatorTestCase.java      |   2 +-
 .../elasticsearch/test/TestSearchContext.java |  39 +--
 .../search/MockSearchServiceTests.java        |  46 +--
 .../index/engine/FrozenEngine.java            |  24 +-
 .../SecuritySearchOperationListener.java      |   4 +-
 .../SecuritySearchOperationListenerTests.java |  35 +-
 30 files changed, 542 insertions(+), 697 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java

diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
index 5aa35d46bc384..5f04391c1889a 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
@@ -158,12 +158,8 @@ public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException {
                         topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE);
                         maxScoreCollector = new MaxScoreCollector();
                     }
-                    try {
-                        for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) {
-                            intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
-                        }
-                    } finally {
-                        clearReleasables(Lifetime.COLLECTION);
+                    for (LeafReaderContext ctx : context.searcher().getIndexReader().leaves()) {
+                        intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
                     }
                     TopDocs topDocs = topDocsCollector.topDocs(from(), size());
                     float maxScore = Float.NaN;
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 16e8c17688906..53d8d266ea795 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -285,13 +285,13 @@ public void writeTo(StreamOutput out) throws IOException {
     public static void registerRequestHandler(TransportService transportService, SearchService searchService) {
         transportService.registerRequestHandler(FREE_CONTEXT_SCROLL_ACTION_NAME, ThreadPool.Names.SAME, ScrollFreeContextRequest::new,
             (request, channel, task) -> {
-                boolean freed = searchService.freeContext(request.id());
+                boolean freed = searchService.freeReaderContext(request.id());
                 channel.sendResponse(new SearchFreeContextResponse(freed));
         });
         TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_SCROLL_ACTION_NAME, SearchFreeContextResponse::new);
         transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, ThreadPool.Names.SAME, SearchFreeContextRequest::new,
             (request, channel, task) -> {
-                boolean freed = searchService.freeContext(request.id());
+                boolean freed = searchService.freeReaderContext(request.id());
                 channel.sendResponse(new SearchFreeContextResponse(freed));
         });
         TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, SearchFreeContextResponse::new);
diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index a560af6826af9..69bf7027442e5 100644
--- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -428,11 +428,7 @@ public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException {
                         topDocsCollector = TopScoreDocCollector.create(topN, Integer.MAX_VALUE);
                         maxScoreCollector = new MaxScoreCollector();
                     }
-                    try {
-                        intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
-                    } finally {
-                        clearReleasables(Lifetime.COLLECTION);
-                    }
+                    intersect(weight, innerHitQueryWeight, MultiCollector.wrap(topDocsCollector, maxScoreCollector), ctx);
 
                     TopDocs td = topDocsCollector.topDocs(from(), size());
                     float maxScore = Float.NaN;
diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
index 5446b1ed0e372..664b631249f68 100644
--- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
+++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
@@ -24,6 +24,8 @@
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.index.shard.SearchOperationListener;
+import org.elasticsearch.search.internal.ReaderContext;
+import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.util.HashMap;
@@ -146,25 +148,25 @@ private StatsHolder groupStats(String group) {
     }
 
     @Override
-    public void onNewContext(SearchContext context) {
+    public void onNewReaderContext(ReaderContext readerContext) {
         openContexts.inc();
     }
 
     @Override
-    public void onFreeContext(SearchContext context) {
+    public void onFreeReaderContext(ReaderContext readerContext) {
         openContexts.dec();
     }
 
     @Override
-    public void onNewScrollContext(SearchContext context) {
+    public void onNewScrollContext(ScrollContext scrollContext) {
         totalStats.scrollCurrent.inc();
     }
 
     @Override
-    public void onFreeScrollContext(SearchContext context) {
+    public void onFreeScrollContext(ScrollContext scrollContext) {
         totalStats.scrollCurrent.dec();
         assert totalStats.scrollCurrent.count() >= 0;
-        totalStats.scrollMetric.inc(TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - context.getOriginNanoTime()));
+        totalStats.scrollMetric.inc(TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - scrollContext.getStartTimeInNano()));
     }
 
     static final class StatsHolder {
diff --git a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
index ede86e6ec222d..77e83edcdd18f 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
@@ -21,6 +21,8 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.search.internal.ReaderContext;
+import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.transport.TransportRequest;
 
@@ -76,34 +78,34 @@ default void onFailedFetchPhase(SearchContext searchContext) {}
     default void onFetchPhase(SearchContext searchContext, long tookInNanos) {}
 
     /**
-     * Executed when a new search context was created
-     * @param context the created context
+     * Executed when a new reader context was created
+     * @param readerContext the created context
      */
-    default void onNewContext(SearchContext context) {}
+    default void onNewReaderContext(ReaderContext readerContext) {}
 
     /**
-     * Executed when a previously created search context is freed.
+     * Executed when a previously created reader context is freed.
      * This happens either when the search execution finishes, if the
      * execution failed or if the search context as idle for and needs to be
      * cleaned up.
-     * @param context the freed search context
+     * @param readerContext the freed reader context
      */
-    default void onFreeContext(SearchContext context) {}
+    default void onFreeReaderContext(ReaderContext readerContext) {}
 
     /**
      * Executed when a new scroll search {@link SearchContext} was created
-     * @param context the created search context
+     * @param scrollContext the created search context
      */
-    default void onNewScrollContext(SearchContext context) {}
+    default void onNewScrollContext(ScrollContext scrollContext) {}
 
     /**
      * Executed when a scroll search {@link SearchContext} is freed.
      * This happens either when the scroll search execution finishes, if the
      * execution failed or if the search context as idle for and needs to be
      * cleaned up.
-     * @param context the freed search context
+     * @param scrollContext the freed search context
      */
-    default void onFreeScrollContext(SearchContext context) {}
+    default void onFreeScrollContext(ScrollContext scrollContext) {}
 
     /**
      * Executed prior to using a {@link SearchContext} that has been retrieved
@@ -114,6 +116,14 @@ default void onFreeScrollContext(SearchContext context) {}
      */
     default void validateSearchContext(SearchContext context, TransportRequest transportRequest) {}
 
+    /**
+     * Executed when a search context was freed. The implementor can implement
+     * this method to release resources used by the search context.
+     */
+    default void onFreeSearchContext(SearchContext context) {
+
+    }
+
     /**
      * A Composite listener that multiplexes calls to each of the listeners methods.
      */
@@ -193,10 +203,10 @@ public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
         }
 
         @Override
-        public void onNewContext(SearchContext context) {
+        public void onNewReaderContext(ReaderContext readerContext) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onNewContext(context);
+                    listener.onNewReaderContext(readerContext);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onNewContext listener [{}] failed", listener), e);
                 }
@@ -204,10 +214,10 @@ public void onNewContext(SearchContext context) {
         }
 
         @Override
-        public void onFreeContext(SearchContext context) {
+        public void onFreeReaderContext(ReaderContext readerContext) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onFreeContext(context);
+                    listener.onFreeReaderContext(readerContext);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onFreeContext listener [{}] failed", listener), e);
                 }
@@ -215,10 +225,10 @@ public void onFreeContext(SearchContext context) {
         }
 
         @Override
-        public void onNewScrollContext(SearchContext context) {
+        public void onNewScrollContext(ScrollContext scrollContext) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onNewScrollContext(context);
+                    listener.onNewScrollContext(scrollContext);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onNewScrollContext listener [{}] failed", listener), e);
                 }
@@ -226,10 +236,10 @@ public void onNewScrollContext(SearchContext context) {
         }
 
         @Override
-        public void onFreeScrollContext(SearchContext context) {
+        public void onFreeScrollContext(ScrollContext scrollContext) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onFreeScrollContext(context);
+                    listener.onFreeScrollContext(scrollContext);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onFreeScrollContext listener [{}] failed", listener), e);
                 }
@@ -248,5 +258,16 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
             }
             ExceptionsHelper.reThrowIfNotNull(exception);
         }
+
+        @Override
+        public void onFreeSearchContext(SearchContext context) {
+            for (SearchOperationListener listener : listeners) {
+                try {
+                    listener.onFreeSearchContext(context);
+                } catch (Exception e) {
+                    logger.warn(() -> new ParameterizedMessage("onFreeSearchContext listener [{}] failed", listener), e);
+                }
+            }
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 9ab786a77ffe4..b293985c3a877 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.WeightFactorFunction;
@@ -61,6 +60,7 @@
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
@@ -75,7 +75,6 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -83,12 +82,11 @@
 
 final class DefaultSearchContext extends SearchContext {
 
-    private final long id;
+    private final ReaderContext readerContext;
     private final ShardSearchRequest request;
     private final SearchShardTarget shardTarget;
     private final LongSupplier relativeTimeSupplier;
     private SearchType searchType;
-    private final Engine.Searcher engineSearcher;
     private final BigArrays bigArrays;
     private final IndexShard indexShard;
     private final ClusterService clusterService;
@@ -102,7 +100,6 @@ final class DefaultSearchContext extends SearchContext {
     // terminate after count
     private int terminateAfter = DEFAULT_TERMINATE_AFTER;
     private List<String> groupStats;
-    private ScrollContext scrollContext;
     private boolean explain;
     private boolean version = false; // by default, we don't return versions
     private boolean seqAndPrimaryTerm = false;
@@ -142,10 +139,6 @@ final class DefaultSearchContext extends SearchContext {
     private SearchContextAggregations aggregations;
     private SearchContextHighlight highlight;
     private SuggestionSearchContext suggest;
-    private List<RescoreContext> rescore;
-    private volatile long keepAlive;
-    private final long originNanoTime = System.nanoTime();
-    private volatile long lastAccessTime = -1;
     private Profilers profilers;
 
     private final Map<String, SearchExtBuilder> searchExtBuilders = new HashMap<>();
@@ -153,36 +146,36 @@ final class DefaultSearchContext extends SearchContext {
     private final QueryShardContext queryShardContext;
     private final FetchPhase fetchPhase;
 
-    DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget,
-                         Engine.Searcher engineSearcher, ClusterService clusterService, IndexService indexService,
-                         IndexShard indexShard, BigArrays bigArrays, LongSupplier relativeTimeSupplier, TimeValue timeout,
-                         FetchPhase fetchPhase) {
-        this.id = id;
+    DefaultSearchContext(ReaderContext readerContext, ShardSearchRequest request, SearchShardTarget shardTarget,
+                         ClusterService clusterService, IndexService indexService, IndexShard indexShard, BigArrays bigArrays,
+                         LongSupplier relativeTimeSupplier, TimeValue timeout, FetchPhase fetchPhase) {
+        this.readerContext = readerContext;
         this.request = request;
         this.fetchPhase = fetchPhase;
         this.searchType = request.searchType();
         this.shardTarget = shardTarget;
-        this.engineSearcher = engineSearcher;
         // SearchContexts use a BigArrays that can circuit break
         this.bigArrays = bigArrays.withCircuitBreaking();
-        this.dfsResult = new DfsSearchResult(id, shardTarget);
-        this.queryResult = new QuerySearchResult(id, shardTarget);
-        this.fetchResult = new FetchSearchResult(id, shardTarget);
+        this.dfsResult = new DfsSearchResult(readerContext.id(), shardTarget);
+        this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget);
+        this.fetchResult = new FetchSearchResult(readerContext.id(), shardTarget);
         this.indexShard = indexShard;
         this.indexService = indexService;
         this.clusterService = clusterService;
+        final Engine.Searcher engineSearcher = readerContext.engineSearcher();
         this.searcher = new ContextIndexSearcher(engineSearcher.getIndexReader(), engineSearcher.getSimilarity(),
             engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy());
+        searcher.setAggregatedDfs(readerContext.aggregatedDfs());
         this.relativeTimeSupplier = relativeTimeSupplier;
         this.timeout = timeout;
-        queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher,
+        queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher,
             request::nowInMillis, shardTarget.getClusterAlias());
         queryBoost = request.indexBoost();
     }
 
     @Override
     public void doClose() {
-        Releasables.close(engineSearcher);
+        readerContext.decRef();
     }
 
     /**
@@ -199,7 +192,7 @@ public void preProcess(boolean rewrite) {
         int maxResultWindow = indexService.getIndexSettings().getMaxResultWindow();
 
         if (resultWindow > maxResultWindow) {
-            if (scrollContext == null) {
+            if (scrollContext() == null) {
                 throw new IllegalArgumentException(
                         "Result window is too large, from + size must be less than or equal to: [" + maxResultWindow + "] but was ["
                                 + resultWindow + "]. See the scroll api for a more efficient way to request large data sets. "
@@ -211,12 +204,12 @@ public void preProcess(boolean rewrite) {
                             + "]. Scroll batch sizes cost as much memory as result windows so they are controlled by the ["
                             + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey() + "] index level setting.");
         }
-        if (rescore != null) {
+        if (rescore().isEmpty() == false) {
             if (sort != null) {
                 throw new IllegalArgumentException("Cannot use [sort] option in conjunction with [rescore].");
             }
             int maxWindow = indexService.getIndexSettings().getMaxRescoreWindow();
-            for (RescoreContext rescoreContext: rescore) {
+            for (RescoreContext rescoreContext: rescore()) {
                 if (rescoreContext.getWindowSize() > maxWindow) {
                     throw new IllegalArgumentException("Rescore window [" + rescoreContext.getWindowSize() + "] is too large. "
                             + "It must be less than [" + maxWindow + "]. This prevents allocating massive heaps for storing the results "
@@ -297,12 +290,12 @@ && new NestedHelper(mapperService()).mightMatchNestedDocs(query)
 
     @Override
     public long id() {
-        return this.id;
+        return readerContext.id();
     }
 
     @Override
     public String source() {
-        return engineSearcher.source();
+        return readerContext.source();
     }
 
     @Override
@@ -330,21 +323,11 @@ public float queryBoost() {
         return queryBoost;
     }
 
-    @Override
-    public long getOriginNanoTime() {
-        return originNanoTime;
-    }
-
     @Override
     public ScrollContext scrollContext() {
-        return this.scrollContext;
+        return readerContext.scrollContext();
     }
 
-    @Override
-    public SearchContext scrollContext(ScrollContext scrollContext) {
-        this.scrollContext = scrollContext;
-        return this;
-    }
 
     @Override
     public SearchContextAggregations aggregations() {
@@ -391,18 +374,7 @@ public void suggest(SuggestionSearchContext suggest) {
 
     @Override
     public List<RescoreContext> rescore() {
-        if (rescore == null) {
-            return Collections.emptyList();
-        }
-        return rescore;
-    }
-
-    @Override
-    public void addRescore(RescoreContext rescore) {
-        if (this.rescore == null) {
-            this.rescore = new ArrayList<>();
-        }
-        this.rescore.add(rescore);
+        return readerContext.rescore();
     }
 
     @Override
@@ -736,26 +708,6 @@ public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int
         return this;
     }
 
-    @Override
-    public void accessed(long accessTime) {
-        this.lastAccessTime = accessTime;
-    }
-
-    @Override
-    public long lastAccessTime() {
-        return this.lastAccessTime;
-    }
-
-    @Override
-    public long keepAlive() {
-        return this.keepAlive;
-    }
-
-    @Override
-    public void keepAlive(long keepAlive) {
-        this.keepAlive = keepAlive;
-    }
-
     @Override
     public DfsSearchResult dfsResult() {
         return dfsResult;
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index c6531fabf264d..e856e8a12a374 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -84,9 +85,8 @@
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.ScrollContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContext.Lifetime;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhase;
@@ -104,7 +104,6 @@
 import org.elasticsearch.threadpool.Scheduler.Cancellable;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Names;
-import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -193,7 +192,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
 
     private final AtomicLong idGenerator = new AtomicLong();
 
-    private final ConcurrentMapLong<SearchContext> activeContexts = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
+    private final ConcurrentMapLong<ReaderContext> activeReaders = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
 
     private final MultiBucketConsumerService multiBucketConsumerService;
 
@@ -283,13 +282,13 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem
 
     }
 
-    protected void putContext(SearchContext context) {
-        final SearchContext previous = activeContexts.put(context.id(), context);
+    protected void putReaderContext(ReaderContext context) {
+        final ReaderContext previous = activeReaders.put(context.id(), context);
         assert previous == null;
     }
 
-    protected SearchContext removeContext(long id) {
-        return activeContexts.remove(id);
+    protected ReaderContext removeReaderContext(long id) {
+        return activeReaders.remove(id);
     }
 
     @Override
@@ -298,8 +297,8 @@ protected void doStart() {
 
     @Override
     protected void doStop() {
-        for (final SearchContext context : activeContexts.values()) {
-            freeContext(context.id());
+        for (final ReaderContext context : activeReaders.values()) {
+            freeReaderContext(context.id());
         }
     }
 
@@ -333,21 +332,15 @@ public void executeDfsPhase(ShardSearchRequest request, SearchShardTask task, Ac
             }, listener::onFailure));
     }
 
-    private DfsSearchResult executeDfsPhase(SearchRewriteContext rewriteContext, SearchShardTask task) throws IOException {
-        final SearchContext context = createAndPutContext(rewriteContext);
-        context.incRef();
-        try {
-            context.setTask(task);
-            contextProcessing(context);
+    private DfsSearchResult executeDfsPhase(SearchRewriteContext rewriteContext, SearchShardTask task) {
+        final ReaderContext reader = createAndPutReaderContext(rewriteContext);
+        try (SearchContext context = createContext(reader, reader.request(), task, true)) {
             dfsPhase.execute(context);
-            contextProcessedSuccessfully(context);
             return context.dfsResult();
         } catch (Exception e) {
             logger.trace("Dfs phase failed", e);
-            processFailure(context, e);
+            processFailure(reader, e);
             throw e;
-        } finally {
-            cleanContext(context);
         }
     }
 
@@ -399,12 +392,8 @@ && canRewriteToMatchNone(rewritten.source())
     }
 
     private void onMatchNoDocs(SearchRewriteContext rewriteContext, ActionListener<SearchPhaseResult> listener) {
-        // creates a lightweight search context that we use to inform context listeners
-        // before closing
-        SearchContext searchContext = createSearchContext(rewriteContext, defaultSearchTimeout);
-        try (searchContext) {
-            onNewContext(searchContext);
-            onFreeContext(searchContext);
+        try {
+            rewriteContext.searcher.close();
         } catch (Exception exc) {
             listener.onFailure(exc);
             return;
@@ -417,19 +406,14 @@ private <T> void runAsync(long id, Supplier<T> executable, ActionListener<T> lis
     }
 
     private SearchPhaseResult executeQueryPhase(SearchRewriteContext rewriteContext, SearchShardTask task) throws Exception {
-        final SearchContext context = createAndPutContext(rewriteContext);
+        final ReaderContext reader = createAndPutReaderContext(rewriteContext);
         final ShardSearchRequest request = rewriteContext.request;
-        context.incRef();
-        try {
-            context.setTask(task);
+        try (SearchContext context = createContext(reader, reader.request(), task, true)) {
             final long afterQueryTime;
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
-                contextProcessing(context);
                 loadOrExecuteQueryPhase(request, context);
                 if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) {
-                    freeContext(context.id());
-                } else {
-                    contextProcessedSuccessfully(context);
+                    freeReaderContext(reader.id());
                 }
                 afterQueryTime = executor.success();
             }
@@ -444,10 +428,8 @@ private SearchPhaseResult executeQueryPhase(SearchRewriteContext rewriteContext,
                     (Exception) e.getCause() : new ElasticsearchException(e.getCause());
             }
             logger.trace("Query phase failed", e);
-            processFailure(context, e);
+            processFailure(reader, e);
             throw e;
-        } finally {
-            cleanContext(context);
         }
     }
 
@@ -456,9 +438,7 @@ private QueryFetchSearchResult executeFetchPhase(SearchContext context, long aft
             shortcutDocIdsToLoad(context);
             fetchPhase.execute(context);
             if (fetchPhaseShouldFreeContext(context)) {
-                freeContext(context.id());
-            } else {
-                contextProcessedSuccessfully(context);
+                freeReaderContext(context.id());
             }
             executor.success();
         }
@@ -469,49 +449,41 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
         runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
-            context.incRef();
-            try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
-                context.setTask(task);
-                contextProcessing(context);
-                processScroll(request, context);
+            final ReaderContext reader = findReaderContext(request.id());
+            try (SearchContext context = createContext(reader, reader.request(), task, false);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
+                processScroll(request, reader, context);
                 queryPhase.execute(context);
-                contextProcessedSuccessfully(context);
                 executor.success();
                 return new ScrollQuerySearchResult(context.queryResult(), context.shardTarget());
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
-                processFailure(context, e);
+                processFailure(reader, e);
                 throw e;
-            } finally {
-                cleanContext(context);
             }
         }, listener);
     }
 
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
         runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
-            context.setTask(task);
-            context.incRef();
-            try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
-                contextProcessing(context);
+            final ReaderContext reader = findReaderContext(request.id());
+            reader.aggregatedDfs(request.dfs());
+            try (SearchContext context = createContext(reader, reader.request(), task, true);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 context.searcher().setAggregatedDfs(request.dfs());
                 queryPhase.execute(context);
                 if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) {
                     // no hits, we can release the context since there will be no fetch phase
-                    freeContext(context.id());
-                } else {
-                    contextProcessedSuccessfully(context);
+                    freeReaderContext(reader.id());
                 }
                 executor.success();
                 return context.queryResult();
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
-                processFailure(context, e);
+                processFailure(reader, e);
                 throw e;
-            } finally {
-                cleanContext(context);
             }
         }, listener);
     }
@@ -527,11 +499,7 @@ private boolean fetchPhaseShouldFreeContext(SearchContext context) {
     }
 
     final Executor getExecutor(long id) {
-        SearchContext context = activeContexts.get(id);
-        if (context == null) {
-            throw new SearchContextMissingException(id);
-        }
-        return getExecutor(context.indexShard());
+        return getExecutor(findReaderContext(id).indexShard());
     }
 
     private Executor getExecutor(IndexShard indexShard) {
@@ -542,33 +510,28 @@ private Executor getExecutor(IndexShard indexShard) {
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
         runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
-            context.setTask(task);
-            context.incRef();
-            try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)){
-                contextProcessing(context);
-                processScroll(request, context);
+            final ReaderContext reader = findReaderContext(request.id());
+            try (SearchContext context = createContext(reader, reader.request(), task, false);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
+                processScroll(request, reader, context);
                 queryPhase.execute(context);
                 final long afterQueryTime = executor.success();
                 QueryFetchSearchResult fetchSearchResult = executeFetchPhase(context, afterQueryTime);
                 return new ScrollQueryFetchSearchResult(fetchSearchResult, context.shardTarget());
             } catch (Exception e) {
                 logger.trace("Fetch phase failed", e);
-                processFailure(context, e);
+                processFailure(reader, e);
                 throw e;
-            } finally {
-                cleanContext(context);
             }
         }, listener);
     }
 
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
         runAsync(request.id(), () -> {
-            final SearchContext context = findContext(request.id(), request);
-            context.incRef();
-            try {
-                context.setTask(task);
-                contextProcessing(context);
+            final ReaderContext reader = findReaderContext(request.id());
+            try (SearchContext context = createContext(reader, reader.request(), task, false)) {
+                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 if (request.lastEmittedDoc() != null) {
                     context.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
                 }
@@ -576,89 +539,73 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
                 try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, System.nanoTime())) {
                     fetchPhase.execute(context);
                     if (fetchPhaseShouldFreeContext(context)) {
-                        freeContext(request.id());
-                    } else {
-                        contextProcessedSuccessfully(context);
+                        freeReaderContext(request.id());
                     }
                     executor.success();
                 }
                 return context.fetchResult();
             } catch (Exception e) {
                 logger.trace("Fetch phase failed", e);
-                processFailure(context, e);
+                processFailure(reader, e);
                 throw e;
-            } finally {
-                cleanContext(context);
             }
         }, listener);
     }
 
-    private SearchContext findContext(long id, TransportRequest request) throws SearchContextMissingException {
-        SearchContext context = activeContexts.get(id);
-        if (context == null) {
+    private ReaderContext findReaderContext(long id) throws SearchContextMissingException {
+        final ReaderContext reader = activeReaders.get(id);
+        if (reader == null) {
             throw new SearchContextMissingException(id);
         }
-
-        SearchOperationListener operationListener = context.indexShard().getSearchOperationListener();
-        try {
-            operationListener.validateSearchContext(context, request);
-            return context;
-        } catch (Exception e) {
-            processFailure(context, e);
-            throw e;
-        }
+        return reader;
     }
 
-    final SearchContext createAndPutContext(SearchRewriteContext rewriteContext) {
-        SearchContext context = createContext(rewriteContext);
-        onNewContext(context);
-        boolean success = false;
+    final ReaderContext createAndPutReaderContext(SearchRewriteContext rewriteContext) {
+        final ShardSearchRequest request = rewriteContext.request;
+        final SearchOperationListener searchOperationListener = rewriteContext.shard.getSearchOperationListener();
+        Engine.Searcher engineSearcher = rewriteContext.searcher;
+        ReaderContext readerContext = null;
         try {
-            putContext(context);
-            success = true;
-            return context;
-        } finally {
-            if (success == false) {
-                freeContext(context.id());
+            if (request.scroll() != null && openScrollContexts.get() >= maxOpenScrollContext) {
+                throw new ElasticsearchException(
+                    "Trying to create too many scroll contexts. Must be less than or equal to: [" +
+                        maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
+                        + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
             }
-        }
-    }
-
-    private void onNewContext(SearchContext context) {
-        boolean success = false;
-        try {
-            if (context.scrollContext() != null) {
+            readerContext = new ReaderContext(idGenerator.incrementAndGet(), rewriteContext.shard, engineSearcher, request);
+            engineSearcher = null;
+            final ReaderContext finalReaderContext = readerContext;
+            searchOperationListener.onNewReaderContext(finalReaderContext);
+            if (finalReaderContext.scrollContext() != null) {
+                searchOperationListener.onNewScrollContext(finalReaderContext.scrollContext());
                 openScrollContexts.incrementAndGet();
-                context.indexShard().getSearchOperationListener().onNewScrollContext(context);
             }
-            context.indexShard().getSearchOperationListener().onNewContext(context);
-            success = true;
-        } finally {
-            // currently, the concrete listener is CompositeListener, which swallows exceptions, but here we anyway try to do the
-            // right thing by closing and notifying onFreeXXX in case one of the listeners fails with an exception in the future.
-            if (success == false) {
-                try (context) {
-                    onFreeContext(context);
+            readerContext.addOnClose(() -> {
+                try {
+                    if (finalReaderContext.scrollContext() != null) {
+                        openScrollContexts.decrementAndGet();
+                        searchOperationListener.onFreeScrollContext(finalReaderContext.scrollContext());
+                    }
+                } finally {
+                    searchOperationListener.onFreeReaderContext(finalReaderContext);
                 }
-            }
+            });
+            putReaderContext(finalReaderContext);
+            readerContext = null;
+            return finalReaderContext;
+        } finally {
+            Releasables.close(engineSearcher, readerContext);
         }
     }
 
-    final SearchContext createContext(SearchRewriteContext rewriteContext) {
-        final DefaultSearchContext context = createSearchContext(rewriteContext, defaultSearchTimeout);
+    final SearchContext createContext(ReaderContext reader, ShardSearchRequest request, SearchShardTask task, boolean includeAggregations) {
+        final DefaultSearchContext context = createSearchContext(reader, request, defaultSearchTimeout);
         try {
-            if (rewriteContext.request != null && openScrollContexts.get() >= maxOpenScrollContext) {
-                throw new ElasticsearchException(
-                    "Trying to create too many scroll contexts. Must be less than or equal to: [" +
-                        maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
-                        + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
-            }
-            final ShardSearchRequest request = rewriteContext.request;
+            context.setTask(task);
             if (request.scroll() != null) {
-                context.scrollContext(new ScrollContext());
                 context.scrollContext().scroll = request.scroll();
             }
-            parseSource(context, request.source());
+            parseSource(reader, context, request.source(), includeAggregations);
 
             // if the from and size are still not set, default them
             if (context.from() == -1) {
@@ -678,8 +625,12 @@ final SearchContext createContext(SearchRewriteContext rewriteContext) {
             if (request.scroll() != null && request.scroll().keepAlive() != null) {
                 keepAlive = request.scroll().keepAlive().millis();
             }
-            contextScrollKeepAlive(context, keepAlive);
+            contextScrollKeepAlive(reader, keepAlive);
             context.lowLevelCancellation(lowLevelCancellation);
+            // Disable timeout while searching and mark the reader as accessed when releasing the context.
+            context.addReleasable(() -> reader.accessed(threadPool.relativeTimeInMillis()));
+            reader.accessed(-1L);
+            context.addReleasable(() -> reader.indexShard().getSearchOperationListener().onFreeSearchContext(context));
         } catch (Exception e) {
             context.close();
             throw e;
@@ -688,24 +639,29 @@ final SearchContext createContext(SearchRewriteContext rewriteContext) {
         return context;
     }
 
-    public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout) throws IOException {
-        IndexShard shard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id());
-        SearchRewriteContext rewriteContext = acquireSearcherAndRewrite(request, shard);
-        // make sure that we wrap the searcher when executing the query
-        return createSearchContext(rewriteContext.wrapSearcher(), timeout);
+    public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout) {
+        IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().getId());
+        Engine.Searcher engineSearcher = indexShard.acquireSearcher("search");
+        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, engineSearcher, request)) {
+            engineSearcher = null; // transfer ownership to readerContext
+            return createSearchContext(readerContext, request, timeout);
+        } finally {
+            Releasables.close(engineSearcher);
+        }
     }
 
-    private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteContext, TimeValue timeout) {
+    private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSearchRequest request, TimeValue timeout) {
         boolean success = false;
+        DefaultSearchContext searchContext = null;
+        reader.incRef();
         try {
-            final ShardSearchRequest request = rewriteContext.request;
-            final Engine.Searcher searcher = rewriteContext.searcher;
             IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
             IndexShard indexShard = indexService.getShard(request.shardId().getId());
             SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(),
                 indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE);
-            DefaultSearchContext searchContext = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget,
-                searcher, clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase);
+            searchContext = new DefaultSearchContext(reader, request, shardTarget,
+                clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase);
+            assert searchContext.getQueryShardContext().isCacheable();
             success = true;
             return searchContext;
         } finally {
@@ -713,79 +669,51 @@ private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteCon
                 // we handle the case where `IndicesService#indexServiceSafe`or `IndexService#getShard`, or the DefaultSearchContext
                 // constructor throws an exception since we would otherwise leak a searcher and this can have severe implications
                 // (unable to obtain shard lock exceptions).
-                IOUtils.closeWhileHandlingException(rewriteContext.searcher);
+                IOUtils.closeWhileHandlingException(searchContext);
+                if (searchContext == null) {
+                    IOUtils.closeWhileHandlingException(reader::decRef);
+                }
             }
         }
     }
 
     private void freeAllContextForIndex(Index index) {
         assert index != null;
-        for (SearchContext ctx : activeContexts.values()) {
+        for (ReaderContext ctx : activeReaders.values()) {
             if (index.equals(ctx.indexShard().shardId().getIndex())) {
-                freeContext(ctx.id());
+                freeReaderContext(ctx.id());
             }
         }
     }
 
 
-    public boolean freeContext(long id) {
-        try (SearchContext context = removeContext(id)) {
-            if (context != null) {
-                onFreeContext(context);
-                return true;
-            }
-            return false;
+    public boolean freeReaderContext(long id) {
+        try (ReaderContext context = removeReaderContext(id)) {
+            return context != null;
         }
     }
 
-    private void onFreeContext(SearchContext context) {
-        assert context.refCount() > 0 : " refCount must be > 0: " + context.refCount();
-        assert activeContexts.containsKey(context.id()) == false;
-        context.indexShard().getSearchOperationListener().onFreeContext(context);
-        if (context.scrollContext() != null) {
-            openScrollContexts.decrementAndGet();
-            context.indexShard().getSearchOperationListener().onFreeScrollContext(context);
-        }
-    }
 
     public void freeAllScrollContexts() {
-        for (SearchContext searchContext : activeContexts.values()) {
-            if (searchContext.scrollContext() != null) {
-                freeContext(searchContext.id());
+        for (ReaderContext readerContext : activeReaders.values()) {
+            if (readerContext.scrollContext() != null) {
+                freeReaderContext(readerContext.id());
             }
         }
     }
 
-    private void contextScrollKeepAlive(SearchContext context, long keepAlive) {
+    private void contextScrollKeepAlive(ReaderContext readerContext, long keepAlive) {
         if (keepAlive > maxKeepAlive) {
             throw new IllegalArgumentException(
                 "Keep alive for scroll (" + TimeValue.timeValueMillis(keepAlive) + ") is too large. " +
                     "It must be less than (" + TimeValue.timeValueMillis(maxKeepAlive) + "). " +
                     "This limit can be set by changing the [" + MAX_KEEPALIVE_SETTING.getKey() + "] cluster level setting.");
         }
-        context.keepAlive(keepAlive);
-    }
-
-    private void contextProcessing(SearchContext context) {
-        // disable timeout while executing a search
-        context.accessed(-1);
-    }
-
-    private void contextProcessedSuccessfully(SearchContext context) {
-        context.accessed(threadPool.relativeTimeInMillis());
-    }
-
-    private void cleanContext(SearchContext context) {
-        try {
-            context.clearReleasables(Lifetime.PHASE);
-            context.setTask(null);
-        } finally {
-            context.decRef();
-        }
+        readerContext.keepAlive(keepAlive);
     }
 
-    private void processFailure(SearchContext context, Exception e) {
-        freeContext(context.id());
+    private void processFailure(ReaderContext context, Exception e) {
+        freeReaderContext(context.id());
         try {
             if (Lucene.isCorruptionException(e)) {
                 context.indexShard().failShard("search execution corruption failure", e);
@@ -796,7 +724,8 @@ private void processFailure(SearchContext context, Exception e) {
         }
     }
 
-    private void parseSource(DefaultSearchContext context, SearchSourceBuilder source) throws SearchException {
+    private void parseSource(ReaderContext reader, DefaultSearchContext context,
+                             SearchSourceBuilder source, boolean includeAggregations) throws SearchException {
         // nothing to parse...
         if (source == null) {
             return;
@@ -852,7 +781,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
             context.timeout(source.timeout());
         }
         context.terminateAfter(source.terminateAfter());
-        if (source.aggregations() != null) {
+        if (source.aggregations() != null && includeAggregations) {
             try {
                 AggregatorFactories factories = source.aggregations().build(queryShardContext, null);
                 context.aggregations(new SearchContextAggregations(factories, multiBucketConsumerService.create()));
@@ -867,10 +796,10 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
                 throw new SearchException(shardTarget, "failed to create SuggestionSearchContext", e);
             }
         }
-        if (source.rescores() != null) {
+        if (source.rescores() != null && reader.rescore().isEmpty()) {
             try {
                 for (RescorerBuilder<?> rescore : source.rescores()) {
-                    context.addRescore(rescore.buildContext(queryShardContext));
+                    reader.addRescore(rescore.buildContext(queryShardContext));
                 }
             } catch (IOException e) {
                 throw new SearchException(shardTarget, "failed to create RescoreSearchContext", e);
@@ -1028,13 +957,13 @@ private void shortcutDocIdsToLoad(SearchContext context) {
         context.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length);
     }
 
-    private void processScroll(InternalScrollSearchRequest request, SearchContext context) {
+    private void processScroll(InternalScrollSearchRequest request, ReaderContext reader, SearchContext context) {
         // process scroll
         context.from(context.from() + context.size());
         context.scrollContext().scroll = request.scroll();
         // update the context keep alive based on the new scroll value
         if (request.scroll() != null && request.scroll().keepAlive() != null) {
-            contextScrollKeepAlive(context, request.scroll().keepAlive().millis());
+            contextScrollKeepAlive(reader, request.scroll().keepAlive().millis());
         }
     }
 
@@ -1043,7 +972,7 @@ private void processScroll(InternalScrollSearchRequest request, SearchContext co
      * SearchService
      */
     public int getActiveContexts() {
-        return this.activeContexts.size();
+        return this.activeReaders.size();
     }
 
     public ResponseCollectorService getResponseCollectorService() {
@@ -1054,7 +983,7 @@ class Reaper implements Runnable {
         @Override
         public void run() {
             final long time = threadPool.relativeTimeInMillis();
-            for (SearchContext context : activeContexts.values()) {
+            for (ReaderContext context : activeReaders.values()) {
                 // Use the same value for both checks since lastAccessTime can
                 // be modified by another thread between checks!
                 final long lastAccessTime = context.lastAccessTime();
@@ -1064,7 +993,7 @@ public void run() {
                 if ((time - lastAccessTime > context.keepAlive())) {
                     logger.debug("freeing search context [{}], time [{}], lastAccessTime [{}], keepAlive [{}]", context.id(), time,
                         lastAccessTime, context.keepAlive());
-                    freeContext(context.id());
+                    freeReaderContext(context.id());
                 }
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
index c150f99cfe196..fca5b14e34be3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
@@ -117,8 +117,6 @@ public void execute(SearchContext context) {
                 context.searcher().search(query, collector);
             } catch (Exception e) {
                 throw new QueryPhaseExecutionException(context.shardTarget(), "Failed to execute global aggregators", e);
-            } finally {
-                context.clearReleasables(SearchContext.Lifetime.COLLECTION);
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
index ea89700521798..43e38c731bbe1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContext.Lifetime;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
 
 import java.io.IOException;
@@ -76,7 +75,7 @@ protected AggregatorBase(String name, AggregatorFactories factories, SearchConte
         this.breakerService = context.bigArrays().breakerService();
         assert factories != null : "sub-factories provided to BucketAggregator must not be null, use AggragatorFactories.EMPTY instead";
         this.subAggregators = factories.createSubAggregators(context, this);
-        context.addReleasable(this, Lifetime.PHASE);
+        context.addReleasable(this);
         final SearchShardTarget shardTarget = context.shardTarget();
         // Register a safeguard to highlight any invalid construction logic (call to this constructor without subsequent preCollection call)
         collectableSubAggregators = new BucketCollector() {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
index 970ef725f027d..5451285a379c8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContext.Lifetime;
 
 import java.io.IOException;
 import java.util.List;
@@ -50,7 +49,7 @@ public static final class MultiBucketAggregatorWrapper extends Aggregator {
             this.parent = parent;
             this.factory = factory;
             this.first = first;
-            context.addReleasable(this, Lifetime.PHASE);
+            context.addReleasable(this);
             aggregators = bigArrays.newObjectArray(1);
             aggregators.set(0, first);
             collectors = bigArrays.newObjectArray(1);
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java
index b4d77952cb3a0..4b3b8870ef935 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java
@@ -48,8 +48,6 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
         } catch (IOException e) {
             throw new FetchPhaseExecutionException(context.shardTarget(),
                 "Failed to explain doc [" + hitContext.hit().getId() + "]", e);
-        } finally {
-            context.clearReleasables(SearchContext.Lifetime.COLLECTION);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java
index 9302f6d9d637d..dce3ec49f9f10 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContext.Lifetime;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -92,8 +91,6 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) {
             }
         } catch (IOException e) {
             throw ExceptionsHelper.convertToElastic(e);
-        } finally {
-            context.clearReleasables(Lifetime.COLLECTION);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index 8c04954a4efcf..ab605062de50a 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -140,21 +140,11 @@ public float queryBoost() {
         return in.queryBoost();
     }
 
-    @Override
-    public long getOriginNanoTime() {
-        return in.getOriginNanoTime();
-    }
-
     @Override
     public ScrollContext scrollContext() {
         return in.scrollContext();
     }
 
-    @Override
-    public SearchContext scrollContext(ScrollContext scroll) {
-        return in.scrollContext(scroll);
-    }
-
     @Override
     public SearchContextAggregations aggregations() {
         return in.aggregations();
@@ -195,11 +185,6 @@ public List<RescoreContext> rescore() {
         return in.rescore();
     }
 
-    @Override
-    public void addRescore(RescoreContext rescore) {
-        in.addRescore(rescore);
-    }
-
     @Override
     public boolean hasScriptFields() {
         return in.hasScriptFields();
@@ -451,26 +436,6 @@ public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int
         return in.docIdsToLoad(docIdsToLoad, docsIdsToLoadFrom, docsIdsToLoadSize);
     }
 
-    @Override
-    public void accessed(long accessTime) {
-        in.accessed(accessTime);
-    }
-
-    @Override
-    public long lastAccessTime() {
-        return in.lastAccessTime();
-    }
-
-    @Override
-    public long keepAlive() {
-        return in.keepAlive();
-    }
-
-    @Override
-    public void keepAlive(long keepAlive) {
-        in.keepAlive(keepAlive);
-    }
-
     @Override
     public SearchLookup lookup() {
         return in.lookup();
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
new file mode 100644
index 0000000000000..7c9dfe14dc9f8
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.search.rescore.RescoreContext;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Holds a reference to a point in time {@link Engine.Searcher} that will be used to construct {@link SearchContext}.
+ * This class also implements {@link org.elasticsearch.common.util.concurrent.RefCounted} since in some situations like
+ * in {@link org.elasticsearch.search.SearchService} a SearchContext can be closed concurrently due to independent events
+ * ie. when an index gets removed. To prevent accessing closed IndexReader / IndexSearcher instances the SearchContext
+ * can be guarded by a reference count and fail if it's been closed by an external event.
+ *
+ * For reference why we use RefCounted here see https://github.com/elastic/elasticsearch/pull/20095.
+ */
+public class ReaderContext extends AbstractRefCounted implements Releasable {
+    private final long id;
+    private final IndexShard indexShard;
+    private final Engine.Searcher engineSearcher;
+    private final AtomicBoolean closed = new AtomicBoolean(false);
+
+    private volatile long keepAlive;
+    private volatile long lastAccessTime = -1L;
+
+    // BWC
+    private final ShardSearchRequest request;
+    private final ScrollContext scrollContext;
+    private AggregatedDfs aggregatedDfs;
+    private List<RescoreContext> rescore;
+
+    private final List<Releasable> onCloses = new CopyOnWriteArrayList<>();
+
+    public ReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher, ShardSearchRequest request) {
+        super("reader_context");
+        this.id = id;
+        this.indexShard = indexShard;
+        this.engineSearcher = engineSearcher;
+        this.request = request;
+        if (request.scroll() != null) {
+            this.scrollContext = new ScrollContext();
+        } else {
+            this.scrollContext = null;
+        }
+    }
+
+    @Override
+    public final void close() {
+        if (closed.compareAndSet(false, true)) {
+            decRef();
+        } else {
+            assert false : "ReaderContext was closed already";
+        }
+    }
+
+    @Override
+    protected void closeInternal() {
+        Releasables.close(Releasables.wrap(onCloses), engineSearcher);
+    }
+
+    public void addOnClose(Releasable releasable) {
+        onCloses.add(releasable);
+    }
+
+    public long id() {
+        return id;
+    }
+
+    public IndexShard indexShard() {
+        return indexShard;
+    }
+
+    public Engine.Searcher engineSearcher() {
+        return engineSearcher;
+    }
+
+    public String source() {
+        return engineSearcher.source();
+    }
+
+    public ShardSearchRequest request() {
+        return request;
+    }
+
+    public ScrollContext scrollContext() {
+        return scrollContext;
+    }
+
+    public AggregatedDfs aggregatedDfs() {
+        return aggregatedDfs;
+    }
+
+    public void aggregatedDfs(AggregatedDfs aggregatedDfs) {
+        this.aggregatedDfs = aggregatedDfs;
+    }
+
+    public void accessed(long accessTime) {
+        this.lastAccessTime = accessTime;
+    }
+
+    public long lastAccessTime() {
+        return this.lastAccessTime;
+    }
+
+    public long keepAlive() {
+        return this.keepAlive;
+    }
+
+    public void keepAlive(long keepAlive) {
+        this.keepAlive = keepAlive;
+    }
+
+    public List<RescoreContext> rescore() {
+        if (rescore == null) {
+            return Collections.emptyList();
+        } else {
+            return Collections.unmodifiableList(rescore);
+        }
+    }
+
+    public void addRescore(RescoreContext rescore) {
+        if (this.rescore == null) {
+            this.rescore = new ArrayList<>();
+        }
+        this.rescore.add(rescore);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java b/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
index 41d7680a780b0..440408646d127 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
@@ -36,6 +36,8 @@ public final class ScrollContext {
     public ScoreDoc lastEmittedDoc;
     public Scroll scroll;
 
+    private final long startTimeInNano = System.nanoTime();
+
     /**
      * Returns the object or <code>null</code> if the given key does not have a
      * value in the context
@@ -54,4 +56,9 @@ public void putInContext(String key, Object value) {
         }
         context.put(key, value);
     }
+
+
+    public long getStartTimeInNano() {
+        return startTimeInNano;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 4473546f76d15..0a1fc3822ad42 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -29,9 +29,6 @@
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
-import org.elasticsearch.common.util.concurrent.RefCounted;
-import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -61,36 +58,29 @@
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
-import java.util.ArrayList;
-import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * This class encapsulates the state needed to execute a search. It holds a reference to the
  * shards point in time snapshot (IndexReader / ContextIndexSearcher) and allows passing on
  * state from one query / fetch phase to another.
- *
- * This class also implements {@link RefCounted} since in some situations like in {@link org.elasticsearch.search.SearchService}
- * a SearchContext can be closed concurrently due to independent events ie. when an index gets removed. To prevent accessing closed
- * IndexReader / IndexSearcher instances the SearchContext can be guarded by a reference count and fail if it's been closed by
- * an external event.
  */
-// For reference why we use RefCounted here see #20095
-public abstract class SearchContext extends AbstractRefCounted implements Releasable {
+public abstract class SearchContext implements Releasable {
 
     public static final int DEFAULT_TERMINATE_AFTER = 0;
     public static final int TRACK_TOTAL_HITS_ACCURATE = Integer.MAX_VALUE;
     public static final int TRACK_TOTAL_HITS_DISABLED = -1;
     public static final int DEFAULT_TRACK_TOTAL_HITS_UP_TO = 10000;
 
-    private Map<Lifetime, List<Releasable>> clearables = null;
+    private final List<Releasable> releasables = new CopyOnWriteArrayList<>();
     private final AtomicBoolean closed = new AtomicBoolean(false);
     private InnerHitsContext innerHitsContext;
 
     protected SearchContext() {
-        super("search_context");
+
     }
 
     public abstract void setTask(SearchShardTask task);
@@ -101,25 +91,15 @@ protected SearchContext() {
 
     @Override
     public final void close() {
-        if (closed.compareAndSet(false, true)) { // prevent double closing
-            decRef();
-        }
-    }
-
-    @Override
-    protected final void closeInternal() {
-        try {
-            clearReleasables(Lifetime.CONTEXT);
-        } finally {
-            doClose();
+        if (closed.compareAndSet(false, true)) {
+            try {
+                Releasables.close(releasables);
+            } finally {
+                doClose();
+            }
         }
     }
 
-    @Override
-    protected void alreadyClosed() {
-        throw new IllegalStateException("search context is already closed can't increment refCount current count [" + refCount() + "]");
-    }
-
     protected abstract void doClose();
 
     /**
@@ -146,12 +126,8 @@ protected void alreadyClosed() {
 
     public abstract float queryBoost();
 
-    public abstract long getOriginNanoTime();
-
     public abstract ScrollContext scrollContext();
 
-    public abstract SearchContext scrollContext(ScrollContext scroll);
-
     public abstract SearchContextAggregations aggregations();
 
     public abstract SearchContext aggregations(SearchContextAggregations aggregations);
@@ -180,8 +156,6 @@ public InnerHitsContext innerHits() {
      */
     public abstract List<RescoreContext> rescore();
 
-    public abstract void addRescore(RescoreContext rescore);
-
     public abstract boolean hasScriptFields();
 
     public abstract ScriptFieldsContext scriptFields();
@@ -322,14 +296,6 @@ public InnerHitsContext innerHits() {
 
     public abstract SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int docsIdsToLoadSize);
 
-    public abstract void accessed(long accessTime);
-
-    public abstract long lastAccessTime();
-
-    public abstract long keepAlive();
-
-    public abstract void keepAlive(long keepAlive);
-
     public SearchLookup lookup() {
         return getQueryShardContext().lookup();
     }
@@ -347,38 +313,14 @@ public SearchLookup lookup() {
      */
     public abstract Profilers getProfilers();
 
+
     /**
-     * Schedule the release of a resource. The time when {@link Releasable#close()} will be called on this object
-     * is function of the provided {@link Lifetime}.
+     * Adds a releasable that will be freed when this context is closed.
      */
-    public void addReleasable(Releasable releasable, Lifetime lifetime) {
-        if (clearables == null) {
-            clearables = new EnumMap<>(Lifetime.class);
-        }
-        List<Releasable> releasables = clearables.get(lifetime);
-        if (releasables == null) {
-            releasables = new ArrayList<>();
-            clearables.put(lifetime, releasables);
-        }
+    public void addReleasable(Releasable releasable) {
         releasables.add(releasable);
     }
 
-    public void clearReleasables(Lifetime lifetime) {
-        if (clearables != null) {
-            List<List<Releasable>>releasables = new ArrayList<>();
-            for (Lifetime lc : Lifetime.values()) {
-                if (lc.compareTo(lifetime) > 0) {
-                    break;
-                }
-                List<Releasable> remove = clearables.remove(lc);
-                if (remove != null) {
-                    releasables.add(remove);
-                }
-            }
-            Releasables.close(Iterables.flatten(releasables));
-        }
-    }
-
     /**
      * @return true if the request contains only suggest
      */
@@ -403,24 +345,6 @@ public final boolean hasOnlySuggest() {
     /** Return a view of the additional query collectors that should be run for this context. */
     public abstract Map<Class<?>, Collector> queryCollectors();
 
-    /**
-     * The life time of an object that is used during search execution.
-     */
-    public enum Lifetime {
-        /**
-         * This life time is for objects that only live during collection time.
-         */
-        COLLECTION,
-        /**
-         * This life time is for objects that need to live until the end of the current search phase.
-         */
-        PHASE,
-        /**
-         * This life time is for objects that need to live until the search context they are attached to is destroyed.
-         */
-        CONTEXT
-    }
-
     public abstract QueryShardContext getQueryShardContext();
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
index a539a77d66be8..9709ec1c1d302 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
 import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
@@ -85,11 +84,6 @@ public Query buildFilteredQuery(Query query) {
         throw new UnsupportedOperationException("this context should be read only");
     }
 
-    @Override
-    public SearchContext scrollContext(ScrollContext scrollContext) {
-        throw new UnsupportedOperationException("Not supported");
-    }
-
     @Override
     public SearchContext aggregations(SearchContextAggregations aggregations) {
         throw new UnsupportedOperationException("Not supported");
@@ -110,11 +104,6 @@ public void suggest(SuggestionSearchContext suggest) {
         throw new UnsupportedOperationException("Not supported");
     }
 
-    @Override
-    public void addRescore(RescoreContext rescore) {
-        throw new UnsupportedOperationException("Not supported");
-    }
-
     @Override
     public boolean hasScriptFields() {
         return scriptFields != null;
@@ -332,16 +321,6 @@ public CollapseContext collapse() {
         return null;
     }
 
-    @Override
-    public void accessed(long accessTime) {
-        throw new UnsupportedOperationException("Not supported");
-    }
-
-    @Override
-    public void keepAlive(long keepAlive) {
-        throw new UnsupportedOperationException("Not supported");
-    }
-
     @Override
     public QuerySearchResult queryResult() {
         return querySearchResult;
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 64e274e734d0c..d3952d2bf83c4 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -343,8 +343,6 @@ private static boolean searchWithCollector(SearchContext searchContext, ContextI
                 throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded");
             }
             queryResult.searchTimedOut(true);
-        } finally {
-            searchContext.clearReleasables(SearchContext.Lifetime.COLLECTION);
         }
         if (searchContext.terminateAfter() != SearchContext.DEFAULT_TERMINATE_AFTER && queryResult.terminatedEarly() == null) {
             queryResult.terminatedEarly(false);
@@ -399,8 +397,6 @@ private static boolean searchWithCollectorManager(SearchContext searchContext, C
                 throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded");
             }
             searchContext.queryResult().searchTimedOut(true);
-        } finally {
-            searchContext.clearReleasables(SearchContext.Lifetime.COLLECTION);
         }
         return false; // no rescoring when sorting by field
     }
diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 5c507ddec67d7..5a175f66e7c9e 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -79,11 +79,10 @@
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.test.engine.MockEngineFactory;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -104,6 +103,7 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.Mockito.mock;
 
 public class IndexModuleTests extends ESTestCase {
     private Index index;
@@ -275,9 +275,8 @@ public void testAddSearchOperationListener() throws IOException {
         IndexModule module = createIndexModule(indexSettings, emptyAnalysisRegistry);
         AtomicBoolean executed = new AtomicBoolean(false);
         SearchOperationListener listener = new SearchOperationListener() {
-
             @Override
-            public void onNewContext(SearchContext context) {
+            public void onNewReaderContext(ReaderContext readerContext) {
                 executed.set(true);
             }
         };
@@ -290,9 +289,8 @@ public void onNewContext(SearchContext context) {
         assertEquals(2, indexService.getSearchOperationListener().size());
         assertEquals(SearchSlowLog.class, indexService.getSearchOperationListener().get(0).getClass());
         assertSame(listener, indexService.getSearchOperationListener().get(1));
-
         for (SearchOperationListener l : indexService.getSearchOperationListener()) {
-            l.onNewContext(new TestSearchContext(null));
+            l.onNewReaderContext(mock(ReaderContext.class));
         }
         assertTrue(executed.get());
         indexService.close("simon says", false);
diff --git a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
index 28bab8da0fdfb..34c528dfeda6a 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.index.shard;
 
+import org.elasticsearch.search.internal.ReaderContext;
+import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
@@ -33,6 +35,7 @@
 
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.mock;
 
 public class SearchOperationListenerTests extends ESTestCase {
 
@@ -90,26 +93,26 @@ public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
             }
 
             @Override
-            public void onNewContext(SearchContext context) {
-                assertNotNull(context);
+            public void onNewReaderContext(ReaderContext readerContext) {
+                assertNotNull(readerContext);
                 newContext.incrementAndGet();
             }
 
             @Override
-            public void onFreeContext(SearchContext context) {
-                assertNotNull(context);
+            public void onFreeReaderContext(ReaderContext readerContext) {
+                assertNotNull(readerContext);
                 freeContext.incrementAndGet();
             }
 
             @Override
-            public void onNewScrollContext(SearchContext context) {
-                assertNotNull(context);
+            public void onNewScrollContext(ScrollContext scrollContext) {
+                assertNotNull(scrollContext);
                 newScrollContext.incrementAndGet();
             }
 
             @Override
-            public void onFreeScrollContext(SearchContext context) {
-                assertNotNull(context);
+            public void onFreeScrollContext(ScrollContext scrollContext) {
+                assertNotNull(scrollContext);
                 freeScrollContext.incrementAndGet();
             }
 
@@ -216,7 +219,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onNewContext(ctx);
+        compositeListener.onNewReaderContext(mock(ReaderContext.class));
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -229,7 +232,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onNewScrollContext(ctx);
+        compositeListener.onNewScrollContext(new ScrollContext());
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -242,7 +245,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onFreeContext(ctx);
+        compositeListener.onFreeReaderContext(mock(ReaderContext.class));
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -255,7 +258,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onFreeScrollContext(ctx);
+        compositeListener.onFreeScrollContext(new ScrollContext());
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 560f052664604..198daf0f41a11 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -50,7 +50,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.ScrollContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.slice.SliceBuilder;
@@ -111,26 +111,33 @@ public void testPreProcess() throws Exception {
 
         try (Directory dir = newDirectory();
              RandomIndexWriter w = new RandomIndexWriter(random(), dir);
-             IndexReader reader = w.getReader();
-             Engine.Searcher searcher = new Engine.Searcher("test", reader,
-                 IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
-                 IndexSearcher.getDefaultQueryCachingPolicy(), reader)) {
+             IndexReader reader = w.getReader()) {
+
+            final Engine.Searcher engineSearcher = new Engine.Searcher("test", w.getReader(),
+                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
+                IndexSearcher.getDefaultQueryCachingPolicy(), reader);
 
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
-            DefaultSearchContext context1 = new DefaultSearchContext(1L, shardSearchRequest, target, searcher, null, indexService,
-                indexShard, bigArrays, null, timeout, null);
-            context1.from(300);
+            ReaderContext readerWithoutScroll = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            DefaultSearchContext contextWithoutScroll = new DefaultSearchContext(readerWithoutScroll, shardSearchRequest, target, null,
+                indexService, indexShard, bigArrays, null, timeout, null);
+            contextWithoutScroll.from(300);
+            contextWithoutScroll.close();
 
             // resultWindow greater than maxResultWindow and scrollContext is null
-            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
+            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> contextWithoutScroll.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Result window is too large, from + size must be less than or equal to:"
                 + " [" + maxResultWindow + "] but was [310]. See the scroll api for a more efficient way to request large data sets. "
                 + "This limit can be set by changing the [" + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey()
                 + "] index level setting."));
 
             // resultWindow greater than maxResultWindow and scrollContext isn't null
-            context1.scrollContext(new ScrollContext());
+            when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMillis(randomInt(1000))));
+            ReaderContext readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            DefaultSearchContext context1 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null, indexService,
+                indexShard, bigArrays, null, timeout, null);
+            context1.from(300);
             exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Batch size is too large, size must be less than or equal to: ["
                 + maxResultWindow + "] but was [310]. Scroll batch sizes cost as much memory as result windows so they are "
@@ -144,7 +151,7 @@ public void testPreProcess() throws Exception {
 
             RescoreContext rescoreContext = mock(RescoreContext.class);
             when(rescoreContext.getWindowSize()).thenReturn(500);
-            context1.addRescore(rescoreContext);
+            readerContext.addRescore(rescoreContext);
 
             exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Cannot use [sort] option in conjunction with [rescore]."));
@@ -158,8 +165,10 @@ public void testPreProcess() throws Exception {
                 + "to be rescored. This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey()
                 + "] index level setting."));
 
+            readerContext.close();
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
             // rescore is null but sliceBuilder is not null
-            DefaultSearchContext context2 = new DefaultSearchContext(2L, shardSearchRequest, target, searcher,
+            DefaultSearchContext context2 = new DefaultSearchContext(readerContext, shardSearchRequest, target,
                 null, indexService, indexShard, bigArrays, null, timeout, null);
 
             SliceBuilder sliceBuilder = mock(SliceBuilder.class);
@@ -176,7 +185,7 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.getAliasFilter()).thenReturn(AliasFilter.EMPTY);
             when(shardSearchRequest.indexBoost()).thenReturn(AbstractQueryBuilder.DEFAULT_BOOST);
 
-            DefaultSearchContext context3 = new DefaultSearchContext(3L, shardSearchRequest, target, searcher, null,
+            DefaultSearchContext context3 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
                 indexService, indexShard, bigArrays, null, timeout, null);
             ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery();
             context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false);
@@ -186,7 +195,9 @@ public void testPreProcess() throws Exception {
             when(queryShardContext.fieldMapper(anyString())).thenReturn(mock(MappedFieldType.class));
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
-            DefaultSearchContext context4 = new DefaultSearchContext(4L, shardSearchRequest, target, searcher, null,
+            readerContext.close();
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            DefaultSearchContext context4 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
                 indexService, indexShard, bigArrays, null, timeout, null);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
             Query query1 = context4.query();
@@ -194,6 +205,7 @@ public void testPreProcess() throws Exception {
             Query query2 = context4.query();
             assertTrue(query1 instanceof MatchNoDocsQuery || query2 instanceof MatchNoDocsQuery);
 
+            readerContext.close();
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 2e61d63c81117..bce588e69adb0 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -47,6 +47,7 @@
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
@@ -76,6 +77,7 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
@@ -169,13 +171,11 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
         public void onIndexModule(IndexModule indexModule) {
             indexModule.addSearchOperationListener(new SearchOperationListener() {
                 @Override
-                public void onNewContext(SearchContext context) {
-                    if (context.query() != null) {
-                        if ("throttled_threadpool_index".equals(context.indexShard().shardId().getIndex().getName())) {
-                            assertThat(Thread.currentThread().getName(), startsWith("elasticsearch[node_s_0][search_throttled]"));
-                        } else {
-                            assertThat(Thread.currentThread().getName(), startsWith("elasticsearch[node_s_0][search]"));
-                        }
+                public void onNewReaderContext(ReaderContext readerContext) {
+                    if ("throttled_threadpool_index".equals(readerContext.indexShard().shardId().getIndex().getName())) {
+                        assertThat(Thread.currentThread().getName(), startsWith("elasticsearch[node_s_0][search_throttled]"));
+                    } else {
+                        assertThat(Thread.currentThread().getName(), startsWith("elasticsearch[node_s_0][search]"));
                     }
                 }
 
@@ -326,7 +326,7 @@ public void onFailure(Exception e) {
                         service.executeFetchPhase(req, new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()), listener);
                         listener.get();
                         if (useScroll) {
-                            service.freeContext(searchPhaseResult.getRequestId());
+                            service.freeReaderContext(searchPhaseResult.getRequestId());
                         }
                     } catch (ExecutionException ex) {
                         assertThat(ex.getCause(), instanceOf(RuntimeException.class));
@@ -335,7 +335,7 @@ public void onFailure(Exception e) {
                 } catch (AlreadyClosedException ex) {
                     throw ex;
                 } catch (IllegalStateException ex) {
-                    assertEquals("search context is already closed can't increment refCount current count [0]", ex.getMessage());
+                    assertEquals("reader_context is already closed can't increment refCount current count [0]", ex.getMessage());
                 } catch (SearchContextMissingException ex) {
                     // that's fine
                 }
@@ -361,34 +361,34 @@ public void testTimeout() throws IOException {
         final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
         final IndexShard indexShard = indexService.getShard(0);
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
-        SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(
-            new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
-                new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null),
-            indexShard);
-        final SearchContext contextWithDefaultTimeout = service.createContext(rewriteContext);
-        try {
+        final ShardSearchRequest requestWithDefaultTimeout = new ShardSearchRequest(
+            OriginalIndices.NONE,
+            searchRequest,
+            indexShard.shardId(),
+            1,
+            new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1.0f, -1, null, null);
+
+        try (ReaderContext reader = createReaderContext(indexShard, requestWithDefaultTimeout);
+             SearchContext contextWithDefaultTimeout = service.createContext(reader, requestWithDefaultTimeout, null, randomBoolean())) {
             // the search context should inherit the default timeout
             assertThat(contextWithDefaultTimeout.timeout(), equalTo(TimeValue.timeValueSeconds(5)));
-        } finally {
-            contextWithDefaultTimeout.decRef();
-            service.freeContext(contextWithDefaultTimeout.id());
         }
 
         final long seconds = randomIntBetween(6, 10);
         searchRequest.source(new SearchSourceBuilder().timeout(TimeValue.timeValueSeconds(seconds)));
-        rewriteContext = service.acquireSearcherAndRewrite(
-            new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
-                new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null),
-            indexShard);
-        final SearchContext context = service.createContext(rewriteContext);
-        try {
+        final ShardSearchRequest requestWithCustomTimeout = new ShardSearchRequest(
+            OriginalIndices.NONE,
+            searchRequest,
+            indexShard.shardId(),
+            1,
+            new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1.0f, -1, null, null);
+        try (ReaderContext reader = createReaderContext(indexShard, requestWithCustomTimeout);
+             SearchContext context = service.createContext(reader, requestWithCustomTimeout, null, randomBoolean())) {
             // the search context should inherit the query timeout
             assertThat(context.timeout(), equalTo(TimeValue.timeValueSeconds(seconds)));
-        } finally {
-            context.decRef();
-            service.freeContext(context.id());
         }
-
     }
 
     /**
@@ -408,25 +408,20 @@ public void testMaxDocvalueFieldsSearch() throws IOException {
         for (int i = 0; i < indexService.getIndexSettings().getMaxDocvalueFields(); i++) {
             searchSourceBuilder.docValueField("field" + i);
         }
-
-        ShardSearchRequest shardRequest =  new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
+        final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
             new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
-
-        {
-            SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(shardRequest, indexShard);
-            try (SearchContext context = service.createContext(rewriteContext)) {
-                assertNotNull(context);
-            }
+        try (ReaderContext reader = createReaderContext(indexShard, request);
+             SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
+            assertNotNull(context);
         }
-
-        {
-            SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(shardRequest, indexShard);
-            searchSourceBuilder.docValueField("one_field_too_much");
+        searchSourceBuilder.docValueField("one_field_too_much");
+        try (ReaderContext reader = createReaderContext(indexShard, request)) {
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> service.createContext(rewriteContext));
+                () -> service.createContext(reader, request, null, randomBoolean()));
             assertEquals(
                 "Trying to retrieve too many docvalue_fields. Must be less than or equal to: [100] but was [101]. "
-                    + "This limit can be set by changing the [index.max_docvalue_fields_search] index level setting.", ex.getMessage());
+                    + "This limit can be set by changing the [index.max_docvalue_fields_search] index level setting.",
+                ex.getMessage());
         }
     }
 
@@ -449,23 +444,17 @@ public void testMaxScriptFieldsSearch() throws IOException {
             searchSourceBuilder.scriptField("field" + i,
                     new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
         }
+        final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
+            indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
 
-        ShardSearchRequest shardRequest = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
-            new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
-
-        {
-            SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(shardRequest, indexShard);
-            try (SearchContext context = service.createContext(rewriteContext)) {
+        try(ReaderContext reader = createReaderContext(indexShard, request)) {
+            try (SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
                 assertNotNull(context);
             }
-        }
-
-        {
             searchSourceBuilder.scriptField("anotherScriptField",
                 new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
-            SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(shardRequest, indexShard);
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                    () -> service.createContext(rewriteContext));
+                () -> service.createContext(reader, request, null, randomBoolean()));
             assertEquals(
                 "Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxScriptFields + "] but was ["
                     + (maxScriptFields + 1)
@@ -487,11 +476,11 @@ public void testIgnoreScriptfieldIfSizeZero() throws IOException {
         searchSourceBuilder.scriptField("field" + 0,
                 new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
         searchSourceBuilder.size(0);
-        SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(
-            new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
-                new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null),
-            indexShard);
-        try (SearchContext context = service.createContext(rewriteContext)) {
+        final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE,
+            searchRequest, indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1.0f, -1, null, null);
+        try (ReaderContext reader = createReaderContext(indexShard, request);
+             SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
             assertEquals(0, context.scriptFields().fields().size());
         }
     }
@@ -499,7 +488,7 @@ public void testIgnoreScriptfieldIfSizeZero() throws IOException {
     /**
      * test that creating more than the allowed number of scroll contexts throws an exception
      */
-    public void testMaxOpenScrollContexts() throws RuntimeException, IOException {
+    public void testMaxOpenScrollContexts() throws Exception {
         createIndex("index");
         client().prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
 
@@ -528,7 +517,7 @@ public void testMaxOpenScrollContexts() throws RuntimeException, IOException {
         SearchService.SearchRewriteContext rewriteContext =
             service.acquireSearcherAndRewrite(new ShardScrollRequestTest(indexShard.shardId()), indexShard);
         ElasticsearchException ex = expectThrows(ElasticsearchException.class,
-            () -> service.createAndPutContext(rewriteContext));
+            () -> service.createAndPutReaderContext(rewriteContext));
         assertEquals(
             "Trying to create too many scroll contexts. Must be less than or equal to: [" +
                 SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY) + "]. " +
@@ -606,7 +595,7 @@ public Scroll scroll() {
         }
     }
 
-    public void testCanMatch() throws IOException, InterruptedException {
+    public void testCanMatch() throws Exception {
         createIndex("index");
         final SearchService service = getInstanceFromNode(SearchService.class);
         final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
@@ -649,7 +638,7 @@ public void testCanMatch() throws IOException, InterruptedException {
             public void onResponse(SearchPhaseResult searchPhaseResult) {
                 try {
                     // make sure that the wrapper is called when the query is actually executed
-                    assertEquals(numWrapReader+1, numWrapInvocations.get());
+                    assertEquals(numWrapReader + 1, numWrapInvocations.get());
                 } finally {
                     latch.countDown();
                 }
@@ -779,19 +768,18 @@ public void testCreateSearchContextFailure() throws IOException {
         final IndexService indexService = createIndex(index);
         final SearchService service = getInstanceFromNode(SearchService.class);
         final ShardId shardId = new ShardId(indexService.index(), 0);
-        IndexShard indexShard = indexService.getShard(0);
-
-        SearchService.SearchRewriteContext rewriteContext = service.acquireSearcherAndRewrite(
-            new ShardSearchRequest(shardId, 0, AliasFilter.EMPTY) {
-                @Override
-                public SearchType searchType() {
-                    // induce an artificial NPE
-                    throw new NullPointerException("expected");
-                }
-            }, indexShard);
-        NullPointerException e = expectThrows(NullPointerException.class,
-            () -> service.createContext(rewriteContext));
-        assertEquals("expected", e.getMessage());
+        final ShardSearchRequest request = new ShardSearchRequest(shardId, 0, null) {
+            @Override
+            public SearchType searchType() {
+                // induce an artificial NPE
+                throw new NullPointerException("expected");
+            }
+        };
+        try (ReaderContext reader = createReaderContext(indexService.getShard(shardId.id()), request)) {
+            NullPointerException e = expectThrows(NullPointerException.class,
+                () -> service.createContext(reader, request, null, randomBoolean()));
+            assertEquals("expected", e.getMessage());
+        }
         assertEquals("should have 2 store refs (IndexService + InternalEngine)", 2, indexService.getShard(0).store().refCount());
     }
 
@@ -929,4 +917,9 @@ public void testDeleteIndexWhileSearch() throws Exception {
             searcher.join();
         }
     }
+
+    private ReaderContext createReaderContext(IndexShard shard, ShardSearchRequest request) {
+        Engine.Searcher searcher = shard.acquireSearcher("test");
+        return new ReaderContext(randomNonNegativeLong(), shard, searcher, request);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
index cc75e8bef5f48..95e3c08aa59a9 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
@@ -310,13 +310,12 @@ public void testInOrderScrollOptimization() throws Exception {
         }
         w.close();
         IndexReader reader = DirectoryReader.open(dir);
-        TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader));
-        context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
         ScrollContext scrollContext = new ScrollContext();
+        TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader), scrollContext);
+        context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
         scrollContext.lastEmittedDoc = null;
         scrollContext.maxScore = Float.NaN;
         scrollContext.totalHits = null;
-        context.scrollContext(scrollContext);
         context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()));
         int size = randomIntBetween(2, 5);
         context.setSize(size);
@@ -539,13 +538,12 @@ public void testIndexSortScrollOptimization() throws Exception {
         // search sort is a prefix of the index sort
         searchSortAndFormats.add(new SortAndFormats(new Sort(indexSort.getSort()[0]), new DocValueFormat[]{DocValueFormat.RAW}));
         for (SortAndFormats searchSortAndFormat : searchSortAndFormats) {
-            TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader));
-            context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
             ScrollContext scrollContext = new ScrollContext();
+            TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader), scrollContext);
+            context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
             scrollContext.lastEmittedDoc = null;
             scrollContext.maxScore = Float.NaN;
             scrollContext.totalHits = null;
-            context.scrollContext(scrollContext);
             context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()));
             context.setSize(10);
             context.sort(searchSortAndFormat);
diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
index 9653f3b66a515..1a5c3d7815440 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.fetch.FetchPhase;
-import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.HashMap;
@@ -40,11 +40,11 @@ public class MockSearchService extends SearchService {
      */
     public static class TestPlugin extends Plugin {}
 
-    private static final Map<SearchContext, Throwable> ACTIVE_SEARCH_CONTEXTS = new ConcurrentHashMap<>();
+    private static final Map<ReaderContext, Throwable> ACTIVE_SEARCH_CONTEXTS = new ConcurrentHashMap<>();
 
     /** Throw an {@link AssertionError} if there are still in-flight contexts. */
     public static void assertNoInFlightContext() {
-        final Map<SearchContext, Throwable> copy = new HashMap<>(ACTIVE_SEARCH_CONTEXTS);
+        final Map<ReaderContext, Throwable> copy = new HashMap<>(ACTIVE_SEARCH_CONTEXTS);
         if (copy.isEmpty() == false) {
             throw new AssertionError(
                     "There are still [" + copy.size()
@@ -56,14 +56,14 @@ public static void assertNoInFlightContext() {
     /**
      * Add an active search context to the list of tracked contexts. Package private for testing.
      */
-    static void addActiveContext(SearchContext context) {
+    static void addActiveContext(ReaderContext context) {
         ACTIVE_SEARCH_CONTEXTS.put(context, new RuntimeException(context.toString()));
     }
 
     /**
      * Clear an active search context from the list of tracked contexts. Package private for testing.
      */
-    static void removeActiveContext(SearchContext context) {
+    static void removeActiveContext(ReaderContext context) {
         ACTIVE_SEARCH_CONTEXTS.remove(context);
     }
 
@@ -74,14 +74,14 @@ public MockSearchService(ClusterService clusterService,
     }
 
     @Override
-    protected void putContext(SearchContext context) {
+    protected void putReaderContext(ReaderContext context) {
         addActiveContext(context);
-        super.putContext(context);
+        super.putReaderContext(context);
     }
 
     @Override
-    protected SearchContext removeContext(long id) {
-        final SearchContext removed = super.removeContext(id);
+    protected ReaderContext removeReaderContext(long id) {
+        final ReaderContext removed = super.removeReaderContext(id);
         if (removed != null) {
             removeActiveContext(removed);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index ee08622fef072..ab7bd742462b7 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -303,7 +303,7 @@ public boolean shouldCache(Query query) {
              * close their sub-aggregations. This is fairly similar to what the production code does. */
             releasables.add((Releasable) invocation.getArguments()[0]);
             return null;
-        }).when(searchContext).addReleasable(anyObject(), anyObject());
+        }).when(searchContext).addReleasable(anyObject());
         return searchContext;
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index e7b5e937c0c02..5ae5d498d7975 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -110,12 +110,18 @@ public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexSh
     }
 
     public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexShard, ContextIndexSearcher searcher) {
+        this(queryShardContext, indexShard, searcher, null);
+    }
+
+    public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexShard,
+                             ContextIndexSearcher searcher, ScrollContext scrollContext) {
         this.bigArrays = null;
         this.indexService = null;
         this.fixedBitSetFilterCache = null;
         this.indexShard = indexShard;
         this.queryShardContext = queryShardContext;
         this.searcher = searcher;
+        this.scrollContext = scrollContext;
     }
 
     public void setSearcher(ContextIndexSearcher searcher) {
@@ -166,22 +172,11 @@ public float queryBoost() {
         return 0;
     }
 
-    @Override
-    public long getOriginNanoTime() {
-        return originNanoTime;
-    }
-
     @Override
     public ScrollContext scrollContext() {
         return scrollContext;
     }
 
-    @Override
-    public SearchContext scrollContext(ScrollContext scrollContext) {
-        this.scrollContext = scrollContext;
-        return this;
-    }
-
     @Override
     public SearchContextAggregations aggregations() {
         return aggregations;
@@ -226,10 +221,6 @@ public List<RescoreContext> rescore() {
         return Collections.emptyList();
     }
 
-    @Override
-    public void addRescore(RescoreContext rescore) {
-    }
-
     @Override
     public boolean hasScriptFields() {
         return false;
@@ -536,24 +527,6 @@ public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int
         return null;
     }
 
-    @Override
-    public void accessed(long accessTime) {
-    }
-
-    @Override
-    public long lastAccessTime() {
-        return 0;
-    }
-
-    @Override
-    public long keepAlive() {
-        return 0;
-    }
-
-    @Override
-    public void keepAlive(long keepAlive) {
-    }
-
     @Override
     public DfsSearchResult dfsResult() {
         return null;
diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
index 684210f13c57f..b49f1fb0acaf5 100644
--- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
@@ -19,59 +19,25 @@
 
 package org.elasticsearch.search;
 
-import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.TestSearchContext;
+
+import static org.mockito.Mockito.mock;
 
 public class MockSearchServiceTests extends ESTestCase {
-    public static final IndexMetaData EMPTY_INDEX_METADATA = IndexMetaData.builder("")
-        .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
-        .numberOfShards(1).numberOfReplicas(0).build();
 
     public void testAssertNoInFlightContext() {
-        final long nowInMillis = randomNonNegativeLong();
-        SearchContext s = new TestSearchContext(new QueryShardContext(0,
-            new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null,
-            xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true)) {
-
-            @Override
-            public SearchShardTarget shardTarget() {
-                return new SearchShardTarget("node", new ShardId("idx", "ignored", 0), null, OriginalIndices.NONE);
-            }
-
-            @Override
-            public SearchType searchType() {
-                return SearchType.DEFAULT;
-            }
-
-            @Override
-            public Query query() {
-                return Queries.newMatchAllQuery();
-            }
-        };
-        MockSearchService.addActiveContext(s);
+        ReaderContext reader = mock(ReaderContext.class);
+        MockSearchService.addActiveContext(reader);
         try {
             Throwable e = expectThrows(AssertionError.class, () -> MockSearchService.assertNoInFlightContext());
             assertEquals("There are still [1] in-flight contexts. The first one's creation site is listed as the cause of this exception.",
                     e.getMessage());
             e = e.getCause();
-            // The next line with throw an exception if the date looks wrong
-            assertEquals("[node][idx][0] query=[*:*]", e.getMessage());
             assertEquals(MockSearchService.class.getName(), e.getStackTrace()[0].getClassName());
             assertEquals(MockSearchServiceTests.class.getName(), e.getStackTrace()[1].getClassName());
         } finally {
-            MockSearchService.removeActiveContext(s);
+            MockSearchService.removeActiveContext(reader);
         }
     }
 }
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index 5a22857494557..e78e937a1f142 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.shard.SearchOperationListener;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.transport.TransportRequest;
 
@@ -285,27 +286,34 @@ public void validateSearchContext(SearchContext context, TransportRequest transp
                 } catch (IOException e) {
                     throw new UncheckedIOException(e);
                 }
-                // also register a release resource in this case if we have multiple roundtrips like in DFS
-                registerRelease(context, lazyDirectoryReader);
             }
         }
 
-        private void registerRelease(SearchContext context, LazyDirectoryReader lazyDirectoryReader) {
-            context.addReleasable(() -> {
+        @Override
+        public void onFreeSearchContext(SearchContext context) {
+            DirectoryReader dirReader = context.searcher().getDirectoryReader();
+            LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
+            if (lazyDirectoryReader != null) {
                 try {
                     lazyDirectoryReader.release();
                 } catch (IOException e) {
                     throw new UncheckedIOException(e);
                 }
-            }, SearchContext.Lifetime.PHASE);
+            }
         }
 
         @Override
-        public void onNewContext(SearchContext context) {
-            DirectoryReader dirReader = context.searcher().getDirectoryReader();
+        public void onNewReaderContext(ReaderContext readerContext) {
+            DirectoryReader dirReader = readerContext.engineSearcher().getDirectoryReader();
             LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
             if (lazyDirectoryReader != null) {
-                registerRelease(context, lazyDirectoryReader);
+                readerContext.addOnClose(() -> {
+                    try {
+                        lazyDirectoryReader.release();
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
+                    }
+                });
             }
         }
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
index adfe0d4dd8e4c..c9b6708cf8dc2 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
@@ -47,9 +47,9 @@ public SecuritySearchOperationListener(SecurityContext securityContext, XPackLic
      * Adds the {@link Authentication} to the {@link ScrollContext}
      */
     @Override
-    public void onNewScrollContext(SearchContext searchContext) {
+    public void onNewScrollContext(ScrollContext scrollContext) {
         if (licenseState.isAuthAllowed()) {
-            searchContext.scrollContext().putInContext(AuthenticationField.AUTHENTICATION_KEY, securityContext.getAuthentication());
+            scrollContext.putInContext(AuthenticationField.AUTHENTICATION_KEY, securityContext.getAuthentication());
         }
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index aa3b612e513ba..bcab618e0ce7f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -51,18 +51,19 @@ public void testUnlicensed() {
         final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
         AuditTrailService auditTrailService = mock(AuditTrailService.class);
         SearchContext searchContext = mock(SearchContext.class);
-        when(searchContext.scrollContext()).thenReturn(new ScrollContext());
+        ScrollContext scrollContext = new ScrollContext();
+        when(searchContext.scrollContext()).thenReturn(scrollContext);
 
         SecuritySearchOperationListener listener = new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService);
-        listener.onNewScrollContext(searchContext);
+        listener.onNewScrollContext(scrollContext);
         listener.validateSearchContext(searchContext, Empty.INSTANCE);
         verify(licenseState, times(2)).isAuthAllowed();
         verifyZeroInteractions(auditTrailService, searchContext);
     }
 
     public void testOnNewContextSetsAuthentication() throws Exception {
-        TestScrollSearchContext testSearchContext = new TestScrollSearchContext();
-        testSearchContext.scrollContext(new ScrollContext());
+        ScrollContext scrollContext = new ScrollContext();
+        TestSearchContext testSearchContext = new TestSearchContext(null, null, null, scrollContext);
         final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(2L));
         testSearchContext.scrollContext().scroll = scroll;
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
@@ -74,7 +75,7 @@ public void testOnNewContextSetsAuthentication() throws Exception {
         authentication.writeToContext(threadContext);
 
         SecuritySearchOperationListener listener = new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService);
-        listener.onNewScrollContext(testSearchContext);
+        listener.onNewScrollContext(scrollContext);
 
         Authentication contextAuth = testSearchContext.scrollContext().getFromContext(AuthenticationField.AUTHENTICATION_KEY);
         assertEquals(authentication, contextAuth);
@@ -85,8 +86,8 @@ public void testOnNewContextSetsAuthentication() throws Exception {
     }
 
     public void testValidateSearchContext() throws Exception {
-        TestScrollSearchContext testSearchContext = new TestScrollSearchContext();
-        testSearchContext.scrollContext(new ScrollContext());
+        ScrollContext scrollContext = new ScrollContext();
+        TestSearchContext testSearchContext = new TestSearchContext(null, null, null, scrollContext);
         testSearchContext.scrollContext().putInContext(AuthenticationField.AUTHENTICATION_KEY,
                 new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null));
         testSearchContext.scrollContext().scroll = new Scroll(TimeValue.timeValueSeconds(2L));
@@ -239,24 +240,4 @@ public void testEnsuredAuthenticatedUserIsSame() {
         verify(auditTrail).accessDenied(eq(auditId), eq(runAsDiffType), eq(action), eq(request),
             authzInfoRoles(original.getUser().roles()));
     }
-
-    static class TestScrollSearchContext extends TestSearchContext {
-
-        private ScrollContext scrollContext;
-
-        TestScrollSearchContext() {
-            super(null);
-        }
-
-        @Override
-        public ScrollContext scrollContext() {
-            return scrollContext;
-        }
-
-        @Override
-        public SearchContext scrollContext(ScrollContext scrollContext) {
-            this.scrollContext = scrollContext;
-            return this;
-        }
-    }
 }

From 206381e6a86c4afb24d70c17c9d854512efc1cbe Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Tue, 3 Mar 2020 14:33:55 -0500
Subject: [PATCH 04/47] Move states of search to coordinating node (#52741)

This commit moves the states of search to the coordinating node instead
of keeping them in the data node.

Relates #46523
---
 .../percolator/PercolatorQuerySearchIT.java   | 12 +++
 .../action/search/DfsQueryPhase.java          | 20 ++---
 .../action/search/FetchSearchPhase.java       | 20 +++--
 .../SearchDfsQueryThenFetchAsyncAction.java   |  9 ++-
 .../SearchQueryThenFetchAsyncAction.java      |  2 +-
 .../action/search/SearchTransportService.java |  9 ++-
 .../search/DefaultSearchContext.java          | 22 ++++--
 .../elasticsearch/search/RescoreDocIds.java   | 51 +++++++++++++
 .../search/SearchPhaseResult.java             | 21 ++++++
 .../elasticsearch/search/SearchService.java   | 69 ++++++++++-------
 .../search/dfs/DfsSearchResult.java           | 13 +++-
 .../search/fetch/ShardFetchRequest.java       | 21 +++++-
 .../search/fetch/ShardFetchSearchRequest.java | 47 ++++++++++--
 .../internal/FilteredSearchContext.java       |  5 ++
 .../search/internal/LegacyReaderContext.java  | 74 +++++++++++++++++++
 .../search/internal/ReaderContext.java        | 67 ++++++-----------
 .../search/internal/SearchContext.java        | 35 +++++++++
 .../search/query/QuerySearchRequest.java      | 31 +++++---
 .../search/query/QuerySearchResult.java       | 13 +++-
 .../search/rescore/RescoreContext.java        |  4 +
 .../action/search/CountedCollectorTests.java  |  2 +-
 .../action/search/DfsQueryPhaseTests.java     | 16 ++--
 .../action/search/FetchSearchPhaseTests.java  | 34 +++++----
 .../search/SearchPhaseControllerTests.java    | 24 +++---
 .../search/DefaultSearchContextTests.java     | 11 +--
 .../search/SearchServiceTests.java            | 29 ++++----
 .../search/query/QuerySearchResultTests.java  | 10 ++-
 .../elasticsearch/test/TestSearchContext.java |  5 ++
 28 files changed, 505 insertions(+), 171 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/search/RescoreDocIds.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java

diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index f7701bfa96be4..453b1c888e6e2 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -36,13 +36,17 @@
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
@@ -71,6 +75,14 @@
 
 public class PercolatorQuerySearchIT extends ESIntegTestCase {
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>();
+        plugins.addAll(super.nodePlugins());
+        plugins.add(PercolatorPlugin.class);
+        return plugins;
+    }
+
     public void testPercolatorQuery() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test")
                 .setMapping("id", "type=keyword", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator")
diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
index b4d52fa418e10..530675dbae7db 100644
--- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.action.search;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -43,13 +42,15 @@
 final class DfsQueryPhase extends SearchPhase {
     private final ArraySearchPhaseResults<SearchPhaseResult> queryResult;
     private final SearchPhaseController searchPhaseController;
-    private final AtomicArray<DfsSearchResult> dfsSearchResults;
+    private final List<DfsSearchResult> searchResults;
+    private final AggregatedDfs dfs;
     private final Function<ArraySearchPhaseResults<SearchPhaseResult>, SearchPhase> nextPhaseFactory;
     private final SearchPhaseContext context;
     private final SearchTransportService searchTransportService;
     private final SearchProgressListener progressListener;
 
-    DfsQueryPhase(AtomicArray<DfsSearchResult> dfsSearchResults,
+    DfsQueryPhase(List<DfsSearchResult> searchResults,
+                  AggregatedDfs dfs,
                   SearchPhaseController searchPhaseController,
                   Function<ArraySearchPhaseResults<SearchPhaseResult>, SearchPhase> nextPhaseFactory,
                   SearchPhaseContext context) {
@@ -57,7 +58,8 @@ final class DfsQueryPhase extends SearchPhase {
         this.progressListener = context.getTask().getProgressListener();
         this.queryResult = searchPhaseController.newSearchPhaseResults(progressListener, context.getRequest(), context.getNumShards());
         this.searchPhaseController = searchPhaseController;
-        this.dfsSearchResults = dfsSearchResults;
+        this.searchResults = searchResults;
+        this.dfs = dfs;
         this.nextPhaseFactory = nextPhaseFactory;
         this.context = context;
         this.searchTransportService = context.getSearchTransport();
@@ -67,18 +69,16 @@ final class DfsQueryPhase extends SearchPhase {
     public void run() throws IOException {
         // TODO we can potentially also consume the actual per shard results from the initial phase here in the aggregateDfs
         // to free up memory early
-        final List<DfsSearchResult> resultList = dfsSearchResults.asList();
-        final AggregatedDfs dfs = searchPhaseController.aggregateDfs(resultList);
         final CountedCollector<SearchPhaseResult> counter = new CountedCollector<>(queryResult::consumeResult,
-            resultList.size(),
+            searchResults.size(),
             () -> context.executeNextPhase(this, nextPhaseFactory.apply(queryResult)), context);
         final SearchSourceBuilder sourceBuilder = context.getRequest().source();
-        progressListener.notifyListShards(progressListener.searchShards(resultList), sourceBuilder == null || sourceBuilder.size() != 0);
-        for (final DfsSearchResult dfsResult : resultList) {
+        progressListener.notifyListShards(progressListener.searchShards(searchResults), sourceBuilder == null || sourceBuilder.size() != 0);
+        for (final DfsSearchResult dfsResult : searchResults) {
             final SearchShardTarget searchShardTarget = dfsResult.getSearchShardTarget();
             Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
             QuerySearchRequest querySearchRequest = new QuerySearchRequest(searchShardTarget.getOriginalIndices(),
-                    dfsResult.getRequestId(), dfs);
+                    dfsResult.getRequestId(), dfsResult.getShardSearchRequest(), dfs);
             final int shardIndex = dfsResult.getShardIndex();
             searchTransportService.sendExecuteQuery(connection, querySearchRequest, context.getTask(),
                 new SearchActionListener<QuerySearchResult>(searchShardTarget, shardIndex) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 41d216072e4b2..980c244d70940 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -25,11 +25,14 @@
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.search.RescoreDocIds;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.Transport;
 
@@ -50,16 +53,19 @@ final class FetchSearchPhase extends SearchPhase {
     private final Logger logger;
     private final SearchPhaseResults<SearchPhaseResult> resultConsumer;
     private final SearchProgressListener progressListener;
+    private final AggregatedDfs aggregatedDfs;
 
     FetchSearchPhase(SearchPhaseResults<SearchPhaseResult> resultConsumer,
                      SearchPhaseController searchPhaseController,
+                     AggregatedDfs aggregatedDfs,
                      SearchPhaseContext context) {
-        this(resultConsumer, searchPhaseController, context,
+        this(resultConsumer, searchPhaseController, aggregatedDfs, context,
             (response, scrollId) -> new ExpandSearchPhase(context, response, scrollId));
     }
 
     FetchSearchPhase(SearchPhaseResults<SearchPhaseResult> resultConsumer,
                      SearchPhaseController searchPhaseController,
+                     AggregatedDfs aggregatedDfs,
                      SearchPhaseContext context, BiFunction<InternalSearchResponse, String, SearchPhase> nextPhaseFactory) {
         super("fetch");
         if (context.getNumShards() != resultConsumer.getNumShards()) {
@@ -69,6 +75,7 @@ final class FetchSearchPhase extends SearchPhase {
         this.fetchResults = new AtomicArray<>(resultConsumer.getNumShards());
         this.searchPhaseController = searchPhaseController;
         this.queryResults = resultConsumer.getAtomicArray();
+        this.aggregatedDfs = aggregatedDfs;
         this.nextPhaseFactory =  nextPhaseFactory;
         this.context = context;
         this.logger = context.getLogger();
@@ -144,7 +151,8 @@ private void innerRun() throws IOException {
                         Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(),
                             searchShardTarget.getNodeId());
                         ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult().getRequestId(), i, entry,
-                            lastEmittedDocPerShard, searchShardTarget.getOriginalIndices());
+                            lastEmittedDocPerShard, searchShardTarget.getOriginalIndices(), queryResult.getShardSearchRequest(),
+                            queryResult.getRescoreDocIds());
                         executeFetch(i, searchShardTarget, counter, fetchSearchRequest, queryResult.queryResult(),
                             connection);
                     }
@@ -153,10 +161,12 @@ private void innerRun() throws IOException {
         }
     }
 
-    protected ShardFetchSearchRequest createFetchRequest(long queryId, int index, IntArrayList entry,
-                                                               ScoreDoc[] lastEmittedDocPerShard, OriginalIndices originalIndices) {
+    protected ShardFetchSearchRequest createFetchRequest(long queryId, int index, IntArrayList entry, ScoreDoc[] lastEmittedDocPerShard,
+                                                         OriginalIndices originalIndices, ShardSearchRequest shardSearchRequest,
+                                                         RescoreDocIds rescoreDocIds) {
         final ScoreDoc lastEmittedDoc = (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[index] : null;
-        return new ShardFetchSearchRequest(originalIndices, queryId, entry, lastEmittedDoc);
+        return new ShardFetchSearchRequest(originalIndices, queryId, shardSearchRequest, entry, lastEmittedDoc,
+            rescoreDocIds, aggregatedDfs);
     }
 
     private void executeFetch(final int shardIndex, final SearchShardTarget shardTarget,
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index 0782fbb310b65..25d604758bd9f 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -23,10 +23,12 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.transport.Transport;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Executor;
@@ -59,7 +61,10 @@ protected void executePhaseOnShard(final SearchShardIterator shardIt, final Shar
 
     @Override
     protected SearchPhase getNextPhase(final SearchPhaseResults<DfsSearchResult> results, final SearchPhaseContext context) {
-        return new DfsQueryPhase(results.getAtomicArray(), searchPhaseController, (queryResults) ->
-            new FetchSearchPhase(queryResults, searchPhaseController, context), context);
+        final List<DfsSearchResult> dfsSearchResults = results.getAtomicArray().asList();
+        final AggregatedDfs aggregatedDfs = searchPhaseController.aggregateDfs(dfsSearchResults);
+
+        return new DfsQueryPhase(dfsSearchResults, aggregatedDfs, searchPhaseController, (queryResults) ->
+            new FetchSearchPhase(queryResults, searchPhaseController, aggregatedDfs, context), context);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index d5060b728347d..f5a9d23928ff4 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -70,6 +70,6 @@ protected void onShardGroupFailure(int shardIndex, Exception exc) {
 
     @Override
     protected SearchPhase getNextPhase(final SearchPhaseResults<SearchPhaseResult> results, final SearchPhaseContext context) {
-        return new FetchSearchPhase(results, searchPhaseController, context);
+        return new FetchSearchPhase(results, searchPhaseController, null, context);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 53d8d266ea795..3b90b7c38766d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.IndicesRequest;
@@ -282,6 +283,10 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
+    static boolean keepStatesInContext(Version version) {
+        return version.before(Version.V_8_0_0);
+    }
+
     public static void registerRequestHandler(TransportService transportService, SearchService searchService) {
         transportService.registerRequestHandler(FREE_CONTEXT_SCROLL_ACTION_NAME, ThreadPool.Names.SAME, ScrollFreeContextRequest::new,
             (request, channel, task) -> {
@@ -306,7 +311,7 @@ public static void registerRequestHandler(TransportService transportService, Sea
 
         transportService.registerRequestHandler(DFS_ACTION_NAME, ThreadPool.Names.SAME, ShardSearchRequest::new,
             (request, channel, task) ->
-                searchService.executeDfsPhase(request, (SearchShardTask) task,
+                searchService.executeDfsPhase(request, keepStatesInContext(channel.getVersion()), (SearchShardTask) task,
                     new ChannelActionListener<>(channel, DFS_ACTION_NAME, request))
         );
 
@@ -314,7 +319,7 @@ public static void registerRequestHandler(TransportService transportService, Sea
 
         transportService.registerRequestHandler(QUERY_ACTION_NAME, ThreadPool.Names.SAME, ShardSearchRequest::new,
             (request, channel, task) -> {
-                searchService.executeQueryPhase(request, (SearchShardTask) task,
+                searchService.executeQueryPhase(request, keepStatesInContext(channel.getVersion()), (SearchShardTask) task,
                     new ChannelActionListener<>(channel, QUERY_ACTION_NAME, request));
             });
         TransportActionProxy.registerProxyActionWithDynamicResponseType(transportService, QUERY_ACTION_NAME,
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 6571d60b0d3ed..9ecffb1571cc9 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -138,6 +138,7 @@ final class DefaultSearchContext extends SearchContext {
     private SearchContextAggregations aggregations;
     private SearchContextHighlight highlight;
     private SuggestionSearchContext suggest;
+    private List<RescoreContext> rescore;
     private Profilers profilers;
 
     private final Map<String, SearchExtBuilder> searchExtBuilders = new HashMap<>();
@@ -155,8 +156,8 @@ final class DefaultSearchContext extends SearchContext {
         this.shardTarget = shardTarget;
         // SearchContexts use a BigArrays that can circuit break
         this.bigArrays = bigArrays.withCircuitBreaking();
-        this.dfsResult = new DfsSearchResult(readerContext.id(), shardTarget);
-        this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget);
+        this.dfsResult = new DfsSearchResult(readerContext.id(), shardTarget, request);
+        this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget, request);
         this.fetchResult = new FetchSearchResult(readerContext.id(), shardTarget);
         this.indexShard = indexShard;
         this.indexService = indexService;
@@ -164,7 +165,6 @@ final class DefaultSearchContext extends SearchContext {
         final Engine.Searcher engineSearcher = readerContext.engineSearcher();
         this.searcher = new ContextIndexSearcher(engineSearcher.getIndexReader(), engineSearcher.getSimilarity(),
             engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy());
-        searcher.setAggregatedDfs(readerContext.aggregatedDfs());
         this.relativeTimeSupplier = relativeTimeSupplier;
         this.timeout = timeout;
         queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher,
@@ -203,7 +203,7 @@ public void preProcess(boolean rewrite) {
                             + "]. Scroll batch sizes cost as much memory as result windows so they are controlled by the ["
                             + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey() + "] index level setting.");
         }
-        if (rescore().isEmpty() == false) {
+        if (rescore != null) {
             if (sort != null) {
                 throw new IllegalArgumentException("Cannot use [sort] option in conjunction with [rescore].");
             }
@@ -327,7 +327,6 @@ public ScrollContext scrollContext() {
         return readerContext.scrollContext();
     }
 
-
     @Override
     public SearchContextAggregations aggregations() {
         return aggregations;
@@ -373,7 +372,18 @@ public void suggest(SuggestionSearchContext suggest) {
 
     @Override
     public List<RescoreContext> rescore() {
-        return readerContext.rescore();
+        if (rescore == null) {
+            return List.of();
+        }
+        return rescore;
+    }
+
+    @Override
+    public void addRescore(RescoreContext rescore) {
+        if (this.rescore == null) {
+            this.rescore = new ArrayList<>();
+        }
+        this.rescore.add(rescore);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/RescoreDocIds.java b/server/src/main/java/org/elasticsearch/search/RescoreDocIds.java
new file mode 100644
index 0000000000000..7d908f53518b9
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/RescoreDocIds.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+public final class RescoreDocIds implements Writeable {
+    public static final RescoreDocIds EMPTY = new RescoreDocIds(Map.of());
+
+    private final Map<Integer, Set<Integer>> docIds;
+
+    public RescoreDocIds(Map<Integer, Set<Integer>> docIds) {
+        this.docIds = docIds;
+    }
+
+    public RescoreDocIds(StreamInput in) throws IOException {
+        docIds = in.readMap(StreamInput::readVInt, i -> i.readSet(StreamInput::readVInt));
+    }
+
+    public Set<Integer> getId(int index) {
+        return docIds.get(index);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(docIds, StreamOutput::writeVInt, (o, v) -> o.writeCollection(v, StreamOutput::writeVInt));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java b/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
index 77af879831e02..b524fd331e1d4 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
@@ -19,9 +19,11 @@
 
 package org.elasticsearch.search;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.TransportResponse;
 
@@ -40,6 +42,8 @@ public abstract class SearchPhaseResult extends TransportResponse {
     private SearchShardTarget searchShardTarget;
     private int shardIndex = -1;
     protected long requestId;
+    private ShardSearchRequest shardSearchRequest;
+    private RescoreDocIds rescoreDocIds = RescoreDocIds.EMPTY;
 
     protected SearchPhaseResult() {
 
@@ -90,6 +94,23 @@ public QuerySearchResult queryResult() {
      */
     public FetchSearchResult fetchResult() { return null; }
 
+    @Nullable
+    public ShardSearchRequest getShardSearchRequest() {
+        return shardSearchRequest;
+    }
+
+    public void setShardSearchRequest(ShardSearchRequest shardSearchRequest) {
+        this.shardSearchRequest = shardSearchRequest;
+    }
+
+    public RescoreDocIds getRescoreDocIds() {
+        return rescoreDocIds;
+    }
+
+    public void setRescoreDocIds(RescoreDocIds rescoreDocIds) {
+        this.rescoreDocIds = rescoreDocIds;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         // TODO: this seems wrong, SearchPhaseResult should have a writeTo?
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index e856e8a12a374..ebddbba761e79 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -85,6 +85,7 @@
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.search.internal.LegacyReaderContext;
 import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
@@ -308,7 +309,8 @@ protected void doClose() {
         keepAliveReaper.cancel();
     }
 
-    public void executeDfsPhase(ShardSearchRequest request, SearchShardTask task, ActionListener<SearchPhaseResult> listener) {
+    public void executeDfsPhase(ShardSearchRequest request, boolean keepStatesInContext,
+                                SearchShardTask task, ActionListener<SearchPhaseResult> listener) {
         rewriteShardRequest(request, ActionListener.wrap(
             // fork the execution in the search thread pool and wraps the searcher
             // to execute the query
@@ -317,7 +319,7 @@ public void executeDfsPhase(ShardSearchRequest request, SearchShardTask task, Ac
                     context.wrapSearcher().execute(() -> {
                         final SearchPhaseResult result;
                         try {
-                            result = executeDfsPhase(context, task);
+                            result = executeDfsPhase(context, keepStatesInContext, task);
                         } catch (Exception exc) {
                             listener.onFailure(exc);
                             return;
@@ -332,9 +334,9 @@ public void executeDfsPhase(ShardSearchRequest request, SearchShardTask task, Ac
             }, listener::onFailure));
     }
 
-    private DfsSearchResult executeDfsPhase(SearchRewriteContext rewriteContext, SearchShardTask task) {
-        final ReaderContext reader = createAndPutReaderContext(rewriteContext);
-        try (SearchContext context = createContext(reader, reader.request(), task, true)) {
+    private DfsSearchResult executeDfsPhase(SearchRewriteContext rewriteContext, boolean keepStatesInContext, SearchShardTask task) {
+        final ReaderContext reader = createAndPutReaderContext(rewriteContext, keepStatesInContext);
+        try (SearchContext context = createContext(reader, rewriteContext.request, task, true)) {
             dfsPhase.execute(context);
             return context.dfsResult();
         } catch (Exception e) {
@@ -357,7 +359,8 @@ private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final Sea
         }
     }
 
-    public void executeQueryPhase(ShardSearchRequest request, SearchShardTask task, ActionListener<SearchPhaseResult> listener) {
+    public void executeQueryPhase(ShardSearchRequest request, boolean keepStatesInContext,
+                                  SearchShardTask task, ActionListener<SearchPhaseResult> listener) {
         assert request.canReturnNullResponseIfMatchNoDocs() == false || request.numberOfShards() > 1
             : "empty responses require more than one shard";
         rewriteShardRequest(request, ActionListener.wrap(
@@ -375,7 +378,7 @@ && canRewriteToMatchNone(rewritten.source())
                         context.wrapSearcher().execute(() -> {
                             final SearchPhaseResult result;
                             try {
-                                result = executeQueryPhase(context, task);
+                                result = executeQueryPhase(context, keepStatesInContext, task);
                             } catch (Exception exc) {
                                 listener.onFailure(exc);
                                 return;
@@ -405,10 +408,11 @@ private <T> void runAsync(long id, Supplier<T> executable, ActionListener<T> lis
         getExecutor(id).execute(ActionRunnable.supply(listener, executable::get));
     }
 
-    private SearchPhaseResult executeQueryPhase(SearchRewriteContext rewriteContext, SearchShardTask task) throws Exception {
-        final ReaderContext reader = createAndPutReaderContext(rewriteContext);
+    private SearchPhaseResult executeQueryPhase(SearchRewriteContext rewriteContext, boolean keepStatesInContext,
+                                                SearchShardTask task) throws Exception {
+        final ReaderContext reader = createAndPutReaderContext(rewriteContext, keepStatesInContext);
         final ShardSearchRequest request = rewriteContext.request;
-        try (SearchContext context = createContext(reader, reader.request(), task, true)) {
+        try (SearchContext context = createContext(reader, request, task, true)) {
             final long afterQueryTime;
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
                 loadOrExecuteQueryPhase(request, context);
@@ -449,13 +453,17 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
         runAsync(request.id(), () -> {
-            final ReaderContext reader = findReaderContext(request.id());
-            try (SearchContext context = createContext(reader, reader.request(), task, false);
+            final LegacyReaderContext reader = (LegacyReaderContext) findReaderContext(request.id());
+            try (SearchContext context = createContext(reader, reader.getShardSearchRequest(null), task, false);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
                 reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
+                context.searcher().setAggregatedDfs(reader.getAggregatedDfs(null));
                 processScroll(request, reader, context);
                 queryPhase.execute(context);
                 executor.success();
+                final RescoreDocIds rescoreDocIds = context.rescoreDocIds();
+                context.queryResult().setRescoreDocIds(rescoreDocIds);
+                reader.setRescoreDocIds(rescoreDocIds);
                 return new ScrollQuerySearchResult(context.queryResult(), context.shardTarget());
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
@@ -468,8 +476,8 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
         runAsync(request.id(), () -> {
             final ReaderContext reader = findReaderContext(request.id());
-            reader.aggregatedDfs(request.dfs());
-            try (SearchContext context = createContext(reader, reader.request(), task, true);
+            reader.setAggregatedDfs(request.dfs());
+            try (SearchContext context = createContext(reader, reader.getShardSearchRequest(request.shardSearchRequest()), task, true);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
                 reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 context.searcher().setAggregatedDfs(request.dfs());
@@ -479,6 +487,9 @@ public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task,
                     freeReaderContext(reader.id());
                 }
                 executor.success();
+                final RescoreDocIds rescoreDocIds = context.rescoreDocIds();
+                context.queryResult().setRescoreDocIds(rescoreDocIds);
+                reader.setRescoreDocIds(rescoreDocIds);
                 return context.queryResult();
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
@@ -510,9 +521,11 @@ private Executor getExecutor(IndexShard indexShard) {
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
         runAsync(request.id(), () -> {
-            final ReaderContext reader = findReaderContext(request.id());
-            try (SearchContext context = createContext(reader, reader.request(), task, false);
+            final LegacyReaderContext reader = (LegacyReaderContext) findReaderContext(request.id());
+            try (SearchContext context = createContext(reader, reader.getShardSearchRequest(null), task, false);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                context.assignRescoreDocIds(reader.getRescoreDocIds(null));
+                context.searcher().setAggregatedDfs(reader.getAggregatedDfs(null));
                 reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 processScroll(request, reader, context);
                 queryPhase.execute(context);
@@ -530,11 +543,14 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
         runAsync(request.id(), () -> {
             final ReaderContext reader = findReaderContext(request.id());
-            try (SearchContext context = createContext(reader, reader.request(), task, false)) {
+            final ShardSearchRequest shardSearchRequest = reader.getShardSearchRequest(request.getShardSearchRequest());
+            try (SearchContext context = createContext(reader, shardSearchRequest, task, false)) {
                 reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 if (request.lastEmittedDoc() != null) {
                     context.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
                 }
+                context.assignRescoreDocIds(reader.getRescoreDocIds(request.getRescoreDocIds()));
+                context.searcher().setAggregatedDfs(reader.getAggregatedDfs(request.getAggregatedDfs()));
                 context.docIdsToLoad(request.docIds(), 0, request.docIdsSize());
                 try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, System.nanoTime())) {
                     fetchPhase.execute(context);
@@ -560,7 +576,7 @@ private ReaderContext findReaderContext(long id) throws SearchContextMissingExce
         return reader;
     }
 
-    final ReaderContext createAndPutReaderContext(SearchRewriteContext rewriteContext) {
+    final ReaderContext createAndPutReaderContext(SearchRewriteContext rewriteContext, boolean keepStatesInContext) {
         final ShardSearchRequest request = rewriteContext.request;
         final SearchOperationListener searchOperationListener = rewriteContext.shard.getSearchOperationListener();
         Engine.Searcher engineSearcher = rewriteContext.searcher;
@@ -572,7 +588,11 @@ final ReaderContext createAndPutReaderContext(SearchRewriteContext rewriteContex
                         maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
                         + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
             }
-            readerContext = new ReaderContext(idGenerator.incrementAndGet(), rewriteContext.shard, engineSearcher, request);
+            if (keepStatesInContext || request.scroll() != null) {
+                readerContext = new LegacyReaderContext(idGenerator.incrementAndGet(), rewriteContext.shard, engineSearcher, request);
+            } else {
+                readerContext = new ReaderContext(idGenerator.incrementAndGet(), rewriteContext.shard, engineSearcher);
+            }
             engineSearcher = null;
             final ReaderContext finalReaderContext = readerContext;
             searchOperationListener.onNewReaderContext(finalReaderContext);
@@ -605,7 +625,7 @@ final SearchContext createContext(ReaderContext reader, ShardSearchRequest reque
             if (request.scroll() != null) {
                 context.scrollContext().scroll = request.scroll();
             }
-            parseSource(reader, context, request.source(), includeAggregations);
+            parseSource(context, request.source(), includeAggregations);
 
             // if the from and size are still not set, default them
             if (context.from() == -1) {
@@ -642,7 +662,7 @@ final SearchContext createContext(ReaderContext reader, ShardSearchRequest reque
     public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout) {
         IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().getId());
         Engine.Searcher engineSearcher = indexShard.acquireSearcher("search");
-        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, engineSearcher, request)) {
+        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, engineSearcher)) {
             engineSearcher = null; // transfer ownership to readerContext
             return createSearchContext(readerContext, request, timeout);
         } finally {
@@ -724,8 +744,7 @@ private void processFailure(ReaderContext context, Exception e) {
         }
     }
 
-    private void parseSource(ReaderContext reader, DefaultSearchContext context,
-                             SearchSourceBuilder source, boolean includeAggregations) throws SearchException {
+    private void parseSource(DefaultSearchContext context, SearchSourceBuilder source, boolean includeAggregations) {
         // nothing to parse...
         if (source == null) {
             return;
@@ -796,10 +815,10 @@ private void parseSource(ReaderContext reader, DefaultSearchContext context,
                 throw new SearchException(shardTarget, "failed to create SuggestionSearchContext", e);
             }
         }
-        if (source.rescores() != null && reader.rescore().isEmpty()) {
+        if (source.rescores() != null) {
             try {
                 for (RescorerBuilder<?> rescore : source.rescores()) {
-                    reader.addRescore(rescore.buildContext(queryShardContext));
+                    context.addRescore(rescore.buildContext(queryShardContext));
                 }
             } catch (IOException e) {
                 throw new SearchException(shardTarget, "failed to create RescoreSearchContext", e);
diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
index 4d245e2bcb002..85d7261b2615b 100644
--- a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
@@ -25,11 +25,13 @@
 import org.apache.lucene.search.CollectionStatistics;
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.collect.HppcMaps;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 
 import java.io.IOException;
 
@@ -58,11 +60,15 @@ public DfsSearchResult(StreamInput in) throws IOException {
         fieldStatistics = readFieldStats(in);
 
         maxDoc = in.readVInt();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new));
+        }
     }
 
-    public DfsSearchResult(long id, SearchShardTarget shardTarget) {
+    public DfsSearchResult(long id, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) {
         this.setSearchShardTarget(shardTarget);
         this.requestId = id;
+        setShardSearchRequest(shardSearchRequest);
     }
 
     public DfsSearchResult maxDoc(int maxDoc) {
@@ -97,7 +103,7 @@ public ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics() {
         return fieldStatistics;
     }
 
-  @Override
+    @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(requestId);
         out.writeVInt(terms.length);
@@ -108,6 +114,9 @@ public void writeTo(StreamOutput out) throws IOException {
         writeTermStats(out, termStatistics);
         writeFieldStats(out, fieldStatistics);
         out.writeVInt(maxDoc);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalWriteable(getShardSearchRequest());
+        }
     }
 
     public static void writeFieldStats(StreamOutput out, ObjectObjectHashMap<String,
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
index 2e7c59b329874..e092abd134ebc 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
@@ -23,9 +23,13 @@
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.search.SearchShardTask;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.search.RescoreDocIds;
+import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
@@ -47,9 +51,6 @@ public class ShardFetchRequest extends TransportRequest {
 
     private ScoreDoc lastEmittedDoc;
 
-    public ShardFetchRequest() {
-    }
-
     public ShardFetchRequest(long id, IntArrayList list, ScoreDoc lastEmittedDoc) {
         this.id = id;
         this.docIds = list.buffer;
@@ -120,4 +121,18 @@ public String getDescription() {
         return "id[" + id + "], size[" + size + "], lastEmittedDoc[" + lastEmittedDoc + "]";
     }
 
+    @Nullable
+    public ShardSearchRequest getShardSearchRequest() {
+        return null;
+    }
+
+    @Nullable
+    public RescoreDocIds getRescoreDocIds() {
+        return null;
+    }
+
+    @Nullable
+    public AggregatedDfs getAggregatedDfs() {
+        return null;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
index 647d629a390c6..44f629a347841 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
@@ -21,11 +21,15 @@
 
 import com.carrotsearch.hppc.IntArrayList;
 import org.apache.lucene.search.ScoreDoc;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.RescoreDocIds;
+import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 
 import java.io.IOException;
 
@@ -35,26 +39,43 @@
  */
 public class ShardFetchSearchRequest extends ShardFetchRequest implements IndicesRequest {
 
-    private OriginalIndices originalIndices;
+    private final OriginalIndices originalIndices;
+    private final ShardSearchRequest shardSearchRequest;
+    private final RescoreDocIds rescoreDocIds;
+    private final AggregatedDfs aggregatedDfs;
 
-    public ShardFetchSearchRequest() {
-
-    }
-
-    public ShardFetchSearchRequest(OriginalIndices originalIndices, long id, IntArrayList list, ScoreDoc lastEmittedDoc) {
+    public ShardFetchSearchRequest(OriginalIndices originalIndices, long id, ShardSearchRequest shardSearchRequest,
+                                   IntArrayList list, ScoreDoc lastEmittedDoc, RescoreDocIds rescoreDocIds, AggregatedDfs aggregatedDfs) {
         super(id, list, lastEmittedDoc);
         this.originalIndices = originalIndices;
+        this.shardSearchRequest = shardSearchRequest;
+        this.rescoreDocIds = rescoreDocIds;
+        this.aggregatedDfs = aggregatedDfs;
     }
 
     public ShardFetchSearchRequest(StreamInput in) throws IOException {
         super(in);
         originalIndices = OriginalIndices.readOriginalIndices(in);
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            shardSearchRequest = in.readOptionalWriteable(ShardSearchRequest::new);
+            rescoreDocIds = new RescoreDocIds(in);
+            aggregatedDfs = in.readOptionalWriteable(AggregatedDfs::new);
+        } else {
+            shardSearchRequest = null;
+            rescoreDocIds = RescoreDocIds.EMPTY;
+            aggregatedDfs = null;
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         OriginalIndices.writeOriginalIndices(originalIndices, out);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalWriteable(shardSearchRequest);
+            rescoreDocIds.writeTo(out);
+            out.writeOptionalWriteable(aggregatedDfs);
+        }
     }
 
     @Override
@@ -73,4 +94,18 @@ public IndicesOptions indicesOptions() {
         return originalIndices.indicesOptions();
     }
 
+    @Override
+    public ShardSearchRequest getShardSearchRequest() {
+        return shardSearchRequest;
+    }
+
+    @Override
+    public RescoreDocIds getRescoreDocIds() {
+        return rescoreDocIds;
+    }
+
+    @Override
+    public AggregatedDfs getAggregatedDfs() {
+        return aggregatedDfs;
     }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index ab605062de50a..636c0341077a9 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -523,4 +523,9 @@ public SearchContext collapse(CollapseContext collapse) {
     public CollapseContext collapse() {
         return in.collapse();
     }
+
+    @Override
+    public void addRescore(RescoreContext rescore) {
+        in.addRescore(rescore);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
new file mode 100644
index 0000000000000..e97aae9fb152a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.search.RescoreDocIds;
+import org.elasticsearch.search.dfs.AggregatedDfs;
+
+import java.util.Objects;
+
+public class LegacyReaderContext extends ReaderContext {
+    private final ShardSearchRequest shardSearchRequest;
+    private final ScrollContext scrollContext;
+    private AggregatedDfs aggregatedDfs;
+    private RescoreDocIds rescoreDocIds;
+
+    public LegacyReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher, ShardSearchRequest shardSearchRequest) {
+        super(id, indexShard, engineSearcher);
+        this.shardSearchRequest = Objects.requireNonNull(shardSearchRequest);
+        if (shardSearchRequest.scroll() != null) {
+            this.scrollContext = new ScrollContext();
+        } else {
+            this.scrollContext = null;
+        }
+    }
+
+    @Override
+    public ShardSearchRequest getShardSearchRequest(ShardSearchRequest other) {
+        return shardSearchRequest;
+    }
+
+    @Override
+    public ScrollContext scrollContext() {
+        return scrollContext;
+    }
+
+    @Override
+    public AggregatedDfs getAggregatedDfs(AggregatedDfs other) {
+        return aggregatedDfs;
+    }
+
+    @Override
+    public void setAggregatedDfs(AggregatedDfs aggregatedDfs) {
+        this.aggregatedDfs = aggregatedDfs;
+    }
+
+    @Override
+    public RescoreDocIds getRescoreDocIds(RescoreDocIds other) {
+        return rescoreDocIds;
+    }
+
+    @Override
+    public void setRescoreDocIds(RescoreDocIds rescoreDocIds) {
+        this.rescoreDocIds = rescoreDocIds;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
index 7c9dfe14dc9f8..e1bd998ed6006 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
@@ -24,12 +24,11 @@
 import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.search.RescoreDocIds;
 import org.elasticsearch.search.dfs.AggregatedDfs;
-import org.elasticsearch.search.rescore.RescoreContext;
 
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -51,25 +50,13 @@ public class ReaderContext extends AbstractRefCounted implements Releasable {
     private volatile long keepAlive;
     private volatile long lastAccessTime = -1L;
 
-    // BWC
-    private final ShardSearchRequest request;
-    private final ScrollContext scrollContext;
-    private AggregatedDfs aggregatedDfs;
-    private List<RescoreContext> rescore;
-
     private final List<Releasable> onCloses = new CopyOnWriteArrayList<>();
 
-    public ReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher, ShardSearchRequest request) {
+    public ReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher) {
         super("reader_context");
         this.id = id;
         this.indexShard = indexShard;
         this.engineSearcher = engineSearcher;
-        this.request = request;
-        if (request.scroll() != null) {
-            this.scrollContext = new ScrollContext();
-        } else {
-            this.scrollContext = null;
-        }
     }
 
     @Override
@@ -106,22 +93,6 @@ public String source() {
         return engineSearcher.source();
     }
 
-    public ShardSearchRequest request() {
-        return request;
-    }
-
-    public ScrollContext scrollContext() {
-        return scrollContext;
-    }
-
-    public AggregatedDfs aggregatedDfs() {
-        return aggregatedDfs;
-    }
-
-    public void aggregatedDfs(AggregatedDfs aggregatedDfs) {
-        this.aggregatedDfs = aggregatedDfs;
-    }
-
     public void accessed(long accessTime) {
         this.lastAccessTime = accessTime;
     }
@@ -138,18 +109,28 @@ public void keepAlive(long keepAlive) {
         this.keepAlive = keepAlive;
     }
 
-    public List<RescoreContext> rescore() {
-        if (rescore == null) {
-            return Collections.emptyList();
-        } else {
-            return Collections.unmodifiableList(rescore);
-        }
+    // BWC
+    public ShardSearchRequest getShardSearchRequest(ShardSearchRequest other) {
+        return Objects.requireNonNull(other);
     }
 
-    public void addRescore(RescoreContext rescore) {
-        if (this.rescore == null) {
-            this.rescore = new ArrayList<>();
-        }
-        this.rescore.add(rescore);
+    public ScrollContext scrollContext() {
+        return null;
+    }
+
+    public AggregatedDfs getAggregatedDfs(AggregatedDfs other) {
+        return other;
+    }
+
+    public void setAggregatedDfs(AggregatedDfs aggregatedDfs) {
+
+    }
+
+    public RescoreDocIds getRescoreDocIds(RescoreDocIds other) {
+        return Objects.requireNonNull(other);
+    }
+
+    public void setRescoreDocIds(RescoreDocIds rescoreDocIds) {
+
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 0a1fc3822ad42..378e34ca87a13 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.search.RescoreDocIds;
 import org.elasticsearch.search.SearchExtBuilder;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
@@ -58,8 +59,10 @@
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -156,6 +159,38 @@ public InnerHitsContext innerHits() {
      */
     public abstract List<RescoreContext> rescore();
 
+    public abstract void addRescore(RescoreContext rescore);
+
+    public final RescoreDocIds rescoreDocIds() {
+        final List<RescoreContext> rescore = rescore();
+        if (rescore == null) {
+            return RescoreDocIds.EMPTY;
+        }
+        Map<Integer, Set<Integer>> rescoreDocIds = null;
+        for (int i = 0; i < rescore.size(); i++) {
+            final Set<Integer> docIds = rescore.get(i).getRescoredDocs();
+            if (docIds != null && docIds.isEmpty() == false) {
+                if (rescoreDocIds == null) {
+                    rescoreDocIds = new HashMap<>();
+                }
+                rescoreDocIds.put(i, docIds);
+            }
+        }
+        return rescoreDocIds == null ? RescoreDocIds.EMPTY : new RescoreDocIds(rescoreDocIds);
+    }
+
+    public final void assignRescoreDocIds(RescoreDocIds rescoreDocIds) {
+        final List<RescoreContext> rescore = rescore();
+        if (rescore != null) {
+            for (int i = 0; i < rescore.size(); i++) {
+                final Set<Integer> docIds = rescoreDocIds.getId(i);
+                if (docIds != null) {
+                    rescore.get(i).setRescoredDocs(docIds);
+                }
+            }
+        }
+    }
+
     public abstract boolean hasScriptFields();
 
     public abstract ScriptFieldsContext scriptFields();
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
index d3919ec3aba48..55ab6088dd822 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
@@ -19,14 +19,17 @@
 
 package org.elasticsearch.search.query;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchShardTask;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
@@ -36,18 +39,15 @@
 
 public class QuerySearchRequest extends TransportRequest implements IndicesRequest {
 
-    private long id;
+    private final long id;
+    private final AggregatedDfs dfs;
+    private final OriginalIndices originalIndices;
+    private final ShardSearchRequest shardSearchRequest;
 
-    private AggregatedDfs dfs;
-
-    private OriginalIndices originalIndices;
-
-    public QuerySearchRequest() {
-    }
-
-    public QuerySearchRequest(OriginalIndices originalIndices, long id, AggregatedDfs dfs) {
+    public QuerySearchRequest(OriginalIndices originalIndices, long id, ShardSearchRequest shardSearchRequest, AggregatedDfs dfs) {
         this.id = id;
         this.dfs = dfs;
+        this.shardSearchRequest = shardSearchRequest;
         this.originalIndices = originalIndices;
     }
 
@@ -56,6 +56,11 @@ public QuerySearchRequest(StreamInput in) throws IOException {
         id = in.readLong();
         dfs = new AggregatedDfs(in);
         originalIndices = OriginalIndices.readOriginalIndices(in);
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            this.shardSearchRequest = in.readOptionalWriteable(ShardSearchRequest::new);
+        } else {
+            this.shardSearchRequest = null;
+        }
     }
 
     @Override
@@ -64,6 +69,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(id);
         dfs.writeTo(out);
         OriginalIndices.writeOriginalIndices(originalIndices, out);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalWriteable(shardSearchRequest);
+        }
     }
 
     public long id() {
@@ -74,6 +82,11 @@ public AggregatedDfs dfs() {
         return dfs;
     }
 
+    @Nullable
+    public ShardSearchRequest shardSearchRequest() {
+        return shardSearchRequest;
+    }
+
     @Override
     public String[] indices() {
         return originalIndices.indices();
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 8c62d229fe766..87ec31e9862e0 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.RescoreDocIds;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.Aggregations;
@@ -33,6 +34,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
 
@@ -82,10 +84,11 @@ public QuerySearchResult(StreamInput in) throws IOException {
         }
     }
 
-    public QuerySearchResult(long id, SearchShardTarget shardTarget) {
+    public QuerySearchResult(long id, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) {
         this.requestId = id;
         setSearchShardTarget(shardTarget);
         isNull = false;
+        setShardSearchRequest(shardSearchRequest);
     }
 
     private QuerySearchResult(boolean isNull) {
@@ -336,6 +339,10 @@ public void readFromWithId(long id, StreamInput in) throws IOException {
         hasProfileResults = profileShardResults != null;
         serviceTimeEWMA = in.readZLong();
         nodeQueueSize = in.readInt();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new));
+            setRescoreDocIds(new RescoreDocIds(in));
+        }
     }
 
     @Override
@@ -388,6 +395,10 @@ public void writeToNoId(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(profileShardResults);
         out.writeZLong(serviceTimeEWMA);
         out.writeInt(nodeQueueSize);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalWriteable(getShardSearchRequest());
+            getRescoreDocIds().writeTo(out);
+        }
     }
 
     public TotalHits getTotalHits() {
diff --git a/server/src/main/java/org/elasticsearch/search/rescore/RescoreContext.java b/server/src/main/java/org/elasticsearch/search/rescore/RescoreContext.java
index 4f44af6321791..1ec5ecd33dc1a 100644
--- a/server/src/main/java/org/elasticsearch/search/rescore/RescoreContext.java
+++ b/server/src/main/java/org/elasticsearch/search/rescore/RescoreContext.java
@@ -66,6 +66,10 @@ public boolean isRescored(int docId) {
         return rescoredDocs.contains(docId);
     }
 
+    public Set<Integer> getRescoredDocs() {
+        return rescoredDocs;
+    }
+
     /**
      * Returns queries associated with the rescorer
      */
diff --git a/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java b/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
index a85f63abd0967..5f826d7168e86 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
@@ -59,7 +59,7 @@ public void testCollect() throws InterruptedException {
                 case 1:
                     state.add(1);
                     executor.execute(() -> {
-                        DfsSearchResult dfsSearchResult = new DfsSearchResult(shardID, null);
+                        DfsSearchResult dfsSearchResult = new DfsSearchResult(shardID, null, null);
                         dfsSearchResult.setShardIndex(shardID);
                         dfsSearchResult.setSearchShardTarget(new SearchShardTarget("foo",
                             new ShardId("bar", "baz", shardID), null, OriginalIndices.NONE));
diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index 8a8e28e15f20d..e791389413c68 100644
--- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -46,7 +46,7 @@
 public class DfsQueryPhaseTests extends ESTestCase {
 
     private static DfsSearchResult newSearchResult(int shardIndex, long requestId, SearchShardTarget target) {
-        DfsSearchResult result = new DfsSearchResult(requestId, target);
+        DfsSearchResult result = new DfsSearchResult(requestId, target, null);
         result.setShardIndex(shardIndex);
         return result;
     }
@@ -67,7 +67,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
                                          SearchActionListener<QuerySearchResult> listener) {
                 if (request.id() == 1) {
                     QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE));
+                        null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                                     new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
@@ -75,7 +75,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
                     listener.onResponse(queryResult);
                 } else if (request.id() == 2) {
                     QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node2", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE));
+                        null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F),
                             new DocValueFormat[0]);
@@ -88,7 +88,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
         };
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
         mockSearchPhaseContext.searchTransport = searchTransportService;
-        DfsQueryPhase phase = new DfsQueryPhase(results, controller,
+        DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, controller,
             (response) -> new SearchPhase("test") {
             @Override
             public void run() throws IOException {
@@ -127,7 +127,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
                                          SearchActionListener<QuerySearchResult> listener) {
                 if (request.id() == 1) {
                     QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE));
+                        null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(
                             new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                             new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
@@ -142,7 +142,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
         };
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
         mockSearchPhaseContext.searchTransport = searchTransportService;
-        DfsQueryPhase phase = new DfsQueryPhase(results, controller,
+        DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, controller,
             (response) -> new SearchPhase("test") {
                 @Override
                 public void run() throws IOException {
@@ -184,7 +184,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
                                          SearchActionListener<QuerySearchResult> listener) {
                 if (request.id() == 1) {
                     QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE));
+                        null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                                     new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
@@ -199,7 +199,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
         };
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
         mockSearchPhaseContext.searchTransport = searchTransportService;
-        DfsQueryPhase phase = new DfsQueryPhase(results, controller,
+        DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, controller,
             (response) -> new SearchPhase("test") {
                 @Override
                 public void run() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index 5eec29dbf8039..8e70c95b6a0d1 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -70,7 +70,7 @@ public void testShortcutQueryAndFetchOptimization() {
             numHits = 0;
         }
 
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
             @Override
             public void run() {
@@ -96,14 +96,15 @@ public void testFetchTwoDocument() {
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
         QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-            null, OriginalIndices.NONE));
+            null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        queryResult = new QuerySearchResult(
+            321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -126,7 +127,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -154,14 +155,15 @@ public void testFailFetchOneDoc() {
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
         QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-            null, OriginalIndices.NONE));
+            null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        queryResult = new QuerySearchResult(
+            321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -183,7 +185,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
 
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -216,7 +218,7 @@ public void testFetchDocsConcurrently() throws InterruptedException {
             mockSearchPhaseContext.getRequest(), numHits);
         for (int i = 0; i < numHits; i++) {
             QuerySearchResult queryResult = new QuerySearchResult(i, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-                null, OriginalIndices.NONE));
+                null, OriginalIndices.NONE), null);
             queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                     new ScoreDoc[] {new ScoreDoc(i+1, i)}), i), new DocValueFormat[0]);
             queryResult.size(resultSetSize); // the size of the result set
@@ -236,7 +238,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
             }
         };
         CountDownLatch latch = new CountDownLatch(1);
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -273,14 +275,15 @@ public void testExceptionFailsPhase() {
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
         QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-            null, OriginalIndices.NONE));
+            null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        queryResult = new QuerySearchResult(
+            321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -306,7 +309,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -329,14 +332,15 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = 1;
         QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-            null, OriginalIndices.NONE));
+            null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        queryResult = new QuerySearchResult(
+            321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -357,7 +361,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index f49d3a69caca0..256340ef9b1d4 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -230,7 +230,7 @@ private static AtomicArray<SearchPhaseResult> generateQueryResults(int nShards,
             String clusterAlias = randomBoolean() ? null : "remote";
             SearchShardTarget searchShardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex),
                 clusterAlias, OriginalIndices.NONE);
-            QuerySearchResult querySearchResult = new QuerySearchResult(shardIndex, searchShardTarget);
+            QuerySearchResult querySearchResult = new QuerySearchResult(shardIndex, searchShardTarget, null);
             final TopDocs topDocs;
             float maxScore = 0;
             if (searchHitsSize == 0) {
@@ -366,7 +366,7 @@ private void consumerTestCase(int numEmptyResponses) {
         }
 
         QuerySearchResult result = new QuerySearchResult(0, new SearchShardTarget("node", new ShardId("a", "b", 0),
-            null, OriginalIndices.NONE));
+            null, OriginalIndices.NONE), null);
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         InternalAggregations aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 1.0D, DocValueFormat.RAW,
@@ -375,7 +375,7 @@ private void consumerTestCase(int numEmptyResponses) {
         result.setShardIndex(0);
         consumer.consumeResult(result);
 
-        result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
+        result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), null);
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 3.0D, DocValueFormat.RAW,
@@ -384,7 +384,7 @@ private void consumerTestCase(int numEmptyResponses) {
         result.setShardIndex(2);
         consumer.consumeResult(result);
 
-        result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE));
+        result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), null);
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 2.0D, DocValueFormat.RAW,
@@ -453,7 +453,7 @@ public void testConsumerConcurrently() throws InterruptedException {
                 int number = randomIntBetween(1, 1000);
                 max.updateAndGet(prev -> Math.max(prev, number));
                 QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new ShardId("a", "b", id),
-                    null, OriginalIndices.NONE));
+                    null, OriginalIndices.NONE), null);
                 result.topDocs(new TopDocsAndMaxScore(
                     new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(0, number)}), number),
                     new DocValueFormat[0]);
@@ -497,7 +497,7 @@ public void testConsumerOnlyAggs() {
             int number = randomIntBetween(1, 1000);
             max.updateAndGet(prev -> Math.max(prev, number));
             QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+                null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), number),
                     new DocValueFormat[0]);
             InternalAggregations aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", (double) number,
@@ -535,7 +535,7 @@ public void testConsumerOnlyHits() {
             int number = randomIntBetween(1, 1000);
             max.updateAndGet(prev -> Math.max(prev, number));
             QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+                null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                     new ScoreDoc[] {new ScoreDoc(0, number)}), number), new DocValueFormat[0]);
             result.setShardIndex(i);
@@ -602,7 +602,7 @@ public void testReduceTopNWithFromOffset() {
         int score = 100;
         for (int i = 0; i < 4; i++) {
             QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+                null, OriginalIndices.NONE), null);
             ScoreDoc[] docs = new ScoreDoc[3];
             for (int j = 0; j < docs.length; j++) {
                 docs[j] = new ScoreDoc(0, score--);
@@ -644,7 +644,7 @@ public void testConsumerSortByField() {
             FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, new Object[]{number})};
             TopDocs topDocs = new TopFieldDocs(new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields);
             QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+                null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats);
             result.setShardIndex(i);
             result.size(size);
@@ -682,7 +682,7 @@ public void testConsumerFieldCollapsing() {
             FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, values)};
             TopDocs topDocs = new CollapseTopFieldDocs("field", new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields, values);
             QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+                null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats);
             result.setShardIndex(i);
             result.size(size);
@@ -715,7 +715,7 @@ public void testConsumerSuggestions() {
         int maxScoreCompletion = -1;
         for (int i = 0; i < expectedNumResults; i++) {
             QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE));
+                null, OriginalIndices.NONE), null);
             List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions =
                 new ArrayList<>();
             {
@@ -841,7 +841,7 @@ public void onReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggr
                     int number = randomIntBetween(1, 1000);
                     max.updateAndGet(prev -> Math.max(prev, number));
                     QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new ShardId("a", "b", id),
-                        null, OriginalIndices.NONE));
+                        null, OriginalIndices.NONE), null);
                     result.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[]{new ScoreDoc(0, number)}), number),
                         new DocValueFormat[0]);
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 198daf0f41a11..c6e41d805657a 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -50,6 +50,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.LegacyReaderContext;
 import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.rescore.RescoreContext;
@@ -119,7 +120,7 @@ public void testPreProcess() throws Exception {
 
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
-            ReaderContext readerWithoutScroll = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            ReaderContext readerWithoutScroll = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher);
             DefaultSearchContext contextWithoutScroll = new DefaultSearchContext(readerWithoutScroll, shardSearchRequest, target, null,
                 indexService, indexShard, bigArrays, null, timeout, null);
             contextWithoutScroll.from(300);
@@ -134,7 +135,7 @@ public void testPreProcess() throws Exception {
 
             // resultWindow greater than maxResultWindow and scrollContext isn't null
             when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMillis(randomInt(1000))));
-            ReaderContext readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            ReaderContext readerContext = new LegacyReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
             DefaultSearchContext context1 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null, indexService,
                 indexShard, bigArrays, null, timeout, null);
             context1.from(300);
@@ -151,7 +152,7 @@ public void testPreProcess() throws Exception {
 
             RescoreContext rescoreContext = mock(RescoreContext.class);
             when(rescoreContext.getWindowSize()).thenReturn(500);
-            readerContext.addRescore(rescoreContext);
+            context1.addRescore(rescoreContext);
 
             exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Cannot use [sort] option in conjunction with [rescore]."));
@@ -166,7 +167,7 @@ public void testPreProcess() throws Exception {
                 + "] index level setting."));
 
             readerContext.close();
-            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher);
             // rescore is null but sliceBuilder is not null
             DefaultSearchContext context2 = new DefaultSearchContext(readerContext, shardSearchRequest, target,
                 null, indexService, indexShard, bigArrays, null, timeout, null);
@@ -196,7 +197,7 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
             readerContext.close();
-            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher);
             DefaultSearchContext context4 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
                 indexService, indexShard, bigArrays, null, timeout, null);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index bce588e69adb0..bed5f018a579c 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -317,6 +317,7 @@ public void onFailure(Exception e) {
                             new ShardSearchRequest(OriginalIndices.NONE, useScroll ? scrollSearchRequest : searchRequest,
                                 indexShard.shardId(), 1,
                                 new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null),
+                            true,
                             new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()), result);
                         SearchPhaseResult searchPhaseResult = result.get();
                         IntArrayList intCursors = new IntArrayList(1);
@@ -369,7 +370,7 @@ public void testTimeout() throws IOException {
             new AliasFilter(null, Strings.EMPTY_ARRAY),
             1.0f, -1, null, null);
 
-        try (ReaderContext reader = createReaderContext(indexShard, requestWithDefaultTimeout);
+        try (ReaderContext reader = createReaderContext(indexShard);
              SearchContext contextWithDefaultTimeout = service.createContext(reader, requestWithDefaultTimeout, null, randomBoolean())) {
             // the search context should inherit the default timeout
             assertThat(contextWithDefaultTimeout.timeout(), equalTo(TimeValue.timeValueSeconds(5)));
@@ -384,7 +385,7 @@ public void testTimeout() throws IOException {
             1,
             new AliasFilter(null, Strings.EMPTY_ARRAY),
             1.0f, -1, null, null);
-        try (ReaderContext reader = createReaderContext(indexShard, requestWithCustomTimeout);
+        try (ReaderContext reader = createReaderContext(indexShard);
              SearchContext context = service.createContext(reader, requestWithCustomTimeout, null, randomBoolean())) {
             // the search context should inherit the query timeout
             assertThat(context.timeout(), equalTo(TimeValue.timeValueSeconds(seconds)));
@@ -410,12 +411,12 @@ public void testMaxDocvalueFieldsSearch() throws IOException {
         }
         final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
             new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
-        try (ReaderContext reader = createReaderContext(indexShard, request);
+        try (ReaderContext reader = createReaderContext(indexShard);
              SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
             assertNotNull(context);
         }
         searchSourceBuilder.docValueField("one_field_too_much");
-        try (ReaderContext reader = createReaderContext(indexShard, request)) {
+        try (ReaderContext reader = createReaderContext(indexShard)) {
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
                 () -> service.createContext(reader, request, null, randomBoolean()));
             assertEquals(
@@ -447,7 +448,7 @@ public void testMaxScriptFieldsSearch() throws IOException {
         final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
             indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
 
-        try(ReaderContext reader = createReaderContext(indexShard, request)) {
+        try(ReaderContext reader = createReaderContext(indexShard)) {
             try (SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
                 assertNotNull(context);
             }
@@ -479,7 +480,7 @@ public void testIgnoreScriptfieldIfSizeZero() throws IOException {
         final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE,
             searchRequest, indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY),
             1.0f, -1, null, null);
-        try (ReaderContext reader = createReaderContext(indexShard, request);
+        try (ReaderContext reader = createReaderContext(indexShard);
              SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
             assertEquals(0, context.scriptFields().fields().size());
         }
@@ -517,7 +518,7 @@ public void testMaxOpenScrollContexts() throws Exception {
         SearchService.SearchRewriteContext rewriteContext =
             service.acquireSearcherAndRewrite(new ShardScrollRequestTest(indexShard.shardId()), indexShard);
         ElasticsearchException ex = expectThrows(ElasticsearchException.class,
-            () -> service.createAndPutReaderContext(rewriteContext));
+            () -> service.createAndPutReaderContext(rewriteContext, randomBoolean()));
         assertEquals(
             "Trying to create too many scroll contexts. Must be less than or equal to: [" +
                 SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY) + "]. " +
@@ -633,7 +634,7 @@ public void testCanMatch() throws Exception {
 
         CountDownLatch latch = new CountDownLatch(1);
         SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
-        service.executeQueryPhase(request, task, new ActionListener<SearchPhaseResult>() {
+        service.executeQueryPhase(request, randomBoolean(), task, new ActionListener<SearchPhaseResult>() {
             @Override
             public void onResponse(SearchPhaseResult searchPhaseResult) {
                 try {
@@ -775,7 +776,7 @@ public SearchType searchType() {
                 throw new NullPointerException("expected");
             }
         };
-        try (ReaderContext reader = createReaderContext(indexService.getShard(shardId.id()), request)) {
+        try (ReaderContext reader = createReaderContext(indexService.getShard(shardId.id()))) {
             NullPointerException e = expectThrows(NullPointerException.class,
                 () -> service.createContext(reader, request, null, randomBoolean()));
             assertEquals("expected", e.getMessage());
@@ -801,7 +802,7 @@ public void testMatchNoDocsEmptyResponse() throws InterruptedException {
         {
             CountDownLatch latch = new CountDownLatch(1);
             shardRequest.source().query(new MatchAllQueryBuilder());
-            service.executeQueryPhase(shardRequest, task, new ActionListener<>() {
+            service.executeQueryPhase(shardRequest, randomBoolean(), task, new ActionListener<>() {
                 @Override
                 public void onResponse(SearchPhaseResult result) {
                     try {
@@ -831,7 +832,7 @@ public void onFailure(Exception exc) {
         {
             CountDownLatch latch = new CountDownLatch(1);
             shardRequest.source().query(new MatchNoneQueryBuilder());
-            service.executeQueryPhase(shardRequest, task, new ActionListener<>() {
+            service.executeQueryPhase(shardRequest, randomBoolean(), task, new ActionListener<>() {
                 @Override
                 public void onResponse(SearchPhaseResult result) {
                     try {
@@ -861,7 +862,7 @@ public void onFailure(Exception exc) {
         {
             CountDownLatch latch = new CountDownLatch(1);
             shardRequest.canReturnNullResponseIfMatchNoDocs(true);
-            service.executeQueryPhase(shardRequest, task, new ActionListener<>() {
+            service.executeQueryPhase(shardRequest, randomBoolean(), task, new ActionListener<>() {
                 @Override
                 public void onResponse(SearchPhaseResult result) {
                     try {
@@ -918,8 +919,8 @@ public void testDeleteIndexWhileSearch() throws Exception {
         }
     }
 
-    private ReaderContext createReaderContext(IndexShard shard, ShardSearchRequest request) {
+    private ReaderContext createReaderContext(IndexShard shard) {
         Engine.Searcher searcher = shard.acquireSearcher("test");
-        return new ReaderContext(randomNonNegativeLong(), shard, searcher, request);
+        return new ReaderContext(randomNonNegativeLong(), shard, searcher);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
index c67850436b393..2b25567e1fc73 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
@@ -24,6 +24,8 @@
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.settings.Settings;
@@ -35,6 +37,8 @@
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalAggregationsTests;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.suggest.SuggestTests;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
@@ -54,7 +58,11 @@ public QuerySearchResultTests() {
 
     private static QuerySearchResult createTestInstance() throws Exception {
         ShardId shardId = new ShardId("index", "uuid", randomInt());
-        QuerySearchResult result = new QuerySearchResult(randomLong(), new SearchShardTarget("node", shardId, null, OriginalIndices.NONE));
+        SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean());
+        ShardSearchRequest shardSearchRequest = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shardId, 1,
+            new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, randomNonNegativeLong(), null, null);
+        QuerySearchResult result = new QuerySearchResult(
+            randomLong(), new SearchShardTarget("node", shardId, null, OriginalIndices.NONE), shardSearchRequest);
         if (randomBoolean()) {
             result.terminatedEarly(randomBoolean());
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index 5ae5d498d7975..e1235bc92a5e4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -599,4 +599,9 @@ public SearchShardTask getTask() {
     public boolean isCancelled() {
         return task.isCancelled();
     }
+
+    @Override
+    public void addRescore(RescoreContext rescore) {
+
+    }
 }

From 395e2a935001fcc13952b9d56b9e4789409b2bc1 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Fri, 13 Mar 2020 15:22:14 -0400
Subject: [PATCH 05/47] Adjust SearchService after merging from master

---
 .../action/search/DfsQueryPhase.java          |  4 --
 .../action/search/FetchSearchPhase.java       |  6 +-
 .../elasticsearch/search/SearchService.java   | 25 ++++---
 .../action/search/DfsQueryPhaseTests.java     | 58 +++++++++-------
 .../action/search/FetchSearchPhaseTests.java  | 58 ++++++++--------
 .../search/SearchPhaseControllerTests.java    | 55 ++++++++--------
 .../search/SearchServiceTests.java            | 66 +++++++++++--------
 .../search/query/QuerySearchResultTests.java  |  8 ++-
 8 files changed, 153 insertions(+), 127 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
index fd92a67e818cc..67d0e5e875d1f 100644
--- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
@@ -40,7 +40,6 @@
  */
 final class DfsQueryPhase extends SearchPhase {
     private final ArraySearchPhaseResults<SearchPhaseResult> queryResult;
-    private final SearchPhaseController searchPhaseController;
     private final List<DfsSearchResult> searchResults;
     private final AggregatedDfs dfs;
     private final Function<ArraySearchPhaseResults<SearchPhaseResult>, SearchPhase> nextPhaseFactory;
@@ -56,7 +55,6 @@ final class DfsQueryPhase extends SearchPhase {
         super("dfs_query");
         this.progressListener = context.getTask().getProgressListener();
         this.queryResult = searchPhaseController.newSearchPhaseResults(progressListener, context.getRequest(), context.getNumShards());
-        this.searchPhaseController = searchPhaseController;
         this.searchResults = searchResults;
         this.dfs = dfs;
         this.nextPhaseFactory = nextPhaseFactory;
@@ -71,8 +69,6 @@ public void run() throws IOException {
         final CountedCollector<SearchPhaseResult> counter = new CountedCollector<>(queryResult::consumeResult,
             searchResults.size(),
             () -> context.executeNextPhase(this, nextPhaseFactory.apply(queryResult)), context);
-        final SearchSourceBuilder sourceBuilder = context.getRequest().source();
-        progressListener.notifyListShards(progressListener.searchShards(searchResults), sourceBuilder == null || sourceBuilder.size() != 0);
         for (final DfsSearchResult dfsResult : searchResults) {
             final SearchShardTarget searchShardTarget = dfsResult.getSearchShardTarget();
             Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 2f6ae0ff4db48..18433cd2590f8 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -175,9 +175,9 @@ private void innerRun() throws IOException {
         }
     }
 
-    protected ShardFetchSearchRequest createFetchRequest(SearchContextId contextId, int index, IntArrayList entry, ScoreDoc[] lastEmittedDocPerShard,
-                                                         OriginalIndices originalIndices, ShardSearchRequest shardSearchRequest,
-                                                         RescoreDocIds rescoreDocIds) {
+    protected ShardFetchSearchRequest createFetchRequest(SearchContextId contextId, int index, IntArrayList entry,
+                                                         ScoreDoc[] lastEmittedDocPerShard, OriginalIndices originalIndices,
+                                                         ShardSearchRequest shardSearchRequest, RescoreDocIds rescoreDocIds) {
         final ScoreDoc lastEmittedDoc = (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[index] : null;
         return new ShardFetchSearchRequest(originalIndices, contextId, shardSearchRequest, entry, lastEmittedDoc,
             rescoreDocIds, aggregatedDfs);
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index b918020533ef9..e3a895aabe8c5 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -33,11 +33,11 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedSupplier;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Setting;
@@ -123,7 +123,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.LongSupplier;
-import java.util.function.Supplier;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueHours;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
@@ -596,29 +595,35 @@ final ReaderContext createAndPutReaderContext(SearchRewriteContext rewriteContex
         final SearchOperationListener searchOperationListener = rewriteContext.shard.getSearchOperationListener();
         Engine.Searcher engineSearcher = rewriteContext.searcher;
         ReaderContext readerContext = null;
+        Releasable decreaseScrollContexts = null;
         try {
-            if (request.scroll() != null && openScrollContexts.get() >= maxOpenScrollContext) {
-                throw new ElasticsearchException(
-                    "Trying to create too many scroll contexts. Must be less than or equal to: [" +
-                        maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
-                        + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
+            if (request.scroll() != null) {
+                decreaseScrollContexts = openScrollContexts::decrementAndGet;
+                if (openScrollContexts.incrementAndGet() > maxOpenScrollContext) {
+                    throw new ElasticsearchException(
+                        "Trying to create too many scroll contexts. Must be less than or equal to: [" +
+                            maxOpenScrollContext + "]. " + "This limit can be set by changing the ["
+                            + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
+                }
             }
             if (keepStatesInContext || request.scroll() != null) {
                 readerContext = new LegacyReaderContext(idGenerator.incrementAndGet(), rewriteContext.shard, engineSearcher, request);
             } else {
                 readerContext = new ReaderContext(idGenerator.incrementAndGet(), rewriteContext.shard, engineSearcher);
             }
+            if (request.scroll() != null) {
+                readerContext.addOnClose(decreaseScrollContexts);
+                decreaseScrollContexts = null;
+            }
             engineSearcher = null;
             final ReaderContext finalReaderContext = readerContext;
             searchOperationListener.onNewReaderContext(finalReaderContext);
             if (finalReaderContext.scrollContext() != null) {
                 searchOperationListener.onNewScrollContext(finalReaderContext.scrollContext());
-                openScrollContexts.incrementAndGet();
             }
             readerContext.addOnClose(() -> {
                 try {
                     if (finalReaderContext.scrollContext() != null) {
-                        openScrollContexts.decrementAndGet();
                         searchOperationListener.onFreeScrollContext(finalReaderContext.scrollContext());
                     }
                 } finally {
@@ -629,7 +634,7 @@ final ReaderContext createAndPutReaderContext(SearchRewriteContext rewriteContex
             readerContext = null;
             return finalReaderContext;
         } finally {
-            Releasables.close(engineSearcher, readerContext);
+            Releasables.close(engineSearcher, readerContext, decreaseScrollContexts);
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index e791389413c68..5312c48ba334b 100644
--- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.dfs.DfsSearchResult;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.test.ESTestCase;
@@ -45,8 +46,8 @@
 
 public class DfsQueryPhaseTests extends ESTestCase {
 
-    private static DfsSearchResult newSearchResult(int shardIndex, long requestId, SearchShardTarget target) {
-        DfsSearchResult result = new DfsSearchResult(requestId, target, null);
+    private static DfsSearchResult newSearchResult(int shardIndex, SearchContextId contextId, SearchShardTarget target) {
+        DfsSearchResult result = new DfsSearchResult(contextId, target, null);
         result.setShardIndex(shardIndex);
         return result;
     }
@@ -54,8 +55,10 @@ private static DfsSearchResult newSearchResult(int shardIndex, long requestId, S
     public void testDfsWith2Shards() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(0, newSearchResult(0, new SearchContextId("", 1),
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(1, newSearchResult(1, new SearchContextId("", 2),
+            new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
@@ -65,24 +68,24 @@ public void testDfsWith2Shards() throws IOException {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
-                if (request.id() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE), null);
+                if (request.contextId().getId() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
+                        new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                                     new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
-                } else if (request.id() == 2) {
-                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node2", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE), null);
+                } else if (request.contextId().getId() == 2) {
+                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
+                        new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F),
                             new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
                 } else {
-                    fail("no such request ID: " + request.id());
+                    fail("no such request ID: " + request.contextId());
                 }
             }
         };
@@ -114,8 +117,10 @@ public void run() throws IOException {
     public void testDfsWith1ShardFailed() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(0, newSearchResult(0, new SearchContextId("", 1),
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(1, newSearchResult(1, new SearchContextId("", 2),
+            new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
@@ -125,18 +130,19 @@ public void testDfsWith1ShardFailed() throws IOException {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
-                if (request.id() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
+                if (request.contextId().getId() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
+                        new SearchShardTarget("node1", new ShardId("test", "na", 0),
                         null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(
                             new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                             new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
-                } else if (request.id() == 2) {
+                } else if (request.contextId().getId() == 2) {
                     listener.onFailure(new MockDirectoryWrapper.FakeIOException());
                 } else {
-                    fail("no such request ID: " + request.id());
+                    fail("no such request ID: " + request.contextId());
                 }
             }
         };
@@ -163,7 +169,7 @@ public void run() throws IOException {
         assertEquals(1, mockSearchPhaseContext.failures.size());
         assertTrue(mockSearchPhaseContext.failures.get(0).getCause() instanceof MockDirectoryWrapper.FakeIOException);
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
-        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(2L));
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(new SearchContextId("", 2L)));
         assertNull(responseRef.get().get(1));
     }
 
@@ -171,8 +177,10 @@ public void run() throws IOException {
     public void testFailPhaseOnException() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(0, newSearchResult(0, new SearchContextId("", 1),
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
+        results.set(1, newSearchResult(1, new SearchContextId("", 2),
+            new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
@@ -182,18 +190,18 @@ public void testFailPhaseOnException() throws IOException {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
-                if (request.id() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0),
-                        null, OriginalIndices.NONE), null);
+                if (request.contextId().getId() == 1) {
+                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
+                        new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                                     new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
-                } else if (request.id() == 2) {
+                } else if (request.contextId().getId() == 2) {
                    throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException());
                 } else {
-                    fail("no such request ID: " + request.id());
+                    fail("no such request ID: " + request.contextId());
                 }
             }
         };
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index 8978f43842cfe..fd38a36d015e8 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -73,7 +73,7 @@ public void testShortcutQueryAndFetchOptimization() {
             numHits = 0;
         }
 
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
             @Override
             public void run() {
@@ -98,18 +98,18 @@ public void testFetchTwoDocument() {
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        final SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 123);
-        QuerySearchResult queryResult = new QuerySearchResult(ctx1,
-            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+        SearchContextId ctx1 = new SearchContextId(UUIDs.base64UUID(), 123);
+        QuerySearchResult queryResult = new QuerySearchResult(ctx1, new SearchShardTarget("node1", new ShardId("test", "na", 0),
+            null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        final SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 312);
-        queryResult = new QuerySearchResult(ctx2,
-            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        final SearchContextId ctx2 = new SearchContextId(UUIDs.base64UUID(), 321);
+        queryResult = new QuerySearchResult(
+            ctx2, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -132,7 +132,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -159,18 +159,17 @@ public void testFailFetchOneDoc() {
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 123);
-        QuerySearchResult queryResult = new QuerySearchResult(ctx1,
-            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+        final SearchContextId ctx = new SearchContextId(UUIDs.base64UUID(), 123);
+        QuerySearchResult queryResult = new QuerySearchResult(ctx,
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 321);
-        queryResult = new QuerySearchResult(ctx2,
-            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+        queryResult = new QuerySearchResult(new SearchContextId("", 321),
+            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -192,7 +191,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
 
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -211,7 +210,7 @@ public void run() {
         assertEquals(1, searchResponse.getShardFailures().length);
         assertTrue(searchResponse.getShardFailures()[0].getCause() instanceof MockDirectoryWrapper.FakeIOException);
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
-        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(ctx1));
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(ctx));
     }
 
     public void testFetchDocsConcurrently() throws InterruptedException {
@@ -225,7 +224,7 @@ public void testFetchDocsConcurrently() throws InterruptedException {
             mockSearchPhaseContext.getRequest(), numHits);
         for (int i = 0; i < numHits; i++) {
             QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", i),
-                new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+                new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
             queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                     new ScoreDoc[] {new ScoreDoc(i+1, i)}), i), new DocValueFormat[0]);
             queryResult.size(resultSetSize); // the size of the result set
@@ -238,14 +237,14 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                                          SearchActionListener<FetchSearchResult> listener) {
                 new Thread(() -> {
                     FetchSearchResult fetchResult = new FetchSearchResult();
-                    fetchResult.hits(new SearchHits(new SearchHit[]{new SearchHit((int) (request.contextId().getId() + 1))},
+                    fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit((int) (request.contextId().getId()+1))},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 100F));
                     listener.onResponse(fetchResult);
                 }).start();
             }
         };
         CountDownLatch latch = new CountDownLatch(1);
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -282,7 +281,8 @@ public void testExceptionFailsPhase() {
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
         QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
-            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+            new SearchShardTarget("node1", new ShardId("test", "na", 0),
+            null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
@@ -290,7 +290,7 @@ public void testExceptionFailsPhase() {
         results.consumeResult(queryResult);
 
         queryResult = new QuerySearchResult(new SearchContextId("", 321),
-            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -309,14 +309,14 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 2.0F));
                 } else {
-                    assertEquals(request.contextId().getId(), 123);
+                    assertEquals(request, 123);
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(42)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F));
                 }
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -338,18 +338,18 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = 1;
-        SearchContextId ctx1 = new SearchContextId(UUIDs.randomBase64UUID(), 123);
+        final SearchContextId ctx1 = new SearchContextId(UUIDs.base64UUID(), 123);
         QuerySearchResult queryResult = new QuerySearchResult(ctx1,
-            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE));
+            new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize); // the size of the result set
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        SearchContextId ctx2 = new SearchContextId(UUIDs.randomBase64UUID(), 321);
+        final SearchContextId ctx2 = new SearchContextId(UUIDs.base64UUID(), 321);
         queryResult = new QuerySearchResult(ctx2,
-            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE));
+            new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
         queryResult.size(resultSetSize);
@@ -361,7 +361,7 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
                 FetchSearchResult fetchResult = new FetchSearchResult();
-                if (request.contextId().equals(ctx2)) {
+                if (request.contextId().getId() == 321) {
                     fetchResult.hits(new SearchHits(new SearchHit[] {new SearchHit(84)},
                         new TotalHits(1, TotalHits.Relation.EQUAL_TO), 2.0F));
                 } else {
@@ -370,7 +370,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index 256340ef9b1d4..012130a6c6978 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -50,6 +50,7 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.Suggest;
@@ -230,7 +231,7 @@ private static AtomicArray<SearchPhaseResult> generateQueryResults(int nShards,
             String clusterAlias = randomBoolean() ? null : "remote";
             SearchShardTarget searchShardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex),
                 clusterAlias, OriginalIndices.NONE);
-            QuerySearchResult querySearchResult = new QuerySearchResult(shardIndex, searchShardTarget, null);
+            QuerySearchResult querySearchResult = new QuerySearchResult(new SearchContextId("", shardIndex), searchShardTarget, null);
             final TopDocs topDocs;
             float maxScore = 0;
             if (searchHitsSize == 0) {
@@ -302,7 +303,7 @@ private static AtomicArray<SearchPhaseResult> generateFetchResults(int nShards,
             float maxScore = -1F;
             String clusterAlias = randomBoolean() ? null : "remote";
             SearchShardTarget shardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex), clusterAlias, OriginalIndices.NONE);
-            FetchSearchResult fetchSearchResult = new FetchSearchResult(shardIndex, shardTarget);
+            FetchSearchResult fetchSearchResult = new FetchSearchResult(new SearchContextId("", shardIndex), shardTarget);
             List<SearchHit> searchHits = new ArrayList<>();
             for (ScoreDoc scoreDoc : mergedSearchDocs) {
                 if (scoreDoc.shardIndex == shardIndex) {
@@ -365,8 +366,8 @@ private void consumerTestCase(int numEmptyResponses) {
             numEmptyResponses --;
         }
 
-        QuerySearchResult result = new QuerySearchResult(0, new SearchShardTarget("node", new ShardId("a", "b", 0),
-            null, OriginalIndices.NONE), null);
+        QuerySearchResult result = new QuerySearchResult(new SearchContextId("", 0),
+            new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), null);
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         InternalAggregations aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 1.0D, DocValueFormat.RAW,
@@ -375,7 +376,8 @@ private void consumerTestCase(int numEmptyResponses) {
         result.setShardIndex(0);
         consumer.consumeResult(result);
 
-        result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), null);
+        result = new QuerySearchResult(new SearchContextId("", 1),
+            new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), null);
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 3.0D, DocValueFormat.RAW,
@@ -384,7 +386,8 @@ private void consumerTestCase(int numEmptyResponses) {
         result.setShardIndex(2);
         consumer.consumeResult(result);
 
-        result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), null);
+        result = new QuerySearchResult(new SearchContextId("", 1),
+            new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), null);
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
         aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 2.0D, DocValueFormat.RAW,
@@ -452,8 +455,8 @@ public void testConsumerConcurrently() throws InterruptedException {
             threads[i] = new Thread(() -> {
                 int number = randomIntBetween(1, 1000);
                 max.updateAndGet(prev -> Math.max(prev, number));
-                QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new ShardId("a", "b", id),
-                    null, OriginalIndices.NONE), null);
+                QuerySearchResult result = new QuerySearchResult(new SearchContextId("", id),
+                    new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE), null);
                 result.topDocs(new TopDocsAndMaxScore(
                     new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(0, number)}), number),
                     new DocValueFormat[0]);
@@ -496,8 +499,8 @@ public void testConsumerOnlyAggs() {
         for (int i = 0; i < expectedNumResults; i++) {
             int number = randomIntBetween(1, 1000);
             max.updateAndGet(prev -> Math.max(prev, number));
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE), null);
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), number),
                     new DocValueFormat[0]);
             InternalAggregations aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", (double) number,
@@ -534,8 +537,8 @@ public void testConsumerOnlyHits() {
         for (int i = 0; i < expectedNumResults; i++) {
             int number = randomIntBetween(1, 1000);
             max.updateAndGet(prev -> Math.max(prev, number));
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE), null);
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                     new ScoreDoc[] {new ScoreDoc(0, number)}), number), new DocValueFormat[0]);
             result.setShardIndex(i);
@@ -601,8 +604,8 @@ public void testReduceTopNWithFromOffset() {
             searchPhaseController.newSearchPhaseResults(NOOP, request, 4);
         int score = 100;
         for (int i = 0; i < 4; i++) {
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE), null);
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             ScoreDoc[] docs = new ScoreDoc[3];
             for (int j = 0; j < docs.length; j++) {
                 docs[j] = new ScoreDoc(0, score--);
@@ -643,8 +646,8 @@ public void testConsumerSortByField() {
             max.updateAndGet(prev -> Math.max(prev, number));
             FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, new Object[]{number})};
             TopDocs topDocs = new TopFieldDocs(new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields);
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE), null);
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats);
             result.setShardIndex(i);
             result.size(size);
@@ -681,8 +684,8 @@ public void testConsumerFieldCollapsing() {
             Object[] values = {randomFrom(collapseValues)};
             FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, values)};
             TopDocs topDocs = new CollapseTopFieldDocs("field", new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields, values);
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE), null);
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats);
             result.setShardIndex(i);
             result.size(size);
@@ -714,8 +717,8 @@ public void testConsumerSuggestions() {
         int maxScorePhrase = -1;
         int maxScoreCompletion = -1;
         for (int i = 0; i < expectedNumResults; i++) {
-            QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i),
-                null, OriginalIndices.NONE), null);
+            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+                new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions =
                 new ArrayList<>();
             {
@@ -814,18 +817,18 @@ public void onQueryResult(int shardIndex) {
                 }
 
                 @Override
-                public void onQueryFailure(int shardIndex, Exception exc) {
+                public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
                     assertThat(shardIndex, lessThan(expectedNumResults));
                     numQueryFailureListener.incrementAndGet();
                 }
 
                 @Override
-                public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int version) {
-                    assertEquals(numReduceListener.incrementAndGet(), version);
+                public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
+                    assertEquals(numReduceListener.incrementAndGet(), reducePhase);
                 }
 
                 @Override
-                public void onReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs) {
+                public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
                     totalHitsListener.set(totalHits);
                     finalAggsListener.set(aggs);
                     numReduceListener.incrementAndGet();
@@ -840,8 +843,8 @@ public void onReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggr
                 threads[i] = new Thread(() -> {
                     int number = randomIntBetween(1, 1000);
                     max.updateAndGet(prev -> Math.max(prev, number));
-                    QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new ShardId("a", "b", id),
-                        null, OriginalIndices.NONE), null);
+                    QuerySearchResult result = new QuerySearchResult(new SearchContextId("", id),
+                        new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE), null);
                     result.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[]{new ScoreDoc(0, number)}), number),
                         new DocValueFormat[0]);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 64129ca7fc79e..376f03b9f08dc 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -85,9 +85,11 @@
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -330,7 +332,7 @@ public void onFailure(Exception e) {
                         service.executeFetchPhase(req, new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()), listener);
                         listener.get();
                         if (useScroll) {
-                            service.freeContext(searchPhaseResult.getContextId());
+                            service.freeReaderContext(searchPhaseResult.getContextId());
                         }
                     } catch (ExecutionException ex) {
                         assertThat(ex.getCause(), instanceOf(RuntimeException.class));
@@ -547,7 +549,7 @@ public void testOpenScrollContextsConcurrently() throws Exception {
                         SearchService.SearchRewriteContext rewriteContext =
                             searchService.acquireSearcherAndRewrite(new ShardScrollRequestTest(indexShard.shardId()), indexShard);
                         try {
-                            searchService.createAndPutContext(rewriteContext);
+                            searchService.createAndPutReaderContext(rewriteContext, false);
                         } catch (ElasticsearchException e) {
                             assertThat(e.getMessage(), equalTo(
                                 "Trying to create too many scroll contexts. Must be less than or equal to: " +
@@ -969,34 +971,44 @@ public void testLookUpSearchContext() throws Exception {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
         IndexShard indexShard = indexService.getShard(0);
-        ShardSearchRequest shardSearchRequest = new ShardSearchRequest(
-            OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
-            indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
         List<SearchContextId> contextIds = new ArrayList<>();
         int numContexts = randomIntBetween(1, 10);
-        for (int i = 0; i < numContexts; i++) {
-            SearchService.SearchRewriteContext rewriteContext = searchService.acquireSearcherAndRewrite(shardSearchRequest, indexShard);
-            final SearchContext searchContext = searchService.createContext(rewriteContext);
-            assertThat(searchContext.id().getId(), equalTo((long) (i + 1)));
-            searchService.putContext(searchContext);
-            contextIds.add(searchContext.id());
-        }
-        assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
-        while (contextIds.isEmpty() == false) {
-            final SearchContextId contextId = randomFrom(contextIds);
-            assertFalse(searchService.freeContext(new SearchContextId(UUIDs.randomBase64UUID(), contextId.getId())));
-            assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
-            if (randomBoolean()) {
-                assertTrue(searchService.freeContext(contextId));
-            } else {
-                assertTrue(searchService.freeContext((new SearchContextId("", contextId.getId()))));
+        CountDownLatch latch = new CountDownLatch(1);
+        indexShard.getThreadPool().executor(ThreadPool.Names.SEARCH).execute(() -> {
+            try {
+                for (int i = 0; i < numContexts; i++) {
+                    ShardSearchRequest request = new ShardSearchRequest(
+                        OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
+                        indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
+                    SearchService.SearchRewriteContext rewriteContext = null;
+                    rewriteContext = searchService.acquireSearcherAndRewrite(request, indexShard);
+                    final ReaderContext reader = searchService.createAndPutReaderContext(rewriteContext, randomBoolean());
+                    assertThat(reader.id().getId(), equalTo((long) (i + 1)));
+                    contextIds.add(reader.id());
+                }
+                assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
+                while (contextIds.isEmpty() == false) {
+                    final SearchContextId contextId = randomFrom(contextIds);
+                    assertFalse(searchService.freeReaderContext(new SearchContextId(UUIDs.randomBase64UUID(), contextId.getId())));
+                    assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
+                    if (randomBoolean()) {
+                        assertTrue(searchService.freeReaderContext(contextId));
+                    } else {
+                        assertTrue(searchService.freeReaderContext((new SearchContextId("", contextId.getId()))));
+                    }
+                    contextIds.remove(contextId);
+                    assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
+                    assertFalse(searchService.freeReaderContext(new SearchContextId("", contextId.getId())));
+                    assertFalse(searchService.freeReaderContext(contextId));
+                    assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
+                }
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            } finally {
+                latch.countDown();
             }
-            contextIds.remove(contextId);
-            assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
-            assertFalse(searchService.freeContext(new SearchContextId("", contextId.getId())));
-            assertFalse(searchService.freeContext(contextId));
-            assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
-        }
+        });
+        latch.await();
     }
 
     private ReaderContext createReaderContext(IndexShard shard) {
diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
index 2b25567e1fc73..a0c3fcd89afb7 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.settings.Settings;
@@ -38,6 +39,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregationsTests;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
 import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.suggest.SuggestTests;
 import org.elasticsearch.test.ESTestCase;
@@ -61,8 +63,8 @@ private static QuerySearchResult createTestInstance() throws Exception {
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean());
         ShardSearchRequest shardSearchRequest = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shardId, 1,
             new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, randomNonNegativeLong(), null, null);
-        QuerySearchResult result = new QuerySearchResult(
-            randomLong(), new SearchShardTarget("node", shardId, null, OriginalIndices.NONE), shardSearchRequest);
+        QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.base64UUID(), randomLong()),
+            new SearchShardTarget("node", shardId, null, OriginalIndices.NONE), shardSearchRequest);
         if (randomBoolean()) {
             result.terminatedEarly(randomBoolean());
         }
@@ -83,7 +85,7 @@ public void testSerialization() throws Exception {
         QuerySearchResult querySearchResult = createTestInstance();
         Version version = VersionUtils.randomVersion(random());
         QuerySearchResult deserialized = copyWriteable(querySearchResult, namedWriteableRegistry, QuerySearchResult::new, version);
-        assertEquals(querySearchResult.getRequestId(), deserialized.getRequestId());
+        assertEquals(querySearchResult.getContextId().getId(), deserialized.getContextId().getId());
         assertNull(deserialized.getSearchShardTarget());
         assertEquals(querySearchResult.topDocs().maxScore, deserialized.topDocs().maxScore, 0f);
         assertEquals(querySearchResult.topDocs().topDocs.totalHits, deserialized.topDocs().topDocs.totalHits);

From 891336950fee3d1fccd45e0d331ea32f7de99382 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 26 Mar 2020 07:42:29 -0400
Subject: [PATCH 06/47] Allow searches with specific reader contexts (#53989)

---
 .ci/bwcVersions                               |   1 +
 .gitignore                                    |   1 +
 .../runConfigurations/Debug_Elasticsearch.xml |  11 +
 CONTRIBUTING.md                               |   8 +-
 build.gradle                                  |   6 +-
 .../elasticsearch/gradle/BuildPlugin.groovy   |  12 +-
 .../gradle/precommit/PrecommitTasks.groovy    |  17 +-
 .../test/StandaloneRestTestPlugin.groovy      |  13 +-
 .../gradle/DistributionDownloadPlugin.java    |   6 +-
 .../gradle/JdkDownloadPlugin.java             |   4 +-
 .../info/GenerateGlobalBuildInfoTask.java     | 293 -------
 .../gradle/info/GlobalBuildInfoPlugin.java    | 229 ++++--
 .../gradle/info/GlobalInfoExtension.java      |  12 -
 .../gradle/info/PrintGlobalBuildInfoTask.java |  75 --
 .../gradle/precommit/FilePermissionsTask.java |   4 +-
 .../precommit/TestingConventionsTasks.java    |  10 +-
 .../gradle/test/DistroTestPlugin.java         |  20 +-
 .../gradle/test/rest/CopyRestApiTask.java     |   4 +-
 .../gradle/test/rest/CopyRestTestsTask.java   |   4 +-
 .../testclusters/RestTestRunnerTask.java      |   4 +-
 .../testclusters/TestClustersPlugin.java      |   6 +-
 .../testfixtures/TestFixturesPlugin.java      |   4 +-
 .../GradleUtils.java}                         |   4 +-
 .../elasticsearch/gradle/{ => util}/Util.java |  27 +-
 .../gradle/vagrant/VagrantMachine.java        |   2 +-
 .../src/main/resources/minimumGradleVersion   |   2 +-
 .../gradle/info/BuildParams.java              |  38 +-
 .../elasticsearch/gradle/info/JavaHome.java   |  18 +-
 .../testKit/jdk-download/reuse/build.gradle   |   2 +-
 buildSrc/version.properties                   |   2 +-
 .../search/TransportNoopSearchAction.java     |   2 +-
 .../client/AsyncSearchClient.java             |  58 +-
 .../client/AsyncSearchRequestConverters.java  |  37 +-
 .../client/MLRequestConverters.java           |  12 +
 .../client/MachineLearningClient.java         |  44 +
 .../client/RequestConverters.java             |   4 +-
 .../asyncsearch/DeleteAsyncSearchRequest.java |  55 ++
 .../asyncsearch/GetAsyncSearchRequest.java    |  93 +++
 .../asyncsearch/SubmitAsyncSearchRequest.java |  26 +-
 .../client/core/IndexerJobStats.java          |  33 +-
 .../client/ml/EstimateModelMemoryRequest.java | 110 +++
 .../ml/EstimateModelMemoryResponse.java       |  80 ++
 .../ml/dataframe/DataFrameAnalyticsStats.java |  27 +-
 .../ml/dataframe/stats/common/DataCounts.java | 119 +++
 .../OutlierDetectionStats.java                |   2 +-
 .../client/rollup/GetRollupJobResponse.java   |  10 +-
 .../transforms/TransformIndexerStats.java     | 119 ++-
 .../AsyncSearchRequestConvertersTests.java    |  47 +-
 .../client/MLRequestConvertersTests.java      |  21 +
 .../client/MachineLearningIT.java             |  33 +-
 .../client/RequestConvertersTests.java        |   4 +-
 .../client/RestHighLevelClientTests.java      |   5 +-
 .../org/elasticsearch/client/SnapshotIT.java  |  25 +-
 .../org/elasticsearch/client/TransformIT.java |   2 +
 .../client/asyncsearch/AsyncSearchIT.java     |  48 +-
 .../asyncsearch/AsyncSearchResponseTests.java |   2 +-
 .../GetAsyncSearchRequestTests.java           |  41 +
 .../MlClientDocumentationIT.java              |  61 ++
 .../DataFrameAnalyticsStatsTests.java         |   5 +
 .../stats/common/DataCountsTests.java         |  51 ++
 .../rollup/GetRollupJobResponseTests.java     |   7 +-
 .../TransformIndexerStatsTests.java           | 118 ++-
 .../hlrc/TransformIndexerStatsTests.java      |  36 +-
 .../transforms/hlrc/TransformStatsTests.java  |  70 +-
 distribution/archives/build.gradle            |   4 +-
 .../aarch64-docker-build-context/build.gradle |  11 +
 .../docker/aarch64-docker-export/build.gradle |   2 +
 .../build.gradle                              |  11 +
 .../aarch64-oss-docker-export/build.gradle    |   2 +
 distribution/docker/build.gradle              |  78 +-
 .../docker/docker-build-context/build.gradle  |   2 +-
 .../oss-docker-build-context/build.gradle     |   2 +-
 distribution/docker/src/docker/Dockerfile     |   8 +-
 .../src/docker/config/tini-arm64.sha512       |   1 +
 .../resources/rest-api-spec/test/11_nodes.yml |   4 +-
 .../settings/AddFileKeyStoreCommand.java      |  54 +-
 .../settings/AddStringKeyStoreCommand.java    |  88 +-
 .../settings/AddFileKeyStoreCommandTests.java |  36 +-
 .../AddStringKeyStoreCommandTests.java        |  36 +-
 .../ml/estimate-model-memory.asciidoc         |  42 +
 .../high-level/supported-apis.asciidoc        |   2 +
 docs/plugins/analysis-nori.asciidoc           | 109 +++
 .../metrics/top-metrics-aggregation.asciidoc  |   2 +-
 .../keyword-marker-tokenfilter.asciidoc       | 418 ++++++++--
 .../uppercase-tokenfilter.asciidoc            |   3 +-
 .../word-delimiter-graph-tokenfilter.asciidoc |   2 +-
 .../apis/get-autoscaling-decision.asciidoc    |   2 +-
 docs/reference/cat/transforms.asciidoc        | 171 +++-
 docs/reference/cluster/nodes-stats.asciidoc   | 218 ++++-
 docs/reference/commands/keystore.asciidoc     |  59 +-
 docs/reference/eql/functions.asciidoc         |  66 ++
 docs/reference/eql/index.asciidoc             |   2 +
 docs/reference/eql/limitations.asciidoc       |   5 +-
 docs/reference/eql/syntax.asciidoc            |  12 +
 docs/reference/frozen-indices.asciidoc        |  13 +-
 .../index-modules/similarity.asciidoc         |   2 +-
 .../ingest/processors/inference.asciidoc      |  69 +-
 docs/reference/mapping.asciidoc               |  19 +-
 docs/reference/mapping/fields.asciidoc        |   2 +-
 .../mapping/fields/routing-field.asciidoc     |   4 +-
 .../mapping/fields/type-field.asciidoc        |   6 +-
 .../mapping/types/date_nanos.asciidoc         |   4 +-
 .../reference/mapping/types/wildcard.asciidoc |  17 +
 .../migration/migrate_8_0/settings.asciidoc   |   7 +
 .../apis/estimate-model-memory.asciidoc       |  87 ++
 .../ml/anomaly-detection/apis/ml-api.asciidoc |   2 +
 docs/reference/ml/ml-shared.asciidoc          |   7 +-
 docs/reference/modules/node.asciidoc          |  46 +-
 .../modules/remote-clusters.asciidoc          |   8 +-
 .../monitoring/how-monitoring-works.asciidoc  |   2 +-
 docs/reference/redirects.asciidoc             |   5 -
 docs/reference/rest-api/common-parms.asciidoc |  79 ++
 docs/reference/rollup/apis/get-job.asciidoc   |  14 +-
 docs/reference/search/async-search.asciidoc   |  24 +-
 docs/reference/search/multi-search.asciidoc   |  37 +-
 docs/reference/search/search.asciidoc         | 146 ++--
 .../apis/get-transform-stats.asciidoc         |  64 +-
 .../transform/apis/preview-transform.asciidoc |  56 +-
 docs/reference/transform/limitations.asciidoc |   7 +
 docs/reference/transform/overview.asciidoc    |  14 +-
 docs/reference/transform/setup.asciidoc       |  20 +-
 .../smoketest/DocsClientYamlTestSuiteIT.java  |  14 +-
 gradle/ide.gradle                             |  25 +-
 gradle/wrapper/gradle-wrapper.jar             | Bin 58695 -> 58694 bytes
 gradle/wrapper/gradle-wrapper.properties      |   4 +-
 .../elasticsearch/kibana/KibanaPlugin.java    | 145 ----
 .../kibana/KibanaPluginTests.java             |  46 --
 .../kibana/KibanaSystemIndexIT.java           | 249 ------
 ...ssions-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-expressions-8.5.0.jar.sha1         |   1 +
 .../expression/CountMethodValueSource.java    |  49 +-
 .../script/expression/DateField.java          |  10 +-
 .../expression/DateMethodValueSource.java     |  39 +-
 .../script/expression/DateObject.java         |  18 +-
 .../expression/DateObjectValueSource.java     |  37 +-
 .../expression/EmptyMemberValueSource.java    |  37 +-
 .../expression/ExpressionScriptEngine.java    |  30 +-
 .../FieldDataBasedDoubleValuesSource.java     |  53 ++
 .../expression/FieldDataValueSource.java      |  36 +-
 .../expression/GeoEmptyValueSource.java       |  43 +-
 .../script/expression/GeoField.java           |  14 +-
 .../expression/GeoLatitudeValueSource.java    |  40 +-
 .../expression/GeoLongitudeValueSource.java   |  37 +-
 .../script/expression/NumericField.java       |   6 +-
 .../ReplaceableConstDoubleValueSource.java    |   2 +-
 .../ReplaceableConstDoubleValues.java         |   6 +-
 .../MultiSearchTemplateResponseTests.java     |   4 +-
 .../mustache/SearchTemplateResponseTests.java |   4 +-
 .../org/elasticsearch/painless/Compiler.java  |   6 +-
 .../painless/node/AExpression.java            |  45 +-
 .../elasticsearch/painless/node/ANode.java    |   8 -
 .../painless/node/AStatement.java             |  19 +-
 .../painless/node/AStoreable.java             |  10 +-
 .../painless/node/EAssignment.java            |  97 ++-
 .../elasticsearch/painless/node/EBinary.java  |  76 +-
 .../elasticsearch/painless/node/EBool.java    |  34 +-
 .../elasticsearch/painless/node/EBoolean.java |  17 +-
 .../painless/node/ECallLocal.java             |  75 +-
 .../painless/node/ECapturingFunctionRef.java  |  27 +-
 .../elasticsearch/painless/node/EComp.java    |  49 +-
 .../painless/node/EConditional.java           |  45 +-
 .../painless/node/EConstant.java              |  16 +-
 .../elasticsearch/painless/node/EDecimal.java |  23 +-
 .../elasticsearch/painless/node/EElvis.java   |  38 +-
 .../painless/node/EExplicit.java              |  23 +-
 .../painless/node/EFunctionRef.java           |  25 +-
 .../painless/node/EInstanceof.java            |  36 +-
 .../elasticsearch/painless/node/ELambda.java  |  69 +-
 .../painless/node/EListInit.java              |  41 +-
 .../elasticsearch/painless/node/EMapInit.java |  62 +-
 .../painless/node/ENewArray.java              |  39 +-
 .../painless/node/ENewArrayFunctionRef.java   |  31 +-
 .../elasticsearch/painless/node/ENewObj.java  |  45 +-
 .../elasticsearch/painless/node/ENull.java    |  16 +-
 .../elasticsearch/painless/node/ENumeric.java |  25 +-
 .../elasticsearch/painless/node/ERegex.java   |  24 +-
 .../elasticsearch/painless/node/EStatic.java  |  18 +-
 .../elasticsearch/painless/node/EString.java  |  17 +-
 .../elasticsearch/painless/node/EUnary.java   |  44 +-
 .../painless/node/EVariable.java              |  27 +-
 .../elasticsearch/painless/node/PBrace.java   |  50 +-
 .../painless/node/PCallInvoke.java            |  42 +-
 .../elasticsearch/painless/node/PField.java   |  54 +-
 .../painless/node/PSubArrayLength.java        |  27 +-
 .../painless/node/PSubBrace.java              |  30 +-
 .../painless/node/PSubCallInvoke.java         |  37 +-
 .../painless/node/PSubDefArray.java           |  33 +-
 .../painless/node/PSubDefCall.java            |  45 +-
 .../painless/node/PSubDefField.java           |  23 +-
 .../painless/node/PSubField.java              |  23 +-
 .../painless/node/PSubListShortcut.java       |  40 +-
 .../painless/node/PSubMapShortcut.java        |  40 +-
 .../painless/node/PSubNullSafeCallInvoke.java |  23 +-
 .../painless/node/PSubNullSafeField.java      |  38 +-
 .../painless/node/PSubShortcut.java           |  29 +-
 .../elasticsearch/painless/node/SBlock.java   |  30 +-
 .../elasticsearch/painless/node/SBreak.java   |  17 +-
 .../elasticsearch/painless/node/SCatch.java   |  36 +-
 .../elasticsearch/painless/node/SClass.java   |  52 +-
 .../painless/node/SContinue.java              |  17 +-
 .../painless/node/SDeclBlock.java             |  28 +-
 .../painless/node/SDeclaration.java           |  34 +-
 .../org/elasticsearch/painless/node/SDo.java  |  38 +-
 .../elasticsearch/painless/node/SEach.java    |  61 +-
 .../painless/node/SExpression.java            |  36 +-
 .../elasticsearch/painless/node/SField.java   |  70 --
 .../org/elasticsearch/painless/node/SFor.java |  73 +-
 .../painless/node/SFunction.java              |  46 +-
 .../org/elasticsearch/painless/node/SIf.java  |  33 +-
 .../elasticsearch/painless/node/SIfElse.java  |  39 +-
 .../elasticsearch/painless/node/SReturn.java  |  28 +-
 .../painless/node/SSubEachArray.java          |  50 +-
 .../painless/node/SSubEachIterable.java       |  54 +-
 .../elasticsearch/painless/node/SThrow.java   |  26 +-
 .../org/elasticsearch/painless/node/STry.java |  38 +-
 .../elasticsearch/painless/node/SWhile.java   |  40 +-
 .../painless/DefOptimizationTests.java        |   1 +
 .../painless/node/NodeToStringTests.java      |  25 -
 .../AbstractAsyncBulkByScrollAction.java      |  20 +-
 .../reindex/AsyncBulkByScrollActionTests.java |  23 +-
 .../ClientScrollableHitSourceTests.java       |   2 +-
 .../tasksplugin/TasksPlugin.java              |   3 +-
 .../tasksplugin/TasksPluginTests.java         |   3 +-
 .../netty4/ByteBufBytesReference.java         | 104 ---
 .../transport/netty4/Netty4Utils.java         |  41 +-
 .../netty4/ByteBufBytesReferenceTests.java    |  84 --
 .../transport/netty4/Netty4UtilsTests.java    |   4 +-
 ...rs-icu-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-analyzers-icu-8.5.0.jar.sha1       |   1 +
 ...romoji-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-analyzers-kuromoji-8.5.0.jar.sha1  |   1 +
 ...s-nori-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-analyzers-nori-8.5.0.jar.sha1      |   1 +
 .../analysis/NoriNumberFilterFactory.java     |  38 +
 .../index/analysis/NoriTokenizerFactory.java  |   5 +-
 .../analysis/nori/AnalysisNoriPlugin.java     |   2 +
 .../analysis/AnalysisNoriFactoryTests.java    |   1 +
 .../index/analysis/NoriAnalysisTests.java     |  51 ++
 .../test/analysis_nori/10_basic.yml           |  17 +
 ...onetic-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-analyzers-phonetic-8.5.0.jar.sha1  |   1 +
 ...martcn-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-analyzers-smartcn-8.5.0.jar.sha1   |   1 +
 ...tempel-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-analyzers-stempel-8.5.0.jar.sha1   |   1 +
 ...ologik-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 ...lucene-analyzers-morfologik-8.5.0.jar.sha1 |   1 +
 plugins/ingest-attachment/build.gradle        |  14 +-
 .../licenses/SparseBitSet-1.2.jar.sha1        |   1 +
 .../licenses/SparseBitSet-LICENSE.txt         | 202 +++++
 .../licenses/SparseBitSet-NOTICE.txt          |   5 +
 .../licenses/commons-compress-1.18.jar.sha1   |   1 -
 .../licenses/commons-compress-1.19.jar.sha1   |   1 +
 .../licenses/commons-math3-3.6.1.jar.sha1     |   1 +
 .../licenses/commons-math3-LICENSE.txt        | 202 +++++
 .../licenses/commons-math3-NOTICE.txt         |   5 +
 .../licenses/fontbox-2.0.16.jar.sha1          |   1 -
 .../licenses/fontbox-2.0.19.jar.sha1          |   1 +
 .../licenses/pdfbox-2.0.16.jar.sha1           |   1 -
 .../licenses/pdfbox-2.0.19.jar.sha1           |   1 +
 .../licenses/poi-4.0.1.jar.sha1               |   1 -
 .../licenses/poi-4.1.2.jar.sha1               |   1 +
 .../licenses/poi-ooxml-4.0.1.jar.sha1         |   1 -
 .../licenses/poi-ooxml-4.1.2.jar.sha1         |   1 +
 .../licenses/poi-ooxml-schemas-4.0.1.jar.sha1 |   1 -
 .../licenses/poi-ooxml-schemas-4.1.2.jar.sha1 |   1 +
 .../licenses/poi-scratchpad-4.0.1.jar.sha1    |   1 -
 .../licenses/poi-scratchpad-4.1.2.jar.sha1    |   1 +
 .../licenses/tika-core-1.22.jar.sha1          |   1 -
 .../licenses/tika-core-1.24.jar.sha1          |   1 +
 .../licenses/tika-parsers-1.22.jar.sha1       |   1 -
 .../licenses/tika-parsers-1.24.jar.sha1       |   1 +
 .../qa/microsoft-azure-storage/build.gradle   |   2 +
 .../azure/AzureBlobContainer.java             |  41 +-
 .../repositories/azure/AzureBlobStore.java    |  20 +-
 .../repositories/azure/AzureRepository.java   |   2 +-
 .../azure/AzureRepositoryPlugin.java          |  15 +
 .../azure/AzureStorageService.java            |  98 ++-
 .../azure/AzureBlobContainerRetriesTests.java |   8 +-
 .../azure/AzureBlobStoreRepositoryTests.java  |   2 +-
 plugins/repository-gcs/build.gradle           |  18 +-
 .../licenses/gax-1.45.0.jar.sha1              |   1 -
 .../licenses/gax-1.54.0.jar.sha1              |   1 +
 .../google-api-client-1.30.8.jar.sha1         |   1 -
 .../google-api-client-1.30.9.jar.sha1         |   1 +
 ...ces-storage-v1-rev20191011-1.30.3.jar.sha1 |   1 -
 ...ces-storage-v1-rev20200226-1.30.9.jar.sha1 |   1 +
 .../google-cloud-core-1.92.5.jar.sha1         |   1 -
 .../google-cloud-core-1.93.3.jar.sha1         |   1 +
 .../google-cloud-core-http-1.77.0.jar.sha1    |   1 -
 .../google-cloud-core-http-1.93.3.jar.sha1    |   1 +
 .../google-cloud-storage-1.104.0.jar.sha1     |   1 -
 .../google-cloud-storage-1.106.0.jar.sha1     |   1 +
 ...ogle-http-client-appengine-1.29.2.jar.sha1 |   1 -
 ...ogle-http-client-appengine-1.34.2.jar.sha1 |   1 +
 ...oogle-http-client-jackson2-1.29.2.jar.sha1 |   1 -
 ...oogle-http-client-jackson2-1.34.2.jar.sha1 |   1 +
 .../licenses/threetenbp-1.3.3.jar.sha1        |   1 -
 .../licenses/threetenbp-1.4.1.jar.sha1        |   1 +
 ...eCloudStorageBlobStoreRepositoryTests.java |   2 -
 plugins/repository-hdfs/build.gradle          |   2 +-
 .../hadoop-annotations-2.8.1.jar.sha1         |   1 -
 .../hadoop-annotations-2.8.5.jar.sha1         |   1 +
 .../licenses/hadoop-auth-2.8.1.jar.sha1       |   1 -
 .../licenses/hadoop-auth-2.8.5.jar.sha1       |   1 +
 .../licenses/hadoop-client-2.8.1.jar.sha1     |   1 -
 .../licenses/hadoop-client-2.8.5.jar.sha1     |   1 +
 .../licenses/hadoop-common-2.8.1.jar.sha1     |   1 -
 .../licenses/hadoop-common-2.8.5.jar.sha1     |   1 +
 .../licenses/hadoop-hdfs-2.8.1.jar.sha1       |   1 -
 .../licenses/hadoop-hdfs-2.8.5.jar.sha1       |   1 +
 .../hadoop-hdfs-client-2.8.1.jar.sha1         |   1 -
 .../hadoop-hdfs-client-2.8.5.jar.sha1         |   1 +
 .../elasticsearch/http/nio/ByteBufUtils.java  | 280 +------
 ...rossClusterSearchUnavailableClusterIT.java |   2 +-
 qa/multi-cluster-search/build.gradle          |   4 +-
 .../org/elasticsearch/search/CCSDuelIT.java   |   4 +-
 .../test/multi_cluster/10_basic.yml           |  91 ++-
 .../test/remote_cluster/10_basic.yml          |  12 +-
 .../packaging/test/DockerTests.java           |   7 +
 .../rest-api-spec/api/clear_reader.json       |  23 +
 .../resources/rest-api-spec/api/msearch.json  |   3 +-
 .../resources/rest-api-spec/api/search.json   |   3 +-
 .../rest-api-spec/test/README.asciidoc        | 103 +++
 .../rest-api-spec/test/cat.nodes/10_basic.yml |   4 +-
 .../test/cat.shards/10_basic.yml              |   6 +-
 .../test/indices.data_stream/10_basic.yml     |  23 +-
 .../test/scroll/20_keep_alive.yml             |   4 +-
 .../test/search/330_reader_context.yml        | 122 +++
 ...common-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-analyzers-common-8.5.0.jar.sha1    |   1 +
 ...codecs-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-backward-codecs-8.5.0.jar.sha1     |   1 +
 ...e-core-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 server/licenses/lucene-core-8.5.0.jar.sha1    |   1 +
 ...ouping-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../licenses/lucene-grouping-8.5.0.jar.sha1   |   1 +
 ...ighter-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-highlighter-8.5.0.jar.sha1         |   1 +
 ...e-join-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 server/licenses/lucene-join-8.5.0.jar.sha1    |   1 +
 ...memory-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 server/licenses/lucene-memory-8.5.0.jar.sha1  |   1 +
 ...e-misc-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 server/licenses/lucene-misc-8.5.0.jar.sha1    |   1 +
 ...ueries-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 server/licenses/lucene-queries-8.5.0.jar.sha1 |   1 +
 ...parser-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-queryparser-8.5.0.jar.sha1         |   1 +
 ...andbox-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 server/licenses/lucene-sandbox-8.5.0.jar.sha1 |   1 +
 ...extras-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../lucene-spatial-extras-8.5.0.jar.sha1      |   1 +
 ...tial3d-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../licenses/lucene-spatial3d-8.5.0.jar.sha1  |   1 +
 ...uggest-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 server/licenses/lucene-suggest-8.5.0.jar.sha1 |   1 +
 .../elasticsearch/ElasticsearchException.java |   7 +-
 .../main/java/org/elasticsearch/Version.java  |   1 +
 .../elasticsearch/action/ActionModule.java    |  16 +-
 .../cluster/node/info/NodesInfoRequest.java   | 227 ++----
 .../node/info/NodesInfoRequestBuilder.java    |  29 +-
 .../node/info/TransportNodesInfoAction.java   |  15 +-
 .../datastream/CreateDataStreamAction.java    |  57 +-
 .../datastream/DeleteDataStreamAction.java    |  66 +-
 .../datastream/GetDataStreamsAction.java      |  32 +-
 .../TransportPutComponentTemplateAction.java  |  12 +-
 .../action/bulk/BulkProcessor.java            |   3 +-
 .../org/elasticsearch/action/bulk/Retry.java  |   3 +-
 .../action/bulk/TransportBulkAction.java      |  12 +-
 .../ingest/PutPipelineTransportAction.java    |   4 +-
 .../search/AbstractSearchAsyncAction.java     |  27 +-
 ...oller.java => ClearContextController.java} |  55 +-
 .../action/search/ClearReaderAction.java      |  32 +
 .../action/search/ClearReaderRequest.java     |  98 +++
 .../action/search/ClearReaderResponse.java    |  21 +-
 .../action/search/ExpandSearchPhase.java      |  12 +-
 .../action/search/FetchSearchPhase.java       |  35 +-
 .../action/search/ParsedScrollId.java         |   6 +-
 ...ollIdForNode.java => ReaderIdForNode.java} |   6 +-
 .../SearchDfsQueryThenFetchAsyncAction.java   |   2 +-
 .../action/search/SearchPhaseContext.java     |   7 +-
 .../action/search/SearchPhaseController.java  |  76 +-
 .../SearchQueryThenFetchAsyncAction.java      |   2 +-
 .../action/search/SearchRequest.java          |  59 +-
 .../action/search/SearchRequestBuilder.java   |  22 +-
 .../action/search/SearchResponse.java         |  33 +-
 .../action/search/SearchResponseMerger.java   |   3 +-
 .../search/SearchScrollAsyncAction.java       |  12 +-
 .../action/search/SearchShardIterator.java    |  28 +
 .../search/TransportClearReaderAction.java    |  49 ++
 .../search/TransportClearScrollAction.java    |   3 +-
 .../action/search/TransportSearchAction.java  | 176 +++-
 .../action/search/TransportSearchHelper.java  |  75 +-
 .../elasticsearch/cluster/ClusterModule.java  |   4 +
 .../elasticsearch/cluster/ClusterState.java   |  80 +-
 .../ElasticsearchNodeCommand.java             | 123 ++-
 .../PublicationTransportHandler.java          |   6 +-
 .../cluster/metadata/AliasMetaData.java       |   2 +
 .../cluster/metadata/DataStream.java          |  15 +-
 .../cluster/metadata/DataStreamMetadata.java  | 188 +++++
 .../cluster/metadata/IndexMetaData.java       |  84 +-
 .../metadata/IndexTemplateMetaData.java       |   3 +-
 .../cluster/metadata/MetaData.java            | 144 ++--
 .../MetaDataIndexTemplateService.java         |  51 +-
 .../cluster/node/DiscoveryNode.java           |  15 +-
 .../cluster/node/DiscoveryNodeRole.java       |  13 +-
 .../cluster/routing/OperationRouting.java     |   5 +
 .../common/bytes/ByteBufferReference.java     |  19 +
 .../common/bytes/CompositeBytesReference.java |  28 +
 .../common/compress/DeflateCompressor.java    |   9 +-
 .../org/elasticsearch/common/io/Streams.java  |  11 -
 .../common/io/stream/DelayableWriteable.java  | 132 +++
 .../common/io/stream/FilterStreamInput.java   |   5 +
 .../NamedWriteableAwareStreamInput.java       |   5 +
 .../common/io/stream/StreamInput.java         |   8 +
 .../common/io/stream/StreamOutput.java        |  16 +-
 .../common/settings/ClusterSettings.java      |  20 +-
 .../common/unit/ByteSizeValue.java            |  12 +-
 .../common/util/concurrent/ThreadContext.java |  87 +-
 .../gateway/PersistedClusterStateService.java |  10 +-
 .../index/mapper/MappedFieldType.java         |   4 +-
 .../index/mapper/NumberFieldMapper.java       |   1 -
 .../index/mapper/StringFieldType.java         |  57 +-
 .../index/query/QueryBuilders.java            |   2 +-
 .../index/reindex/RetryListener.java          |   3 +-
 .../reindex/WorkerBulkByScrollTaskState.java  |   8 +-
 .../elasticsearch/index/shard/IndexShard.java |  11 +-
 .../BlobStoreIndexShardSnapshots.java         |  18 +-
 .../snapshots/blobstore/SnapshotFiles.java    |  21 +-
 .../indices/IndicesRequestCache.java          |   6 +-
 .../java/org/elasticsearch/node/Node.java     |   6 +-
 .../plugins/SystemIndexPlugin.java            |   4 +-
 .../repositories/FilterRepository.java        |   8 +-
 .../repositories/Repository.java              |   8 +-
 .../blobstore/BlobStoreRepository.java        | 137 ++--
 .../elasticsearch/rest/BaseRestHandler.java   |  53 --
 .../elasticsearch/rest/RestController.java    |  17 +-
 .../action/RestCancellableNodeClient.java     |  52 +-
 .../cluster/RestNodesHotThreadsAction.java    |  57 +-
 .../admin/cluster/RestNodesInfoAction.java    |  25 +-
 .../rest/action/cat/RestNodeAttrsAction.java  |   3 +-
 .../rest/action/cat/RestNodesAction.java      |   6 +-
 .../rest/action/cat/RestPluginsAction.java    |   3 +-
 .../rest/action/cat/RestShardsAction.java     |   9 +-
 .../rest/action/cat/RestThreadPoolAction.java |   4 +-
 .../rest/action/document/RestIndexAction.java |  11 +-
 .../action/search/RestClearReaderAction.java  |  55 ++
 .../action/search/RestMultiSearchAction.java  |  10 +-
 .../rest/action/search/RestSearchAction.java  |   4 +-
 .../search/DefaultSearchContext.java          |  23 +-
 .../elasticsearch/search/SearchModule.java    |  20 +-
 .../elasticsearch/search/SearchService.java   | 335 ++++----
 .../search/aggregations/AggregationPhase.java |   9 +-
 .../aggregations/AggregatorFactories.java     |  51 +-
 .../aggregations/InternalAggregation.java     |  42 +-
 .../aggregations/InternalAggregations.java    |  79 +-
 .../PipelineAggregationBuilder.java           | 148 +++-
 .../CompositeValuesSourceParserHelper.java    |   2 +-
 .../DateHistogramValuesSourceBuilder.java     |  17 +-
 .../geogrid/GeoGridAggregationBuilder.java    |  14 +-
 .../GeoHashGridAggregationBuilder.java        |  14 +-
 .../GeoTileGridAggregationBuilder.java        |  14 +-
 .../AutoDateHistogramAggregationBuilder.java  |  25 +-
 .../DateHistogramAggregationBuilder.java      |  36 +-
 .../bucket/histogram/DateIntervalWrapper.java |   2 +-
 .../HistogramAggregationBuilder.java          |  19 +-
 .../range/DateRangeAggregationBuilder.java    |   7 +-
 .../range/IpRangeAggregationBuilder.java      |  26 +-
 .../bucket/range/RangeAggregationBuilder.java |   8 +-
 .../SignificantTermsAggregatorFactory.java    |  11 +-
 .../SignificantTextAggregatorFactory.java     |   8 +
 .../metrics/GeoBoundsAggregationBuilder.java  |  19 +-
 .../GeoCentroidAggregationBuilder.java        |  15 +-
 .../AbstractPipelineAggregationBuilder.java   |  42 -
 ...cketMetricsPipelineAggregationBuilder.java |  30 +-
 ...ucketScriptPipelineAggregationBuilder.java |   5 +
 ...ketSelectorPipelineAggregationBuilder.java |   5 +
 .../BucketSortPipelineAggregationBuilder.java |   9 +-
 ...mulativeSumPipelineAggregationBuilder.java |  13 +-
 .../DerivativePipelineAggregationBuilder.java |  13 +-
 ...StatsBucketPipelineAggregationBuilder.java |  15 +-
 .../MovFnPipelineAggregationBuilder.java      |  12 +-
 ...tilesBucketPipelineAggregationBuilder.java |  14 +-
 .../SerialDiffPipelineAggregationBuilder.java |  11 +-
 .../search/builder/SearchSourceBuilder.java   | 118 ++-
 .../search/internal/ContextIndexSearcher.java |  21 +-
 .../internal/ExitableDirectoryReader.java     |   2 +-
 .../search/internal/LegacyReaderContext.java  |  12 +-
 .../search/internal/ReaderContext.java        |  55 +-
 .../search/internal/SearchContextId.java      |   2 +-
 .../search/internal/ShardSearchRequest.java   |  86 +-
 .../search/query/QueryPhase.java              |   8 +-
 .../search/query/QuerySearchResult.java       |  95 ++-
 .../snapshots/SnapshotShardsService.java      |  30 +-
 .../elasticsearch/threadpool/Scheduler.java   |  12 +-
 .../elasticsearch/threadpool/ThreadPool.java  |  10 +-
 .../CompressibleBytesOutputStream.java        |  11 -
 .../transport/InboundMessage.java             |  35 +-
 .../transport/OutboundMessage.java            |   3 +-
 .../transport/ProxyConnectionStrategy.java    |   4 +-
 .../transport/RemoteClusterService.java       |  34 +-
 .../transport/TransportLogger.java            |  16 +-
 .../transport/TransportService.java           |   9 +-
 .../action/ActionModuleTests.java             |   6 +-
 .../node/info/NodesInfoRequestTests.java      | 128 +--
 .../reroute/ClusterRerouteResponseTests.java  |  18 +-
 .../admin/cluster/stats/ClusterStatsIT.java   |  16 +-
 .../CreateDataStreamRequestTests.java         |  47 ++
 .../DeleteDataStreamRequestTests.java         |  46 ++
 .../GetDataStreamsRequestTests.java           |  36 +
 .../GetDataStreamsResponseTests.java          |   5 +-
 .../AbstractSearchAsyncActionTests.java       |  19 +-
 ....java => ClearContextControllerTests.java} |  15 +-
 .../action/search/ExpandSearchPhaseTests.java |   2 +-
 .../action/search/FetchSearchPhaseTests.java  |  13 +-
 .../action/search/MockSearchPhaseContext.java |  10 +-
 .../search/MultiSearchActionTookTests.java    |   2 +-
 .../search/MultiSearchResponseTests.java      |  10 +-
 .../action/search/SearchAsyncActionTests.java |   2 +-
 .../search/SearchPhaseControllerTests.java    |   3 +-
 .../SearchQueryThenFetchAsyncActionTests.java |  16 +-
 .../search/SearchResponseMergerTests.java     |  24 +-
 .../action/search/SearchResponseTests.java    |   8 +-
 .../search/SearchScrollAsyncActionTests.java  |  68 +-
 .../TransportMultiSearchActionTests.java      |   2 +-
 .../search/TransportSearchActionTests.java    | 185 +++--
 .../search/TransportSearchHelperTests.java    |  54 +-
 .../cluster/ClusterHealthIT.java              |   3 +-
 .../cluster/ClusterStateTests.java            | 771 ++++++++++++++++++
 .../ElasticsearchNodeCommandTests.java        | 113 +++
 .../metadata/ComponentTemplateTests.java      |   2 +-
 .../metadata/DataStreamMetadataTests.java     |  59 ++
 .../cluster/metadata/DataStreamTests.java     |  23 +-
 .../cluster/metadata/IndexMetaDataTests.java  |   3 +-
 .../MetaDataIndexTemplateServiceTests.java    |  73 +-
 .../cluster/metadata/MetaDataTests.java       |  10 +-
 .../metadata/ToAndFromJsonMetaDataTests.java  | 704 +++++++++++-----
 .../cluster/node/DiscoveryNodeTests.java      |  30 +
 .../common/compress/DeflateCompressTests.java |   7 -
 .../common/io/stream/BytesStreamsTests.java   |  27 +
 .../io/stream/DelayableWriteableTests.java    | 186 +++++
 .../util/concurrent/ThreadContextTests.java   | 133 +--
 .../env/NodeRepurposeCommandTests.java        |   2 +-
 .../env/OverrideNodeVersionCommandTests.java  |   4 +-
 .../gateway/MetaDataStateFormatTests.java     | 159 +---
 .../PersistedClusterStateServiceTests.java    |   2 +-
 .../index/query/PrefixQueryBuilderTests.java  |   2 +-
 .../query/QueryStringQueryBuilderTests.java   |   2 +-
 .../WorkerBulkByScrollTaskStateTests.java     |   5 +-
 .../RemoveCorruptedShardDataCommandTests.java |   2 +-
 .../indices/IndicesRequestCacheIT.java        |  56 +-
 .../state/CloseWhileRelocatingShardsIT.java   |   3 +-
 .../RepositoriesServiceTests.java             |   4 +-
 .../repositories/fs/FsRepositoryTests.java    |   8 +-
 .../rest/RestControllerTests.java             |  33 +-
 .../rest/RestHttpResponseHeadersTests.java    |   2 +-
 .../cluster/RestNodesInfoActionTests.java     |  85 +-
 .../indices/RestValidateQueryActionTests.java |   2 +-
 .../cat/RestCatRecoveryActionTests.java       |   2 +-
 .../action/cat/RestIndicesActionTests.java    |   3 +-
 .../action/cat/RestShardsActionTests.java     | 121 +++
 .../action/document/RestIndexActionTests.java |   7 +-
 .../search/DefaultSearchContextTests.java     |  25 +-
 .../search/SearchCancellationTests.java       |  14 +-
 .../search/SearchModuleTests.java             |   3 +
 .../search/SearchServiceTests.java            |  38 +-
 .../InternalAggregationsTests.java            |  58 +-
 .../aggregations/bucket/DateRangeTests.java   |   2 +-
 .../aggregations/bucket/RangeTests.java       |   4 +-
 .../geogrid/GeoHashGridParserTests.java       |  26 +-
 .../geogrid/GeoTileGridParserTests.java       |  16 +-
 .../SignificantTermsAggregatorTests.java      |  26 +
 .../SignificantTextAggregatorTests.java       |  27 +
 .../metrics/AvgAggregatorTests.java           |  14 +
 .../metrics/MinAggregatorTests.java           |  29 +-
 .../metrics/StatsAggregatorTests.java         | 364 ++++++++-
 .../search/aggregations/metrics/StatsIT.java  | 259 ------
 .../search/aggregations/metrics/SumIT.java    |  36 -
 .../AbstractBucketMetricsTestCase.java        |   1 -
 .../aggregations/pipeline/AvgBucketTests.java |  25 +-
 .../aggregations/pipeline/BucketSortIT.java   |  10 +-
 .../CumulativeSumAggregatorTests.java         |  33 -
 .../pipeline/CumulativeSumTests.java          |  29 +-
 .../pipeline/DerivativeTests.java             |  28 +-
 .../pipeline/ExtendedStatsBucketTests.java    |  24 +-
 .../aggregations/pipeline/MaxBucketTests.java |  25 +-
 .../aggregations/pipeline/MinBucketTests.java |  25 +-
 ...nitTests.java => MovFnAggrgatorTests.java} |  36 +-
 ...eAggregationBuilderSerializationTests.java |  42 +-
 .../pipeline/PercentilesBucketTests.java      |  24 +-
 .../PipelineAggregationHelperTests.java       |  47 +-
 .../pipeline/SerialDifferenceTests.java       |  41 +-
 .../pipeline/StatsBucketTests.java            |  25 +-
 .../aggregations/pipeline/SumBucketTests.java |  25 +-
 .../internal/ContextIndexSearcherTests.java   |   2 +-
 .../internal/ShardSearchRequestTests.java     |  50 +-
 .../profile/query/QueryProfilerTests.java     |   4 +-
 .../search/query/QueryPhaseTests.java         |   6 +-
 .../search/query/QuerySearchResultTests.java  |  20 +-
 .../search/scroll/SearchScrollIT.java         |   4 +-
 .../SearchWithReaderContextIT.java            | 183 +++++
 .../snapshots/BlobStoreIncrementalityIT.java  | 217 +++++
 .../RepositoryFilterUserMetadataIT.java       |  10 +-
 .../threadpool/ThreadPoolTests.java           |  33 +
 .../CompressibleBytesOutputStreamTests.java   |  13 -
 .../ProxyConnectionStrategyTests.java         |  14 +-
 .../transport/RemoteClusterClientTests.java   |  19 +-
 .../RemoteClusterConnectionTests.java         |   2 +-
 .../transport/RemoteClusterServiceTests.java  |  26 +
 .../transport/RemoteClusterSettingsTests.java |  16 +-
 settings.gradle                               |   2 +-
 .../java/fixture/azure/AzureHttpFixture.java  |   2 +-
 .../java/fixture/azure/AzureHttpHandler.java  |  55 +-
 test/fixtures/hdfs-fixture/build.gradle       |   2 +-
 .../coordination/DeterministicTaskQueue.java  |   5 -
 .../index/shard/IndexShardTestCase.java       |   3 +-
 .../index/shard/RestoreOnlyRepository.java    |   4 +-
 .../aggregations/AggregatorTestCase.java      | 116 ++-
 .../BasePipelineAggregationTestCase.java      |  42 +-
 .../metrics/AbstractNumericTestCase.java      |  30 +-
 .../elasticsearch/test/ESIntegTestCase.java   |  94 +++
 .../org/elasticsearch/test/ESTestCase.java    |   8 +
 .../test/InternalAggregationTestCase.java     |  22 +-
 .../test/rest/ESRestTestCase.java             |  33 +
 .../test/rest/RestActionTestCase.java         |   2 +-
 .../AbstractSimpleTransportTestCase.java      |  12 +-
 .../security/create-role-mappings.asciidoc    |  54 +-
 ...CardinalityPipelineAggregationBuilder.java |  19 +-
 .../analytics/StubAggregatorFactory.java      |  56 --
 .../boxplot/BoxplotAggregatorTests.java       | 127 +++
 .../CumulativeCardinalityAggregatorTests.java |  66 --
 .../CumulativeCardinalityTests.java           |  73 ++
 .../StringStatsAggregatorTests.java           | 188 +++++
 .../xpack/search/AsyncSearchSecurityIT.java   |   6 +-
 .../xpack/search/AsyncSearchIndexService.java |   2 +-
 .../xpack/search/AsyncSearchTask.java         |  31 +-
 .../xpack/search/MutableSearchResponse.java   |   2 +-
 .../search/RestGetAsyncSearchAction.java      |   4 +-
 .../search/RestSubmitAsyncSearchAction.java   |  11 +-
 .../TransportSubmitAsyncSearchAction.java     |  19 +-
 .../xpack/search/AsyncSearchActionTests.java  |  28 +-
 .../search/AsyncSearchIntegTestCase.java      | 172 +---
 .../search/AsyncSearchResponseTests.java      |  41 +-
 .../xpack/search/AsyncSearchTaskTests.java    |  27 +-
 .../xpack/search/BlockingQueryBuilder.java    | 118 +++
 .../search/CancellingAggregationBuilder.java  | 102 +++
 .../xpack/search/SearchTestPlugin.java        |  42 +
 .../xpack/search/ShardIdLatch.java            |  29 +
 .../search/SubmitAsyncSearchRequestTests.java |  13 +-
 .../autoscaling/AutoscalingDecision.java      |  85 ++
 .../autoscaling/AutoscalingDecisionType.java  |  73 ++
 .../autoscaling/AutoscalingDecisions.java     |  77 ++
 .../action/GetAutoscalingDecisionAction.java  |  26 +-
 ...TransportGetAutoscalingDecisionAction.java |   4 +-
 .../autoscaling/AutoscalingDecisionTests.java |  31 +
 ...alingDecisionTypeWireSerializingTests.java |  54 ++
 ...toscalingDecisionWireSerializingTests.java |  24 +
 .../AutoscalingDecisionsTests.java            |  35 +
 ...oscalingDecisionsWireSerializingTests.java |  24 +
 .../autoscaling/AutoscalingTestCase.java      |  56 ++
 .../xpack/ccr/repository/CcrRepository.java   |   6 +-
 .../xpack/ccr/IndexFollowingIT.java           |   1 +
 .../license/XPackLicenseState.java            |  23 +-
 .../SourceOnlySnapshotRepository.java         |   8 +-
 .../xpack/core/ClientHelper.java              |   1 +
 .../validation/SourceDestValidator.java       |   2 +-
 .../core/indexing/AsyncTwoPhaseIndexer.java   |   7 +-
 .../xpack/core/indexing/IndexerJobStats.java  |  42 +-
 .../elasticsearch/xpack/core/ml/MlTasks.java  |  12 +-
 .../GetDataFrameAnalyticsStatsAction.java     |  29 +-
 .../ml/dataframe/analyses/Classification.java |   9 +-
 .../core/ml/dataframe/analyses/MapUtils.java  |  40 +
 .../ml/dataframe/analyses/Regression.java     |   7 +-
 .../ml/dataframe/stats/common/DataCounts.java | 120 +++
 .../OutlierDetectionStats.java                |   2 +-
 .../ClassificationInferenceResults.java       |  11 +-
 .../inference/results/FeatureImportance.java  |  97 +++
 .../results/RawInferenceResults.java          |   6 +-
 .../results/RegressionInferenceResults.java   |  14 +-
 .../results/SingleValueInferenceResults.java  |  31 +-
 .../trainedmodel/InferenceHelpers.java        |  40 +-
 .../inference/trainedmodel/TrainedModel.java  |   4 +-
 .../trainedmodel/ensemble/Ensemble.java       |  37 +-
 .../langident/LangIdentNeuralNetwork.java     |   4 +-
 .../ml/inference/trainedmodel/tree/Tree.java  |  73 +-
 .../xpack/core/ml/job/messages/Messages.java  |   2 +-
 .../rollup/job/RollupIndexerJobStats.java     |  17 +-
 .../search/action/AsyncSearchResponse.java    |   4 +-
 .../action/SubmitAsyncSearchRequest.java      |  47 +-
 .../xpack/core/security/SecurityContext.java  |  16 +
 .../core/security/authc/Authentication.java   |   4 +
 .../support/SecondaryAuthentication.java      |   5 +
 .../support/mapper/TemplateRoleName.java      |  12 +
 .../ManageOwnApiKeyClusterPrivilege.java      |   6 +-
 .../authz/store/ReservedRolesStore.java       |   9 +-
 .../support/RestorableContextClassLoader.java |   2 +-
 .../transforms/TransformIndexerStats.java     |  86 +-
 .../TransformInternalIndexConstants.java      |   2 +-
 .../plugin-metadata/plugin-security.policy    |   4 +
 .../xpack/core/ml/stats_index_mappings.json   |   9 +
 .../idp/saml-service-provider-template.json   |  99 +++
 .../LicensesMetaDataSerializationTests.java   |   2 +-
 .../license/XPackLicenseStateTests.java       |   6 +-
 .../SourceOnlySnapshotShardTests.java         |  14 +-
 .../xpack/core/ClientHelperTests.java         |   6 +-
 .../indexing/AsyncTwoPhaseIndexerTests.java   |   4 +-
 .../xpack/core/ml/MlTasksTests.java           |   3 +-
 ...rameAnalyticsStatsActionResponseTests.java |   5 +-
 .../analyses/ClassificationTests.java         |  43 +-
 .../dataframe/analyses/RegressionTests.java   |   6 +-
 .../stats/common/DataCountsTests.java         |  66 ++
 .../ClassificationInferenceResultsTests.java  |  44 +
 .../results/FeatureImportanceTests.java       |  44 +
 .../results/RawInferenceResultsTests.java     |   7 +-
 .../RegressionInferenceResultsTests.java      |  39 +-
 .../trainedmodel/ensemble/EnsembleTests.java  |  42 +-
 .../trainedmodel/tree/TreeTests.java          |  18 +-
 .../job/JobWrapperSerializingTests.java       |   3 +-
 .../job/RollupIndexerJobStatsTests.java       |   3 +-
 .../support/mapper/TemplateRoleNameTests.java | 117 +++
 ...ityIndexReaderWrapperIntegrationTests.java |  10 +-
 .../ManageOwnApiKeyClusterPrivilegeTests.java |  24 +-
 .../authz/store/ReservedRolesStoreTests.java  |  16 +
 .../TransformIndexerStatsTests.java           | 109 ++-
 .../transforms/TransformStatsTests.java       |  30 +-
 .../xpack/enrich/EnrichPlugin.java            |   2 +-
 .../action/EnrichShardMultiSearchAction.java  |   3 +-
 .../xpack/enrich/GeoMatchProcessorTests.java  |   3 +-
 .../xpack/enrich/MatchProcessorTests.java     |   3 +-
 .../xpack/enrich/action/CoordinatorTests.java |   2 +-
 .../resources/test_queries_unsupported.toml   |  28 -
 .../function/scalar/string/Substring.java     |   3 +-
 .../xpack/eql/parser/EqlParser.java           |  47 +-
 .../xpack/eql/plugin/eql_whitelist.txt        |  10 +-
 .../xpack/eql/analysis/VerifierTests.java     |  16 +-
 .../xpack/eql/planner/QueryFolderOkTests.java |   2 +-
 .../src/test/resources/mapping-default.json   |   6 +
 .../test/resources/queries-unsupported.eql    |  70 +-
 .../src/test/resources/queryfolder_tests.txt  |   8 +
 x-pack/plugin/identity-provider/build.gradle  | 368 +++++++++
 .../forbidden/xml-signatures.txt              |   8 +
 .../licenses/cryptacular-1.2.4.jar.sha1       |   1 +
 .../licenses/cryptacular-LICENSE.txt          | 202 +++++
 .../licenses/cryptacular-NOTICE.txt           | 710 ++++++++++++++++
 .../licenses/guava-19.0.jar.sha1              |   1 +
 .../licenses/guava-LICENSE.txt                | 202 +++++
 .../licenses/guava-NOTICE.txt                 |   0
 .../licenses/httpclient-LICENSE.txt           | 558 +++++++++++++
 .../licenses/httpclient-NOTICE.txt            |   6 +
 .../licenses/httpclient-cache-4.5.10.jar.sha1 |   1 +
 .../licenses/java-support-7.5.1.jar.sha1      |   1 +
 .../licenses/log4j-slf4j-impl-2.11.1.jar.sha1 |   1 +
 .../licenses/log4j-slf4j-impl-LICENSE.txt     | 202 +++++
 .../licenses/log4j-slf4j-impl-NOTICE.txt      |   8 +
 .../licenses/metrics-core-3.2.2.jar.sha1      |   1 +
 .../licenses/metrics-core-LICENSE.txt         | 202 +++++
 .../licenses/metrics-core-NOTICE.txt          |   4 +
 .../licenses/opensaml-core-3.4.5.jar.sha1     |   1 +
 .../opensaml-messaging-api-3.4.5.jar.sha1     |   1 +
 .../opensaml-messaging-impl-3.4.5.jar.sha1    |   1 +
 .../opensaml-profile-api-3.4.5.jar.sha1       |   1 +
 .../opensaml-profile-impl-3.4.5.jar.sha1      |   1 +
 .../licenses/opensaml-saml-api-3.4.5.jar.sha1 |   1 +
 .../opensaml-saml-impl-3.4.5.jar.sha1         |   1 +
 .../opensaml-security-api-3.4.5.jar.sha1      |   1 +
 .../opensaml-security-impl-3.4.5.jar.sha1     |   1 +
 .../licenses/opensaml-soap-api-3.4.5.jar.sha1 |   1 +
 .../opensaml-soap-impl-3.4.5.jar.sha1         |   1 +
 .../opensaml-storage-api-3.4.5.jar.sha1       |   1 +
 .../opensaml-storage-impl-3.4.5.jar.sha1      |   1 +
 .../opensaml-xmlsec-api-3.4.5.jar.sha1        |   1 +
 .../opensaml-xmlsec-impl-3.4.5.jar.sha1       |   1 +
 .../licenses/shibboleth-LICENSE.txt           | 202 +++++
 .../licenses/shibboleth-NOTICE.txt            |   0
 .../licenses/slf4j-api-1.6.2.jar.sha1         |   1 +
 .../licenses/slf4j-api-LICENSE.txt            |  21 +
 .../licenses/slf4j-api-NOTICE.txt             |   0
 .../licenses/xmlsec-2.1.4.jar.sha1            |   1 +
 .../licenses/xmlsec-LICENSE.txt               | 202 +++++
 .../licenses/xmlsec-NOTICE.txt                |   9 +
 .../plugin/identity-provider/qa/build.gradle  |   0
 .../qa/idp-rest-tests/build.gradle            |  33 +
 .../xpack/idp/IdpRestTestCase.java            |  32 +
 .../idp/ManageServiceProviderRestIT.java      | 138 ++++
 .../src/test/resources/idp-sign.crt           |  22 +
 .../src/test/resources/idp-sign.key           |  27 +
 .../src/test/resources/roles.yml              |   6 +
 .../xpack/idp/IdentityProviderPlugin.java     | 160 ++++
 .../DeleteSamlServiceProviderAction.java      |  22 +
 .../DeleteSamlServiceProviderRequest.java     |  79 ++
 .../DeleteSamlServiceProviderResponse.java    |  99 +++
 .../action/PutSamlServiceProviderAction.java  |  19 +
 .../action/PutSamlServiceProviderRequest.java | 138 ++++
 .../PutSamlServiceProviderResponse.java       |  97 +++
 .../SamlInitiateSingleSignOnAction.java       |  21 +
 .../SamlInitiateSingleSignOnRequest.java      |  78 ++
 .../SamlInitiateSingleSignOnResponse.java     |  51 ++
 .../xpack/idp/action/SamlMetadataAction.java  |  18 +
 .../xpack/idp/action/SamlMetadataRequest.java |  50 ++
 .../idp/action/SamlMetadataResponse.java      |  36 +
 .../SamlValidateAuthnRequestAction.java       |  18 +
 .../SamlValidateAuthnRequestRequest.java      |  57 ++
 .../SamlValidateAuthnRequestResponse.java     |  60 ++
 ...nsportDeleteSamlServiceProviderAction.java |  62 ++
 ...TransportPutSamlServiceProviderAction.java | 120 +++
 ...ansportSamlInitiateSingleSignOnAction.java | 146 ++++
 .../action/TransportSamlMetadataAction.java   |  37 +
 ...ansportSamlValidateAuthnRequestAction.java |  42 +
 .../xpack/idp/authc/AuthenticationMethod.java |  17 +
 .../xpack/idp/authc/NetworkControl.java       |  15 +
 .../privileges/ServiceProviderPrivileges.java |  55 ++
 .../idp/privileges/UserPrivilegeResolver.java | 127 +++
 ...dAuthenticationResponseMessageBuilder.java | 108 +++
 .../saml/authn/SamlAuthnRequestValidator.java | 354 ++++++++
 ...lAuthenticationResponseMessageBuilder.java | 274 +++++++
 .../saml/authn/UserServiceAuthentication.java |  85 ++
 .../idp/saml/idp/SamlIdPMetadataBuilder.java  | 264 ++++++
 .../idp/saml/idp/SamlIdentityProvider.java    | 208 +++++
 .../saml/idp/SamlIdentityProviderBuilder.java | 356 ++++++++
 .../idp/saml/idp/SamlMetadataGenerator.java   | 104 +++
 .../saml/rest/action/IdpBaseRestHandler.java  |  50 ++
 .../RestDeleteSamlServiceProviderAction.java  |  60 ++
 .../RestPutSamlServiceProviderAction.java     |  63 ++
 .../RestSamlInitiateSingleSignOnAction.java   |  77 ++
 .../rest/action/RestSamlMetadataAction.java   |  58 ++
 ...mlValidateAuthenticationRequestAction.java |  72 ++
 .../idp/saml/sp/CloudServiceProvider.java     | 116 +++
 .../idp/saml/sp/SamlServiceProvider.java      |  56 ++
 .../saml/sp/SamlServiceProviderDocument.java  | 539 ++++++++++++
 .../idp/saml/sp/SamlServiceProviderIndex.java | 337 ++++++++
 .../saml/sp/SamlServiceProviderResolver.java  | 151 ++++
 .../idp/saml/sp/ServiceProviderDefaults.java  |  59 ++
 .../idp/saml/sp/ServiceProviderException.java |  24 +
 .../saml/support/SamlAuthenticationState.java | 151 ++++
 .../xpack/idp/saml/support/SamlFactory.java   | 359 ++++++++
 .../xpack/idp/saml/support/SamlInit.java      |  54 ++
 .../idp/saml/support/SamlObjectSigner.java    |  58 ++
 .../xpack/idp/saml/support/XmlValidator.java  |  95 +++
 .../plugin-metadata/plugin-security.policy    |  29 +
 .../support/saml-schema-assertion-2.0.xsd     | 283 +++++++
 .../saml/support/saml-schema-metadata-2.0.xsd | 337 ++++++++
 .../saml/support/saml-schema-protocol-2.0.xsd | 302 +++++++
 .../xpack/idp/saml/support/xenc-schema.xsd    | 135 +++
 .../xpack/idp/saml/support/xml.xsd            | 286 +++++++
 .../idp/saml/support/xmldsig-core-schema.xsd  | 308 +++++++
 .../idp/LocalStateIdentityProviderPlugin.java |  41 +
 ...DeleteSamlServiceProviderRequestTests.java |  33 +
 ...eleteSamlServiceProviderResponseTests.java |  38 +
 .../PutSamlServiceProviderRequestTests.java   | 154 ++++
 .../idp/action/SamlIdentityProviderTests.java | 349 ++++++++
 .../SamlInitiateSingleSignOnRequestTests.java |  37 +
 .../SamlValidateAuthnRequestRequestTests.java |  36 +
 ...portPutSamlServiceProviderActionTests.java | 174 ++++
 ...tSamlInitiateSingleSignOnRequestTests.java | 175 ++++
 .../UserPrivilegeResolverTests.java           | 168 ++++
 ...ledAuthenticationResponseBuilderTests.java |  63 ++
 .../authn/SamlAuthnRequestValidatorTests.java | 312 +++++++
 ...enticationResponseMessageBuilderTests.java |  79 ++
 .../idp/SamlIdentityProviderBuilderTests.java | 536 ++++++++++++
 .../saml/idp/SamlIdpMetadataBuilderTests.java | 177 ++++
 .../saml/idp/SamlMetadataGeneratorTests.java  | 132 +++
 .../rest/action/IdpBaseRestHandlerTests.java  |  58 ++
 .../sp/SamlServiceProviderDocumentTests.java  | 162 ++++
 .../sp/SamlServiceProviderIndexTests.java     | 303 +++++++
 .../sp/SamlServiceProviderResolverTests.java  | 169 ++++
 .../support/SamlAuthenticationStateTests.java | 110 +++
 .../saml/support/SamlObjectSignerTests.java   | 166 ++++
 .../test/IdentityProviderIntegTestCase.java   | 297 +++++++
 .../xpack/idp/saml/test/IdpSamlTestCase.java  | 186 +++++
 .../src/test/resources/keypair/README.txt     |   6 +
 .../resources/keypair/keypair_RSA2_4096.crt   |  29 +
 .../resources/keypair/keypair_RSA2_4096.key   |  52 ++
 .../resources/keypair/keypair_RSA_1024.crt    |  15 +
 .../resources/keypair/keypair_RSA_1024.key    |  15 +
 .../resources/keypair/keypair_RSA_2048.crt    |  21 +
 .../resources/keypair/keypair_RSA_2048.key    |  27 +
 .../resources/keypair/keypair_RSA_4096.crt    |  29 +
 .../resources/keypair/keypair_RSA_4096.key    |  52 ++
 .../xpack/idp/saml/idp/README.txt             |  24 +
 .../xpack/idp/saml/idp/multi_signing.jks      | Bin 0 -> 6640 bytes
 .../xpack/idp/saml/idp/multi_signing.p12      | Bin 0 -> 7462 bytes
 .../xpack/idp/saml/idp/signing.p12            | Bin 0 -> 2419 bytes
 .../xpack/idp/saml/idp/signing1.crt           |  17 +
 .../xpack/idp/saml/idp/signing1.key           |  28 +
 .../xpack/idp/saml/idp/signing2.crt           |  17 +
 .../xpack/idp/saml/idp/signing2.key           |  28 +
 .../xpack/idp/saml/idp/signing3.crt           |  17 +
 .../xpack/idp/saml/idp/signing3.key           |  28 +
 .../xpack/idp/saml/idp/signing4.crt           |   9 +
 .../xpack/idp/saml/idp/signing4.key           |   4 +
 .../ilm/TimeSeriesLifecycleActionsIT.java     |   1 +
 .../xpack/slm/SnapshotLifecycleRestIT.java    |   1 +
 .../xpack/logstash/Logstash.java              |   3 +-
 .../ml/integration/ClassificationIT.java      |  53 +-
 .../ml/integration/InferenceIngestIT.java     |  12 +-
 ...NativeDataFrameAnalyticsIntegTestCase.java |   3 +-
 .../OutlierDetectionWithMissingFieldsIT.java  |   7 +
 .../xpack/ml/integration/RegressionIT.java    |  30 +-
 .../integration/RunDataFrameAnalyticsIT.java  |   5 +
 .../xpack/ml/MachineLearning.java             |   2 +-
 ...ansportDeleteDataFrameAnalyticsAction.java |  52 +-
 .../TransportEstimateModelMemoryAction.java   |  80 +-
 ...sportGetDataFrameAnalyticsStatsAction.java |   9 +
 .../ml/action/TransportOpenJobAction.java     |   6 +-
 .../action/TransportStartDatafeedAction.java  |  25 +-
 .../action/TransportStopDatafeedAction.java   |   8 +-
 .../xpack/ml/datafeed/DatafeedJobBuilder.java |   8 +-
 .../ml/datafeed/DatafeedNodeSelector.java     |  10 +-
 .../extractor/DataFrameDataExtractor.java     |  23 +-
 .../DataFrameDataExtractorContext.java        |   6 +-
 .../DataFrameDataExtractorFactory.java        |  30 +-
 .../ExtractedFieldsDetectorFactory.java       |  13 +-
 .../process/AnalyticsProcessManager.java      |  51 +-
 .../process/AnalyticsResultProcessor.java     |  44 +-
 .../CrossValidationSplitter.java              |   2 +-
 .../CrossValidationSplitterFactory.java       |  73 +-
 .../RandomCrossValidationSplitter.java        |  35 +-
 .../StratifiedCrossValidationSplitter.java    |  90 ++
 .../ml/dataframe/stats/DataCountsTracker.java |  37 +
 .../xpack/ml/dataframe/stats/StatsHolder.java |   6 +
 .../ml/dataframe/stats/StatsPersister.java    |  66 ++
 ...ansportEstimateModelMemoryActionTests.java |   8 +-
 .../ml/datafeed/DatafeedJobBuilderTests.java  |   4 +-
 .../DataFrameDataExtractorTests.java          |  57 +-
 .../AnalyticsResultProcessorTests.java        |   8 +-
 .../RandomCrossValidationSplitterTests.java   |  46 +-
 ...tratifiedCrossValidationSplitterTests.java | 239 ++++++
 .../ingest/InferenceProcessorTests.java       |  25 +-
 .../integration/BasicDistributedJobsIT.java   |  55 ++
 .../xpack/ml/support/BaseMlIntegTestCase.java |   5 +
 .../ResultsPersisterServiceTests.java         |   4 +-
 .../ClusterStatsMonitoringDocTests.java       |   3 +-
 .../function/scalar/BinaryScalarFunction.java |   7 +-
 .../function/scalar/ScalarFunction.java       |   2 +-
 .../whitelist/InternalQlScriptUtils.java      |   8 +
 .../ql/expression/gen/script/Scripts.java     |  20 +-
 .../xpack/ql/expression/predicate/Range.java  |   2 +-
 .../ql/expression/predicate/logical/Not.java  |   2 +-
 .../expression/predicate/nulls/IsNotNull.java |   2 +-
 .../ql/expression/predicate/nulls/IsNull.java |   2 +-
 .../predicate/operator/arithmetic/Neg.java    |   2 +-
 .../predicate/operator/comparison/In.java     |   2 +-
 .../rollup/RollupResponseTranslator.java      |   2 +-
 .../action/TransportRollupSearchAction.java   |   4 +-
 .../RollupResponseTranslationTests.java       |   4 +-
 .../xpack/rollup/job/IndexerUtilsTests.java   |  12 +-
 .../job/RollupIndexerIndexingTests.java       |   2 +-
 .../rollup/job/RollupIndexerStateTests.java   |  13 +-
 .../SecurityOnTrialLicenseRestTestCase.java   |   2 +-
 .../xpack/security/TlsWithBasicLicenseIT.java |   1 +
 .../xpack/security/Security.java              |   2 +-
 .../security/SecurityInfoTransportAction.java |   8 +-
 .../SecurityUsageTransportAction.java         |  11 +-
 .../action/filter/SecurityActionFilter.java   |   2 +-
 .../xpack/security/authc/ApiKeyService.java   |   7 +-
 .../xpack/security/authc/TokenService.java    |  34 +-
 .../authc/saml/SamlAuthenticator.java         |   4 +-
 .../xpack/security/authc/saml/SamlRealm.java  |   6 +-
 .../authc/saml/SamlRequestHandler.java        |  24 +-
 .../xpack/security/authc/saml/SamlUtils.java  |   8 +-
 .../mapper/NativeRoleMappingStore.java        |   5 +
 .../security/authz/AuthorizationService.java  |   3 +-
 .../security/authz/AuthorizationUtils.java    |   2 +
 .../xpack/security/authz/RBACEngine.java      |  11 +-
 .../rest/action/SecurityBaseRestHandler.java  |   2 +-
 .../support/SecurityStatusChangeListener.java |   2 +-
 .../plugin-metadata/plugin-security.policy    |   3 +-
 .../security/ScrollHelperIntegTests.java      |   2 +-
 .../SecurityInfoTransportActionTests.java     |  27 +-
 ...sportSamlInvalidateSessionActionTests.java |   4 +-
 .../authc/AuthenticationServiceTests.java     |   2 +-
 .../mapper/NativeRoleMappingStoreTests.java   |  15 +
 .../xpack/security/authz/RBACEngineTests.java |   5 +-
 .../security/authz/ReadActionsTests.java      |   2 +
 .../store/NativePrivilegeStoreTests.java      |   8 +-
 .../action/SecurityBaseRestHandlerTests.java  |   8 +-
 .../apikey/RestCreateApiKeyActionTests.java   |   1 +
 .../apikey/RestGetApiKeyActionTests.java      |   1 +
 .../RestInvalidateApiKeyActionTests.java      |   1 +
 .../SecurityStatusChangeListenerTests.java    |   8 +-
 .../elasticsearch/xpack/sql/jdbc/Debug.java   |  15 +-
 .../xpack/sql/jdbc/DebugLog.java              |  30 +-
 .../xpack/sql/jdbc/JdbcConfiguration.java     |  13 +-
 .../sql/jdbc/JdbcConfigurationTests.java      |  17 +
 .../xpack/sql/qa/rest/RestSqlTestCase.java    |   5 +-
 ...e-core-8.5.0-snapshot-7f057455901.jar.sha1 |   1 -
 .../licenses/lucene-core-8.5.0.jar.sha1       |   1 +
 .../function/SqlFunctionRegistry.java         |   2 +-
 .../scalar/string/BinaryStringFunction.java   |   3 +-
 .../function/scalar/string/Concat.java        |   3 +-
 .../function/scalar/string/Insert.java        |   3 +-
 .../function/scalar/string/Locate.java        |   3 +-
 .../function/scalar/string/Replace.java       |   3 +-
 .../scalar/string/UnaryStringFunction.java    |   3 +-
 .../scalar/string/UnaryStringIntFunction.java |   3 +-
 .../whitelist/InternalSqlScriptUtils.java     |   5 -
 .../xpack/sql/plugin/sql_whitelist.txt        |  17 +-
 .../xpack/sql/planner/QueryFolderTests.java   |  15 +-
 .../sql/planner/QueryTranslatorTests.java     | 182 +++--
 .../rest-api-spec/api/async_search.get.json   |   8 +-
 .../api/async_search.submit.json              |  10 +-
 ....cat_transform.json => cat.transform.json} |   2 +-
 .../test/analytics/string_stats.yml           |  53 ++
 .../test/async_search/10_basic.yml            |  10 +-
 .../test/ml/data_frame_analytics_crud.yml     |  56 +-
 .../test/ml/estimate_model_memory.yml         | 128 ++-
 .../rest-api-spec/test/ml/jobs_crud.yml       |   2 +-
 .../rest-api-spec/test/rollup/delete_job.yml  |   6 +
 .../rest-api-spec/test/rollup/get_jobs.yml    |   4 +
 .../rest-api-spec/test/rollup/put_job.yml     |   2 +
 .../test/transform/transforms_cat_apis.yml    |   8 +-
 .../test/wildcard/10_wildcard_basic.yml       |  66 +-
 .../TransformPivotRestSpecialCasesIT.java     | 111 +++
 .../integration/TransformUsageIT.java         |   5 -
 .../xpack/transform/Transform.java            |   8 +-
 .../TransformInfoTransportAction.java         |  18 +-
 .../TransportPreviewTransformAction.java      |   4 +-
 .../action/TransportPutTransformAction.java   |   4 +-
 .../action/TransportStartTransformAction.java |   4 +-
 .../TransportUpdateTransformAction.java       |   4 +-
 .../persistence/TransformInternalIndex.java   |   7 +
 .../TransformPersistentTasksExecutor.java     |   2 +-
 .../pivot/AggregationResultUtils.java         |   9 +-
 .../TransformInfoTransportActionTests.java    |  16 +-
 .../xpack/transform/TransformTests.java       |  19 +-
 .../transforms/TransformIndexerTests.java     |   3 +-
 ...TransformPersistentTasksExecutorTests.java |  23 +-
 .../pivot/AggregationResultUtilsTests.java    |   5 +
 .../transforms/pivot/PivotTests.java          |   1 +
 .../elasticsearch/xpack/watcher/Watcher.java  |   2 +-
 .../WatcherMetaDataSerializationTests.java    |   2 +-
 .../xpack/watcher/WatcherServiceTests.java    |   4 +-
 .../CompareConditionSearchTests.java          |   2 +-
 .../condition/ScriptConditionTests.java       |  10 +-
 .../execution/TriggeredWatchStoreTests.java   |   4 +-
 .../test/integration/SearchInputTests.java    |   6 +-
 .../wildcard/mapper/WildcardFieldMapper.java  | 101 ++-
 ...SpnegoHttpClientConfigCallbackHandler.java |   6 +-
 .../build.gradle                              |   4 +-
 .../build.gradle                              |   4 +-
 .../authc/oidc/OpenIdConnectAuthIT.java       |   2 +-
 .../reindex-tests-with-security/build.gradle  |   6 +-
 .../80_transform_jobs_crud.yml                |   6 +-
 .../authc/saml/SamlAuthenticationIT.java      |   8 +-
 .../SmokeTestMonitoringWithSecurityIT.java    |   1 +
 1046 files changed, 32535 insertions(+), 8907 deletions(-)
 create mode 100644 .idea/runConfigurations/Debug_Elasticsearch.xml
 delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java
 delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java
 delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java
 rename buildSrc/src/main/java/org/elasticsearch/gradle/{tool/Boilerplate.java => util/GradleUtils.java} (98%)
 rename buildSrc/src/main/java/org/elasticsearch/gradle/{ => util}/Util.java (60%)
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateModelMemoryRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateModelMemoryResponse.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCounts.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequestTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCountsTests.java
 create mode 100644 distribution/docker/aarch64-docker-build-context/build.gradle
 create mode 100644 distribution/docker/aarch64-docker-export/build.gradle
 create mode 100644 distribution/docker/aarch64-oss-docker-build-context/build.gradle
 create mode 100644 distribution/docker/aarch64-oss-docker-export/build.gradle
 create mode 100644 distribution/docker/src/docker/config/tini-arm64.sha512
 create mode 100644 docs/java-rest/high-level/ml/estimate-model-memory.asciidoc
 create mode 100644 docs/reference/eql/functions.asciidoc
 create mode 100644 docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc
 delete mode 100644 modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java
 delete mode 100644 modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java
 delete mode 100644 modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-8.5.0.jar.sha1
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SField.java
 delete mode 100644 modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java
 delete mode 100644 modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0.jar.sha1
 create mode 100644 plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriNumberFilterFactory.java
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0.jar.sha1
 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/SparseBitSet-1.2.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/SparseBitSet-LICENSE.txt
 create mode 100644 plugins/ingest-attachment/licenses/SparseBitSet-NOTICE.txt
 delete mode 100644 plugins/ingest-attachment/licenses/commons-compress-1.18.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/commons-compress-1.19.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/commons-math3-3.6.1.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/commons-math3-LICENSE.txt
 create mode 100644 plugins/ingest-attachment/licenses/commons-math3-NOTICE.txt
 delete mode 100644 plugins/ingest-attachment/licenses/fontbox-2.0.16.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/fontbox-2.0.19.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/pdfbox-2.0.16.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/pdfbox-2.0.19.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-4.0.1.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-4.1.2.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-4.0.1.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-4.1.2.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-schemas-4.0.1.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-schemas-4.1.2.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-scratchpad-4.0.1.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-scratchpad-4.1.2.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/tika-core-1.22.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/tika-core-1.24.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/tika-parsers-1.22.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/tika-parsers-1.24.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/gax-1.45.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/gax-1.54.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-api-client-1.30.8.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-api-client-1.30.9.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20191011-1.30.3.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20200226-1.30.9.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-cloud-core-1.92.5.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-cloud-core-1.93.3.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-cloud-core-http-1.77.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-cloud-core-http-1.93.3.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-cloud-storage-1.104.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-cloud-storage-1.106.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-http-client-appengine-1.29.2.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-http-client-appengine-1.34.2.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-http-client-jackson2-1.29.2.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-http-client-jackson2-1.34.2.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/threetenbp-1.3.3.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/threetenbp-1.4.1.jar.sha1
 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/hadoop-annotations-2.8.5.jar.sha1
 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/hadoop-auth-2.8.5.jar.sha1
 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/hadoop-client-2.8.5.jar.sha1
 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/hadoop-common-2.8.5.jar.sha1
 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.5.jar.sha1
 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.5.jar.sha1
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml
 delete mode 100644 server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-analyzers-common-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-backward-codecs-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-core-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-grouping-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-highlighter-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-join-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-memory-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-misc-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-queries-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-queryparser-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-sandbox-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-spatial-extras-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-spatial3d-8.5.0.jar.sha1
 delete mode 100644 server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 server/licenses/lucene-suggest-8.5.0.jar.sha1
 rename server/src/main/java/org/elasticsearch/action/search/{ClearScrollController.java => ClearContextController.java} (71%)
 create mode 100644 server/src/main/java/org/elasticsearch/action/search/ClearReaderAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/search/ClearReaderRequest.java
 rename modules/kibana/build.gradle => server/src/main/java/org/elasticsearch/action/search/ClearReaderResponse.java (67%)
 rename server/src/main/java/org/elasticsearch/action/search/{ScrollIdForNode.java => ReaderIdForNode.java} (92%)
 create mode 100644 server/src/main/java/org/elasticsearch/action/search/TransportClearReaderAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
 create mode 100644 server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
 create mode 100644 server/src/main/java/org/elasticsearch/rest/action/search/RestClearReaderAction.java
 rename server/src/test/java/org/elasticsearch/action/search/{ClearScrollControllerTests.java => ClearContextControllerTests.java} (95%)
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java
 rename server/src/test/java/org/elasticsearch/search/aggregations/pipeline/{MovFnUnitTests.java => MovFnAggrgatorTests.java} (79%)
 create mode 100644 server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
 create mode 100644 server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
 rename {server/src/test => test/framework/src/main}/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java (85%)
 delete mode 100644 x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/StubAggregatorFactory.java
 create mode 100644 x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityTests.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SearchTestPlugin.java
 create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ShardIdLatch.java
 create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecision.java
 create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionType.java
 create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisions.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTests.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTypeWireSerializingTests.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionWireSerializingTests.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsTests.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsWireSerializingTests.java
 create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MapUtils.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCounts.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportance.java
 rename x-pack/plugin/{security/src/main/java/org/elasticsearch/xpack => core/src/main/java/org/elasticsearch/xpack/core}/security/support/RestorableContextClassLoader.java (96%)
 create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportanceTests.java
 create mode 100644 x-pack/plugin/identity-provider/build.gradle
 create mode 100644 x-pack/plugin/identity-provider/forbidden/xml-signatures.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/cryptacular-1.2.4.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/cryptacular-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/cryptacular-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/guava-19.0.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/guava-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/guava-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/httpclient-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/httpclient-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/httpclient-cache-4.5.10.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/java-support-7.5.1.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-2.11.1.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/metrics-core-3.2.2.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/metrics-core-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/metrics-core-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-core-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-messaging-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-messaging-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-profile-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-profile-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-saml-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-saml-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-security-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-security-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-soap-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-soap-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-storage-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-storage-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-api-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-impl-3.4.5.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/shibboleth-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/shibboleth-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/slf4j-api-1.6.2.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/slf4j-api-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/slf4j-api-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/xmlsec-2.1.4.jar.sha1
 create mode 100644 x-pack/plugin/identity-provider/licenses/xmlsec-LICENSE.txt
 create mode 100644 x-pack/plugin/identity-provider/licenses/xmlsec-NOTICE.txt
 create mode 100644 x-pack/plugin/identity-provider/qa/build.gradle
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/ManageServiceProviderRestIT.java
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.crt
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.key
 create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/roles.yml
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequest.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponse.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequest.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderResponse.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequest.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnResponse.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataRequest.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataResponse.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequest.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestResponse.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/AuthenticationMethod.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/NetworkControl.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/ServiceProviderPrivileges.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolver.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseMessageBuilder.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidator.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilder.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/UserServiceAuthentication.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdPMetadataBuilder.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProvider.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGenerator.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandler.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestDeleteSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestPutSamlServiceProviderAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlInitiateSingleSignOnAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlMetadataAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlValidateAuthenticationRequestAction.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/CloudServiceProvider.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProvider.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocument.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolver.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderDefaults.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderException.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationState.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlFactory.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlInit.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSigner.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/XmlValidator.java
 create mode 100644 x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-assertion-2.0.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-metadata-2.0.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-protocol-2.0.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xenc-schema.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xml.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xmldsig-core-schema.xsd
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/LocalStateIdentityProviderPlugin.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponseTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequestTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequestTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnRequestTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolverTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseBuilderTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidatorTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilderTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdpMetadataBuilderTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGeneratorTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandlerTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndexTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolverTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSignerTests.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdpSamlTestCase.java
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/README.txt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/README.txt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.jks
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.p12
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing.p12
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.key
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.crt
 create mode 100644 x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.key
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitter.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/DataCountsTracker.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitterTests.java
 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0.jar.sha1
 rename x-pack/plugin/src/test/resources/rest-api-spec/api/{transform.cat_transform.json => cat.transform.json} (98%)
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/string_stats.yml

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index caaf385149c48..5229f87c80c9a 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -18,4 +18,5 @@ BWC_VERSION:
   - "7.6.1"
   - "7.6.2"
   - "7.7.0"
+  - "7.8.0"
   - "8.0.0"
diff --git a/.gitignore b/.gitignore
index 309342f3b611d..ff9e964289f37 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,7 @@ out/
 # include shared intellij config
 !.idea/scopes/x_pack.xml
 !.idea/inspectionProfiles/Project_Default.xml
+!.idea/runConfigurations/Debug_Elasticsearch.xml
 
 # These files are generated in the main tree by IntelliJ
 benchmarks/src/main/generated/*
diff --git a/.idea/runConfigurations/Debug_Elasticsearch.xml b/.idea/runConfigurations/Debug_Elasticsearch.xml
new file mode 100644
index 0000000000000..185d6c6fc48f2
--- /dev/null
+++ b/.idea/runConfigurations/Debug_Elasticsearch.xml
@@ -0,0 +1,11 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Debug Elasticsearch" type="Remote">
+    <option name="USE_SOCKET_TRANSPORT" value="true" />
+    <option name="SERVER_MODE" value="true" />
+    <option name="SHMEM_ADDRESS" />
+    <option name="HOST" value="localhost" />
+    <option name="PORT" value="5005" />
+    <option name="AUTO_RESTART" value="true" />
+    <method v="2" />
+  </configuration>
+</component>
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c072b7b3649ad..e6764b505d5f3 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -103,9 +103,6 @@ be used to test against other JDKs as well, this is not only limited to JDK 11.
 > Note: It is also required to have `JAVA8_HOME`, `JAVA9_HOME`, `JAVA10_HOME`
 and `JAVA11_HOME`, and `JAVA12_HOME` available so that the tests can pass.
 
-> Warning: do not use `sdkman` for Java installations which do not have proper
-`jrunscript` for jdk distributions.
-
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
 using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat`
 script on Windows in the root of the repository. The examples below show the
@@ -125,10 +122,13 @@ You can access Elasticsearch with:
 
 ### Importing the project into IntelliJ IDEA
 
-Elasticsearch builds using Java 13. Before importing into IntelliJ you will need
+Elasticsearch builds using Java 13. When importing into IntelliJ you will need
 to define an appropriate SDK. The convention is that **this SDK should be named
 "13"** so that the project import will detect it automatically. For more details
 on defining an SDK in IntelliJ please refer to [their documentation](https://www.jetbrains.com/help/idea/sdk.html#define-sdk).
+SDK definitions are global, so you can add the JDK from any project, or after
+project import. Importing with a missing JDK will still work, IntelliJ will
+simply report a problem and will refuse to build until resolved.
 
 You can import the Elasticsearch project into IntelliJ IDEA via:
 
diff --git a/build.gradle b/build.gradle
index 17cd5f6b93cd0..120fb889c59d1 100644
--- a/build.gradle
+++ b/build.gradle
@@ -31,7 +31,7 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import org.gradle.util.DistributionLocator
 import org.gradle.util.GradleVersion
 
-import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
+import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
 
 plugins {
   id 'lifecycle-base'
@@ -222,8 +222,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53989" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index be8d69da60c4b..7580671cc4cd6 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -32,7 +32,7 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.elasticsearch.gradle.test.ErrorReportingTestListener
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
-import org.elasticsearch.gradle.tool.Boilerplate
+import org.elasticsearch.gradle.util.GradleUtils
 import org.gradle.api.Action
 import org.gradle.api.GradleException
 import org.gradle.api.InvalidUserDataException
@@ -82,7 +82,7 @@ import org.gradle.util.GradleVersion
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 
-import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
+import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -148,7 +148,7 @@ class BuildPlugin implements Plugin<Project> {
             File securityPolicy = buildResources.copy("fips_java.policy")
             File bcfksKeystore = buildResources.copy("cacerts.bcfks")
             // This configuration can be removed once system modules are available
-            Boilerplate.maybeCreate(project.configurations, 'extraJars') {
+            GradleUtils.maybeCreate(project.configurations, 'extraJars') {
                 project.dependencies.add('extraJars', "org.bouncycastle:bc-fips:1.0.1")
                 project.dependencies.add('extraJars', "org.bouncycastle:bctls-fips:1.0.9")
             }
@@ -235,7 +235,7 @@ class BuildPlugin implements Plugin<Project> {
     static String getJavaHome(final Task task, final int version) {
         requireJavaHome(task, version)
         JavaHome java = BuildParams.javaVersions.find { it.version == version }
-        return java == null ? null : java.javaHome.absolutePath
+        return java == null ? null : java.javaHome.get().absolutePath
     }
 
     /**
@@ -453,7 +453,9 @@ class BuildPlugin implements Plugin<Project> {
 
         project.pluginManager.withPlugin('com.github.johnrengelman.shadow') {
             // Ensure that when we are compiling against the "original" JAR that we also include any "shadow" dependencies on the compile classpath
-            project.configurations.getByName(JavaPlugin.API_ELEMENTS_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME))
+            project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME).dependencies.all { Dependency dependency ->
+                project.configurations.getByName(JavaPlugin.API_ELEMENTS_CONFIGURATION_NAME).dependencies.add(dependency)
+            }
         }
     }
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index 52f98330ee50d..d95514f975a13 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -143,16 +143,13 @@ class PrecommitTasks {
         ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
         project.tasks.withType(CheckForbiddenApis).configureEach {
             dependsOn(buildResources)
-            doFirst {
-                // we need to defer this configuration since we don't know the runtime java version until execution time
-                targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
-                if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
-                    project.logger.warn(
-                            "Forbidden APIs does not support Java versions past 13. Will use the signatures from 13 for {}.",
-                            BuildParams.runtimeJavaVersion
-                    )
-                    targetCompatibility = JavaVersion.VERSION_13.majorVersion
-                }
+            targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
+            if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
+                project.logger.warn(
+                        "Forbidden APIs does not support Java versions past 13. Will use the signatures from 13 for {}.",
+                        BuildParams.runtimeJavaVersion
+                )
+                targetCompatibility = JavaVersion.VERSION_13.majorVersion
             }
             bundledSignatures = [
                     "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
index d668aa4b6b7fa..6085a12538552 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
@@ -32,13 +32,11 @@ import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
-import org.gradle.api.plugins.ExtraPropertiesExtension
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.plugins.JavaPlugin
 import org.gradle.api.plugins.JavaPluginExtension
 import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.SourceSetContainer
-import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.api.tasks.testing.Test
 import org.gradle.plugins.ide.eclipse.model.EclipseModel
 import org.gradle.plugins.ide.idea.model.IdeaModel
@@ -67,8 +65,8 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         BuildPlugin.configureTestTasks(project)
         BuildPlugin.configureInputNormalization(project)
         BuildPlugin.configureFips140(project)
+        BuildPlugin.configureCompile(project)
 
-        ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension)
         project.extensions.getByType(JavaPluginExtension).sourceCompatibility = BuildParams.minimumRuntimeVersion
         project.extensions.getByType(JavaPluginExtension).targetCompatibility = BuildParams.minimumRuntimeVersion
 
@@ -96,13 +94,6 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         idea.module.scopes.put('TEST', [plus: [project.configurations.getByName(JavaPlugin.TEST_RUNTIME_CLASSPATH_CONFIGURATION_NAME)]] as Map<String, Collection<Configuration>>)
 
         PrecommitTasks.create(project, false)
-        project.tasks.getByName('check').dependsOn(project.tasks.getByName('precommit'))
-
-        project.tasks.withType(JavaCompile) { JavaCompile task ->
-            // This will be the default in Gradle 5.0
-            if (task.options.compilerArgs.contains("-processor") == false) {
-                task.options.compilerArgs << '-proc:none'
-            }
-        }
+        project.tasks.named('check').configure { it.dependsOn(project.tasks.named('precommit')) }
     }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
index 0982fe0267cc5..048403d83f14d 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.elasticsearch.gradle.info.BuildParams;
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.GradleException;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
@@ -52,7 +52,7 @@
 import java.util.concurrent.Callable;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.gradle.Util.capitalize;
+import static org.elasticsearch.gradle.util.Util.capitalize;
 
 /**
  * A plugin to manage getting and extracting distributions of Elasticsearch.
@@ -77,7 +77,7 @@ public void apply(Project project) {
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
         project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
 
-        Provider<DockerSupportService> dockerSupport = Boilerplate.getBuildService(
+        Provider<DockerSupportService> dockerSupport = GradleUtils.getBuildService(
             project.getGradle().getSharedServices(),
             DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
         );
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
index daaf1a0fa54e9..a2dca7c247c0d 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
@@ -47,8 +47,8 @@
 import java.util.concurrent.Callable;
 import java.util.stream.StreamSupport;
 
-import static org.elasticsearch.gradle.tool.Boilerplate.findByName;
-import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate;
+import static org.elasticsearch.gradle.util.GradleUtils.findByName;
+import static org.elasticsearch.gradle.util.GradleUtils.maybeCreate;
 
 public class JdkDownloadPlugin implements Plugin<Project> {
 
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java
deleted file mode 100644
index 75ae735e8a9f1..0000000000000
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java
+++ /dev/null
@@ -1,293 +0,0 @@
-package org.elasticsearch.gradle.info;
-
-import org.elasticsearch.gradle.OS;
-import org.gradle.api.DefaultTask;
-import org.gradle.api.GradleException;
-import org.gradle.api.JavaVersion;
-import org.gradle.api.file.RegularFileProperty;
-import org.gradle.api.model.ObjectFactory;
-import org.gradle.api.tasks.CacheableTask;
-import org.gradle.api.tasks.Input;
-import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.Nested;
-import org.gradle.api.tasks.OutputFile;
-import org.gradle.api.tasks.PathSensitive;
-import org.gradle.api.tasks.PathSensitivity;
-import org.gradle.api.tasks.TaskAction;
-import org.gradle.internal.jvm.Jvm;
-import org.gradle.process.ExecResult;
-
-import javax.inject.Inject;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.io.Writer;
-import java.nio.file.Files;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Locale;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-@CacheableTask
-public class GenerateGlobalBuildInfoTask extends DefaultTask {
-    private JavaVersion minimumCompilerVersion;
-    private JavaVersion minimumRuntimeVersion;
-    private File compilerJavaHome;
-    private File runtimeJavaHome;
-    private List<JavaHome> javaVersions;
-    private final RegularFileProperty outputFile;
-    private final RegularFileProperty compilerVersionFile;
-    private final RegularFileProperty runtimeVersionFile;
-
-    @Inject
-    public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) {
-        this.outputFile = objectFactory.fileProperty();
-        this.compilerVersionFile = objectFactory.fileProperty();
-        this.runtimeVersionFile = objectFactory.fileProperty();
-    }
-
-    @Input
-    public JavaVersion getMinimumCompilerVersion() {
-        return minimumCompilerVersion;
-    }
-
-    public void setMinimumCompilerVersion(JavaVersion minimumCompilerVersion) {
-        this.minimumCompilerVersion = minimumCompilerVersion;
-    }
-
-    @Input
-    public JavaVersion getMinimumRuntimeVersion() {
-        return minimumRuntimeVersion;
-    }
-
-    public void setMinimumRuntimeVersion(JavaVersion minimumRuntimeVersion) {
-        this.minimumRuntimeVersion = minimumRuntimeVersion;
-    }
-
-    @InputDirectory
-    @PathSensitive(PathSensitivity.RELATIVE)
-    public File getCompilerJavaHome() {
-        return compilerJavaHome;
-    }
-
-    public void setCompilerJavaHome(File compilerJavaHome) {
-        this.compilerJavaHome = compilerJavaHome;
-    }
-
-    @InputDirectory
-    @PathSensitive(PathSensitivity.RELATIVE)
-    public File getRuntimeJavaHome() {
-        return runtimeJavaHome;
-    }
-
-    public void setRuntimeJavaHome(File runtimeJavaHome) {
-        this.runtimeJavaHome = runtimeJavaHome;
-    }
-
-    @Nested
-    public List<JavaHome> getJavaVersions() {
-        return javaVersions;
-    }
-
-    public void setJavaVersions(List<JavaHome> javaVersions) {
-        this.javaVersions = javaVersions;
-    }
-
-    @OutputFile
-    public RegularFileProperty getOutputFile() {
-        return outputFile;
-    }
-
-    @OutputFile
-    public RegularFileProperty getCompilerVersionFile() {
-        return compilerVersionFile;
-    }
-
-    @OutputFile
-    public RegularFileProperty getRuntimeVersionFile() {
-        return runtimeVersionFile;
-    }
-
-    @TaskAction
-    public void generate() {
-        String javaVendorVersion = System.getProperty("java.vendor.version", System.getProperty("java.vendor"));
-        String gradleJavaVersion = System.getProperty("java.version");
-        String gradleJavaVersionDetails = javaVendorVersion
-            + " "
-            + gradleJavaVersion
-            + " ["
-            + System.getProperty("java.vm.name")
-            + " "
-            + System.getProperty("java.vm.version")
-            + "]";
-
-        String compilerJavaVersionDetails = gradleJavaVersionDetails;
-        JavaVersion compilerJavaVersionEnum = JavaVersion.current();
-        String runtimeJavaVersionDetails = gradleJavaVersionDetails;
-        JavaVersion runtimeJavaVersionEnum = JavaVersion.current();
-        File gradleJavaHome = Jvm.current().getJavaHome();
-
-        try {
-            if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) {
-                if (compilerJavaHome.exists()) {
-                    compilerJavaVersionDetails = findJavaVersionDetails(compilerJavaHome);
-                    compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(compilerJavaHome));
-                } else {
-                    throw new RuntimeException("Compiler Java home path of '" + compilerJavaHome + "' does not exist");
-                }
-            }
-
-            if (Files.isSameFile(runtimeJavaHome.toPath(), gradleJavaHome.toPath()) == false) {
-                if (runtimeJavaHome.exists()) {
-                    runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome);
-                    runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome));
-                } else {
-                    throw new RuntimeException("Runtime Java home path of '" + compilerJavaHome + "' does not exist");
-                }
-            }
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-
-        try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile.getAsFile().get()))) {
-            final String osName = System.getProperty("os.name");
-            final String osVersion = System.getProperty("os.version");
-            final String osArch = System.getProperty("os.arch");
-            final JavaVersion parsedVersion = JavaVersion.toVersion(gradleJavaVersion);
-
-            writer.write("  Gradle Version        : " + getProject().getGradle().getGradleVersion() + "\n");
-            writer.write("  OS Info               : " + osName + " " + osVersion + " (" + osArch + ")\n");
-
-            if (gradleJavaVersionDetails.equals(compilerJavaVersionDetails) == false
-                || gradleJavaVersionDetails.equals(runtimeJavaVersionDetails) == false) {
-                writer.write("  Compiler JDK Version  : " + compilerJavaVersionEnum + " (" + compilerJavaVersionDetails + ")\n");
-                writer.write("  Compiler java.home    : " + compilerJavaHome + "\n");
-                writer.write("  Runtime JDK Version   : " + runtimeJavaVersionEnum + " (" + runtimeJavaVersionDetails + ")\n");
-                writer.write("  Runtime java.home     : " + runtimeJavaHome + "\n");
-                writer.write("  Gradle JDK Version    : " + parsedVersion + " (" + gradleJavaVersionDetails + ")\n");
-                writer.write("  Gradle java.home      : " + gradleJavaHome);
-            } else {
-                writer.write("  JDK Version           : " + parsedVersion + " (" + gradleJavaVersionDetails + ")\n");
-                writer.write("  JAVA_HOME             : " + gradleJavaHome);
-            }
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-
-        // enforce Java version
-        if (compilerJavaVersionEnum.compareTo(minimumCompilerVersion) < 0) {
-            String message = String.format(
-                Locale.ROOT,
-                "The compiler java.home must be set to a JDK installation directory for Java %s but is [%s] " + "corresponding to [%s]",
-                minimumCompilerVersion,
-                compilerJavaHome,
-                compilerJavaVersionEnum
-            );
-            throw new GradleException(message);
-        }
-
-        if (runtimeJavaVersionEnum.compareTo(minimumRuntimeVersion) < 0) {
-            String message = String.format(
-                Locale.ROOT,
-                "The runtime java.home must be set to a JDK installation directory for Java %s but is [%s] " + "corresponding to [%s]",
-                minimumRuntimeVersion,
-                runtimeJavaHome,
-                runtimeJavaVersionEnum
-            );
-            throw new GradleException(message);
-        }
-
-        for (JavaHome javaVersion : javaVersions) {
-            File javaHome = javaVersion.getJavaHome();
-            if (javaHome == null) {
-                continue;
-            }
-            JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(javaHome));
-            JavaVersion expectedJavaVersionEnum;
-            int version = javaVersion.getVersion();
-            if (version < 9) {
-                expectedJavaVersionEnum = JavaVersion.toVersion("1." + version);
-            } else {
-                expectedJavaVersionEnum = JavaVersion.toVersion(Integer.toString(version));
-            }
-            if (javaVersionEnum != expectedJavaVersionEnum) {
-                String message = String.format(
-                    Locale.ROOT,
-                    "The environment variable JAVA%d_HOME must be set to a JDK installation directory for Java"
-                        + " %s but is [%s] corresponding to [%s]",
-                    version,
-                    expectedJavaVersionEnum,
-                    javaHome,
-                    javaVersionEnum
-                );
-                throw new GradleException(message);
-            }
-        }
-
-        writeToFile(compilerVersionFile.getAsFile().get(), compilerJavaVersionEnum.name());
-        writeToFile(runtimeVersionFile.getAsFile().get(), runtimeJavaVersionEnum.name());
-    }
-
-    private void writeToFile(File file, String content) {
-        try (Writer writer = new FileWriter(file)) {
-            writer.write(content);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
-    /**
-     * Finds printable java version of the given JAVA_HOME
-     */
-    private String findJavaVersionDetails(File javaHome) {
-        String versionInfoScript = "print("
-            + "java.lang.System.getProperty(\"java.vendor.version\", java.lang.System.getProperty(\"java.vendor\")) + \" \" + "
-            + "java.lang.System.getProperty(\"java.version\") + \" [\" + "
-            + "java.lang.System.getProperty(\"java.vm.name\") + \" \" + "
-            + "java.lang.System.getProperty(\"java.vm.version\") + \"]\");";
-        return runJavaAsScript(javaHome, versionInfoScript).trim();
-    }
-
-    /**
-     * Finds the parsable java specification version
-     */
-    private String findJavaSpecificationVersion(File javaHome) {
-        String versionScript = "print(java.lang.System.getProperty(\"java.specification.version\"));";
-        return runJavaAsScript(javaHome, versionScript);
-    }
-
-    /**
-     * Runs the given javascript using jjs from the jdk, and returns the output
-     */
-    private String runJavaAsScript(File javaHome, String script) {
-        ByteArrayOutputStream stdout = new ByteArrayOutputStream();
-        ByteArrayOutputStream stderr = new ByteArrayOutputStream();
-        if (OS.current() == OS.WINDOWS) {
-            // gradle/groovy does not properly escape the double quote for windows
-            script = script.replace("\"", "\\\"");
-        }
-        File jrunscriptPath = new File(javaHome, "bin/jrunscript");
-        String finalScript = script;
-        ExecResult result = getProject().exec(spec -> {
-            spec.setExecutable(jrunscriptPath);
-            spec.args("-e", finalScript);
-            spec.setStandardOutput(stdout);
-            spec.setErrorOutput(stderr);
-            spec.setIgnoreExitValue(true);
-        });
-
-        if (result.getExitValue() != 0) {
-            getLogger().error("STDOUT:");
-            Arrays.stream(stdout.toString(UTF_8).split(System.getProperty("line.separator"))).forEach(getLogger()::error);
-            getLogger().error("STDERR:");
-            Arrays.stream(stderr.toString(UTF_8).split(System.getProperty("line.separator"))).forEach(getLogger()::error);
-            result.rethrowFailure();
-        }
-        return stdout.toString(UTF_8).trim();
-    }
-
-}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
index 431e66e9cde94..21642b30c0480 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
@@ -1,18 +1,27 @@
 package org.elasticsearch.gradle.info;
 
 import org.elasticsearch.gradle.OS;
+import org.elasticsearch.gradle.util.Util;
 import org.gradle.api.GradleException;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
 import org.gradle.internal.jvm.Jvm;
+import org.gradle.jvm.toolchain.JavaInstallation;
+import org.gradle.jvm.toolchain.JavaInstallationRegistry;
+import org.gradle.util.GradleVersion;
 
+import javax.inject.Inject;
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -33,102 +42,168 @@
 import java.util.stream.Stream;
 
 public class GlobalBuildInfoPlugin implements Plugin<Project> {
-    private static final String GLOBAL_INFO_EXTENSION_NAME = "globalInfo";
+    private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
     private static Integer _defaultParallel = null;
 
+    private final JavaInstallationRegistry javaInstallationRegistry;
+    private final ObjectFactory objects;
+    private final ProviderFactory providers;
+
+    @Inject
+    public GlobalBuildInfoPlugin(JavaInstallationRegistry javaInstallationRegistry, ObjectFactory objects, ProviderFactory providers) {
+        this.javaInstallationRegistry = javaInstallationRegistry;
+        this.objects = objects;
+        this.providers = providers;
+    }
+
     @Override
     public void apply(Project project) {
         if (project != project.getRootProject()) {
             throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
         }
 
-        GlobalInfoExtension extension = project.getExtensions().create(GLOBAL_INFO_EXTENSION_NAME, GlobalInfoExtension.class);
-
-        JavaVersion minimumCompilerVersion = JavaVersion.toVersion(getResourceContents("/minimumCompilerVersion"));
-        JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion"));
+        JavaVersion minimumCompilerVersion = JavaVersion.toVersion(Util.getResourceContents("/minimumCompilerVersion"));
+        JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(Util.getResourceContents("/minimumRuntimeVersion"));
 
         File compilerJavaHome = findCompilerJavaHome();
         File runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome);
 
-        String testSeedProperty = System.getProperty("tests.seed");
-        final String testSeed;
-        if (testSeedProperty == null) {
-            long seed = new Random(System.currentTimeMillis()).nextLong();
-            testSeed = Long.toUnsignedString(seed, 16).toUpperCase(Locale.ROOT);
-        } else {
-            testSeed = testSeedProperty;
-        }
-
-        final String buildSnapshotSystemProperty = System.getProperty("build.snapshot", "true");
-        final boolean isSnapshotBuild;
-        switch (buildSnapshotSystemProperty) {
-            case "true":
-                isSnapshotBuild = true;
-                break;
-            case "false":
-                isSnapshotBuild = false;
-                break;
-            default:
-                throw new IllegalArgumentException(
-                    "build.snapshot was set to [" + buildSnapshotSystemProperty + "] but can only be unset or [true|false]"
-                );
-        }
-        final List<JavaHome> javaVersions = new ArrayList<>();
-        for (int version = 8; version <= Integer.parseInt(minimumCompilerVersion.getMajorVersion()); version++) {
-            if (System.getenv(getJavaHomeEnvVarName(Integer.toString(version))) != null) {
-                javaVersions.add(JavaHome.of(version, new File(findJavaHome(Integer.toString(version)))));
-            }
-        }
-
-        GenerateGlobalBuildInfoTask generateTask = project.getTasks()
-            .create("generateGlobalBuildInfo", GenerateGlobalBuildInfoTask.class, task -> {
-                task.setJavaVersions(javaVersions);
-                task.setMinimumCompilerVersion(minimumCompilerVersion);
-                task.setMinimumRuntimeVersion(minimumRuntimeVersion);
-                task.setCompilerJavaHome(compilerJavaHome);
-                task.setRuntimeJavaHome(runtimeJavaHome);
-                task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info"));
-                task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version"));
-                task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version"));
-            });
-
-        PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> {
-            task.getBuildInfoFile().set(generateTask.getOutputFile());
-            task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile());
-            task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile());
-            task.setGlobalInfoListeners(extension.listeners);
-        });
-
         // Initialize global build parameters
         BuildParams.init(params -> {
             params.reset();
             params.setCompilerJavaHome(compilerJavaHome);
             params.setRuntimeJavaHome(runtimeJavaHome);
+            params.setCompilerJavaVersion(determineJavaVersion("compiler java.home", compilerJavaHome, minimumCompilerVersion));
+            params.setRuntimeJavaVersion(determineJavaVersion("runtime java.home", runtimeJavaHome, minimumRuntimeVersion));
             params.setIsRutimeJavaHomeSet(compilerJavaHome.equals(runtimeJavaHome) == false);
-            params.setJavaVersions(javaVersions);
+            params.setJavaVersions(getAvailableJavaVersions(minimumCompilerVersion));
             params.setMinimumCompilerVersion(minimumCompilerVersion);
             params.setMinimumRuntimeVersion(minimumRuntimeVersion);
             params.setGradleJavaVersion(Jvm.current().getJavaVersion());
             params.setGitRevision(gitRevision(project.getRootProject().getRootDir()));
             params.setBuildDate(ZonedDateTime.now(ZoneOffset.UTC));
-            params.setTestSeed(testSeed);
+            params.setTestSeed(getTestSeed());
             params.setIsCi(System.getenv("JENKINS_URL") != null);
             params.setIsInternal(GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null);
             params.setDefaultParallel(findDefaultParallel(project));
-            params.setInFipsJvm(isInFipsJvm());
-            params.setIsSnapshotBuild(isSnapshotBuild);
+            params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
+            params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));
         });
 
-        project.allprojects(
-            p -> {
-                // Make sure than any task execution generates and prints build info
-                p.getTasks().configureEach(task -> {
-                    if (task != generateTask && task != printTask) {
-                        task.dependsOn(printTask);
+        // Print global build info header just before task execution
+        project.getGradle().getTaskGraph().whenReady(graph -> logGlobalBuildInfo());
+    }
+
+    private void logGlobalBuildInfo() {
+        final String osName = System.getProperty("os.name");
+        final String osVersion = System.getProperty("os.version");
+        final String osArch = System.getProperty("os.arch");
+        final Jvm gradleJvm = Jvm.current();
+        final String gradleJvmDetails = getJavaInstallation(gradleJvm.getJavaHome()).getImplementationName();
+
+        LOGGER.quiet("=======================================");
+        LOGGER.quiet("Elasticsearch Build Hamster says Hello!");
+        LOGGER.quiet("  Gradle Version        : " + GradleVersion.current().getVersion());
+        LOGGER.quiet("  OS Info               : " + osName + " " + osVersion + " (" + osArch + ")");
+        if (Jvm.current().getJavaVersion().equals(BuildParams.getCompilerJavaVersion()) == false || BuildParams.getIsRuntimeJavaHomeSet()) {
+            String compilerJvmDetails = getJavaInstallation(BuildParams.getCompilerJavaHome()).getImplementationName();
+            String runtimeJvmDetails = getJavaInstallation(BuildParams.getRuntimeJavaHome()).getImplementationName();
+
+            LOGGER.quiet("  Compiler JDK Version  : " + BuildParams.getCompilerJavaVersion() + " (" + compilerJvmDetails + ")");
+            LOGGER.quiet("  Compiler java.home    : " + BuildParams.getCompilerJavaHome());
+            LOGGER.quiet("  Runtime JDK Version   : " + BuildParams.getRuntimeJavaVersion() + " (" + runtimeJvmDetails + ")");
+            LOGGER.quiet("  Runtime java.home     : " + BuildParams.getRuntimeJavaHome());
+            LOGGER.quiet("  Gradle JDK Version    : " + gradleJvm.getJavaVersion() + " (" + gradleJvmDetails + ")");
+            LOGGER.quiet("  Gradle java.home      : " + gradleJvm.getJavaHome());
+        } else {
+            LOGGER.quiet("  JDK Version           : " + gradleJvm.getJavaVersion() + " (" + gradleJvmDetails + ")");
+            LOGGER.quiet("  JAVA_HOME             : " + gradleJvm.getJavaHome());
+        }
+        LOGGER.quiet("  Random Testing Seed   : " + BuildParams.getTestSeed());
+        LOGGER.quiet("  In FIPS 140 mode      : " + BuildParams.isInFipsJvm());
+        LOGGER.quiet("=======================================");
+    }
+
+    private JavaVersion determineJavaVersion(String description, File javaHome, JavaVersion requiredVersion) {
+        JavaInstallation installation = getJavaInstallation(javaHome);
+        JavaVersion actualVersion = installation.getJavaVersion();
+        if (actualVersion.isCompatibleWith(requiredVersion) == false) {
+            throwInvalidJavaHomeException(
+                description,
+                javaHome,
+                Integer.parseInt(requiredVersion.getMajorVersion()),
+                Integer.parseInt(actualVersion.getMajorVersion())
+            );
+        }
+
+        return actualVersion;
+    }
+
+    private JavaInstallation getJavaInstallation(File javaHome) {
+        JavaInstallation installation;
+        if (isCurrentJavaHome(javaHome)) {
+            installation = javaInstallationRegistry.getInstallationForCurrentVirtualMachine().get();
+        } else {
+            installation = javaInstallationRegistry.installationForDirectory(objects.directoryProperty().fileValue(javaHome)).get();
+        }
+
+        return installation;
+    }
+
+    private List<JavaHome> getAvailableJavaVersions(JavaVersion minimumCompilerVersion) {
+        final List<JavaHome> javaVersions = new ArrayList<>();
+        for (int v = 8; v <= Integer.parseInt(minimumCompilerVersion.getMajorVersion()); v++) {
+            int version = v;
+            String javaHomeEnvVarName = getJavaHomeEnvVarName(Integer.toString(version));
+            if (System.getenv(javaHomeEnvVarName) != null) {
+                File javaHomeDirectory = new File(findJavaHome(Integer.toString(version)));
+                Provider<JavaInstallation> javaInstallationProvider = javaInstallationRegistry.installationForDirectory(
+                    objects.directoryProperty().fileValue(javaHomeDirectory)
+                );
+                JavaHome javaHome = JavaHome.of(version, providers.provider(() -> {
+                    int actualVersion = Integer.parseInt(javaInstallationProvider.get().getJavaVersion().getMajorVersion());
+                    if (actualVersion != version) {
+                        throwInvalidJavaHomeException("env variable " + javaHomeEnvVarName, javaHomeDirectory, version, actualVersion);
                     }
-                });
+                    return javaHomeDirectory;
+                }));
+                javaVersions.add(javaHome);
             }
+        }
+        return javaVersions;
+    }
+
+    private static boolean isCurrentJavaHome(File javaHome) {
+        try {
+            return Files.isSameFile(javaHome.toPath(), Jvm.current().getJavaHome().toPath());
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static String getTestSeed() {
+        String testSeedProperty = System.getProperty("tests.seed");
+        final String testSeed;
+        if (testSeedProperty == null) {
+            long seed = new Random(System.currentTimeMillis()).nextLong();
+            testSeed = Long.toUnsignedString(seed, 16).toUpperCase(Locale.ROOT);
+        } else {
+            testSeed = testSeedProperty;
+        }
+        return testSeed;
+    }
+
+    private static void throwInvalidJavaHomeException(String description, File javaHome, int expectedVersion, int actualVersion) {
+        String message = String.format(
+            Locale.ROOT,
+            "The %s must be set to a JDK installation directory for Java %d but is [%s] corresponding to [%s]",
+            description,
+            expectedVersion,
+            javaHome,
+            actualVersion
         );
+
+        throw new GradleException(message);
     }
 
     private static File findCompilerJavaHome() {
@@ -174,28 +249,6 @@ private static String getJavaHomeEnvVarName(String version) {
         return "JAVA" + version + "_HOME";
     }
 
-    private static boolean isInFipsJvm() {
-        return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
-    }
-
-    private static String getResourceContents(String resourcePath) {
-        try (
-            BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)))
-        ) {
-            StringBuilder b = new StringBuilder();
-            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
-                if (b.length() != 0) {
-                    b.append('\n');
-                }
-                b.append(line);
-            }
-
-            return b.toString();
-        } catch (IOException e) {
-            throw new UncheckedIOException("Error trying to read classpath resource: " + resourcePath, e);
-        }
-    }
-
     private static int findDefaultParallel(Project project) {
         // Since it costs IO to compute this, and is done at configuration time we want to cache this if possible
         // It's safe to store this in a static variable since it's just a primitive so leaking memory isn't an issue
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java
deleted file mode 100644
index a2daa4a5767c0..0000000000000
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.elasticsearch.gradle.info;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class GlobalInfoExtension {
-    final List<Runnable> listeners = new ArrayList<>();
-
-    public void ready(Runnable block) {
-        listeners.add(block);
-    }
-}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java
deleted file mode 100644
index fdd79e0fc9cb3..0000000000000
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java
+++ /dev/null
@@ -1,75 +0,0 @@
-package org.elasticsearch.gradle.info;
-
-import org.gradle.api.DefaultTask;
-import org.gradle.api.JavaVersion;
-import org.gradle.api.file.RegularFileProperty;
-import org.gradle.api.model.ObjectFactory;
-import org.gradle.api.resources.TextResource;
-import org.gradle.api.tasks.InputFile;
-import org.gradle.api.tasks.TaskAction;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.List;
-
-public class PrintGlobalBuildInfoTask extends DefaultTask {
-    private final RegularFileProperty buildInfoFile;
-    private final RegularFileProperty compilerVersionFile;
-    private final RegularFileProperty runtimeVersionFile;
-    private List<Runnable> globalInfoListeners = new ArrayList<>();
-
-    @Inject
-    public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) {
-        this.buildInfoFile = objectFactory.fileProperty();
-        this.compilerVersionFile = objectFactory.fileProperty();
-        this.runtimeVersionFile = objectFactory.fileProperty();
-    }
-
-    @InputFile
-    public RegularFileProperty getBuildInfoFile() {
-        return buildInfoFile;
-    }
-
-    @InputFile
-    public RegularFileProperty getCompilerVersionFile() {
-        return compilerVersionFile;
-    }
-
-    @InputFile
-    public RegularFileProperty getRuntimeVersionFile() {
-        return runtimeVersionFile;
-    }
-
-    public void setGlobalInfoListeners(List<Runnable> globalInfoListeners) {
-        this.globalInfoListeners = globalInfoListeners;
-    }
-
-    @TaskAction
-    public void print() {
-        getLogger().quiet("=======================================");
-        getLogger().quiet("Elasticsearch Build Hamster says Hello!");
-        getLogger().quiet(getFileText(getBuildInfoFile()).asString());
-        getLogger().quiet("  Random Testing Seed   : " + BuildParams.getTestSeed());
-        getLogger().quiet("  In FIPS 140 mode      : " + BuildParams.isInFipsJvm());
-        getLogger().quiet("=======================================");
-
-        setGlobalProperties();
-        globalInfoListeners.forEach(Runnable::run);
-
-        // Since all tasks depend on this task, and it always runs for every build, this makes sure that lifecycle tasks will still
-        // correctly report as UP-TO-DATE, since the convention is a lifecycle task (i.e. assemble, build, etc) will only be marked as
-        // UP-TO-DATE if all upstream tasks were also UP-TO-DATE.
-        setDidWork(false);
-    }
-
-    private TextResource getFileText(RegularFileProperty regularFileProperty) {
-        return getProject().getResources().getText().fromFile(regularFileProperty.getAsFile().get());
-    }
-
-    private void setGlobalProperties() {
-        BuildParams.init(params -> {
-            params.setCompilerJavaVersion(JavaVersion.valueOf(getFileText(getCompilerVersionFile()).asString()));
-            params.setRuntimeJavaVersion(JavaVersion.valueOf(getFileText(getRuntimeVersionFile()).asString()));
-        });
-    }
-}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/FilePermissionsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/FilePermissionsTask.java
index 68839f86c1035..35ad889b34125 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/FilePermissionsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/FilePermissionsTask.java
@@ -28,7 +28,7 @@
 import java.util.stream.Collectors;
 
 import org.apache.tools.ant.taskdefs.condition.Os;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.file.FileCollection;
@@ -80,7 +80,7 @@ private static boolean isExecutableFile(File file) {
     @InputFiles
     @SkipWhenEmpty
     public FileCollection getFiles() {
-        return Boilerplate.getJavaSourceSets(getProject())
+        return GradleUtils.getJavaSourceSets(getProject())
             .stream()
             .map(sourceSet -> sourceSet.getAllSource().matching(filesFilter))
             .reduce(FileTree::plus)
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
index a662ff4dbcaeb..3657808b91979 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.gradle.precommit;
 
 import groovy.lang.Closure;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Task;
@@ -65,7 +65,7 @@ public class TestingConventionsTasks extends DefaultTask {
     public TestingConventionsTasks() {
         setDescription("Tests various testing conventions");
         // Run only after everything is compiled
-        Boilerplate.getJavaSourceSets(getProject()).all(sourceSet -> dependsOn(sourceSet.getOutput().getClassesDirs()));
+        GradleUtils.getJavaSourceSets(getProject()).all(sourceSet -> dependsOn(sourceSet.getOutput().getClassesDirs()));
         naming = getProject().container(TestingConventionRule.class);
     }
 
@@ -81,7 +81,7 @@ public Map<String, Set<File>> getClassFilesPerEnabledTask() {
     @Input
     public Map<String, File> getTestClassNames() {
         if (testClassNames == null) {
-            testClassNames = Boilerplate.getJavaSourceSets(getProject())
+            testClassNames = GradleUtils.getJavaSourceSets(getProject())
                 .getByName("test")
                 .getOutput()
                 .getClassesDirs()
@@ -110,7 +110,7 @@ public void naming(Closure<TestingConventionRule> action) {
 
     @Input
     public Set<String> getMainClassNamedLikeTests() {
-        SourceSetContainer javaSourceSets = Boilerplate.getJavaSourceSets(getProject());
+        SourceSetContainer javaSourceSets = GradleUtils.getJavaSourceSets(getProject());
         if (javaSourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME) == null) {
             // some test projects don't have a main source set
             return Collections.emptySet();
@@ -351,7 +351,7 @@ private FileCollection getTestsClassPath() {
         // running the tests.
         return getProject().files(
             getProject().getConfigurations().getByName("testRuntime").resolve(),
-            Boilerplate.getJavaSourceSets(getProject())
+            GradleUtils.getJavaSourceSets(getProject())
                 .stream()
                 .flatMap(sourceSet -> sourceSet.getOutput().getClassesDirs().getFiles().stream())
                 .collect(Collectors.toList())
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
index b09441148558f..c63726cfadb93 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.gradle.docker.DockerSupportPlugin;
 import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.elasticsearch.gradle.info.BuildParams;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.elasticsearch.gradle.vagrant.BatsProgressLogger;
 import org.elasticsearch.gradle.vagrant.VagrantBasePlugin;
 import org.elasticsearch.gradle.vagrant.VagrantExtension;
@@ -90,7 +90,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(DistributionDownloadPlugin.class);
         project.getPluginManager().apply("elasticsearch.build");
 
-        Provider<DockerSupportService> dockerSupport = Boilerplate.getBuildService(
+        Provider<DockerSupportService> dockerSupport = GradleUtils.getBuildService(
             project.getGradle().getSharedServices(),
             DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
         );
@@ -108,10 +108,12 @@ public void apply(Project project) {
         TaskProvider<Copy> copyUpgradeTask = configureCopyUpgradeTask(project, upgradeVersion, upgradeDir);
         TaskProvider<Copy> copyPluginsTask = configureCopyPluginsTask(project, pluginsDir);
 
+        Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks = lifecyleTasks(project, "distructiveDistroTest");
         TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
         for (ElasticsearchDistribution distribution : distributions) {
             TaskProvider<?> destructiveTask = configureDistroTest(project, distribution, dockerSupport);
             destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
+            lifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask));
         }
         Map<String, TaskProvider<?>> batsTests = new HashMap<>();
         configureBatsTest(project, "plugins", distributionsDir, copyDistributionsTask, copyPluginsTask).configure(
@@ -127,6 +129,7 @@ public void apply(Project project) {
             List<Object> vmDependencies = new ArrayList<>(configureVM(vmProject));
             vmDependencies.add(project.getConfigurations().getByName("testRuntimeClasspath"));
 
+            Map<ElasticsearchDistribution.Type, TaskProvider<?>> vmLifecyleTasks = lifecyleTasks(vmProject, "distroTest");
             TaskProvider<Task> distroTest = vmProject.getTasks().register("distroTest");
             for (ElasticsearchDistribution distribution : distributions) {
                 String destructiveTaskName = destructiveDistroTestTaskName(distribution);
@@ -141,6 +144,7 @@ public void apply(Project project) {
                     );
                     vmTask.configure(t -> t.dependsOn(distribution));
 
+                    vmLifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(vmTask));
                     distroTest.configure(t -> {
                         // Only VM sub-projects that are specifically opted-in to testing Docker should
                         // have the Docker task added as a dependency. Although we control whether Docker
@@ -151,6 +155,7 @@ public void apply(Project project) {
                         //
                         // The shouldTestDocker property could be null, hence we use Boolean.TRUE.equals()
                         boolean shouldExecute = distribution.getType() != Type.DOCKER
+
                             || Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker"));
 
                         if (shouldExecute) {
@@ -170,6 +175,17 @@ public void apply(Project project) {
         });
     }
 
+    private static Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks(Project project, String taskPrefix) {
+        Map<ElasticsearchDistribution.Type, TaskProvider<?>> lifecyleTasks = new HashMap<>();
+
+        lifecyleTasks.put(Type.DOCKER, project.getTasks().register(taskPrefix + ".docker"));
+        lifecyleTasks.put(Type.ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
+        lifecyleTasks.put(Type.DEB, project.getTasks().register(taskPrefix + ".packages"));
+        lifecyleTasks.put(Type.RPM, lifecyleTasks.get(Type.DEB));
+
+        return lifecyleTasks;
+    }
+
     private static Jdk createJdk(
         NamedDomainObjectContainer<Jdk> jdksContainer,
         String name,
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
index 1cb3034419bbe..a059400844c3d 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
@@ -20,7 +20,7 @@
 
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.info.BuildParams;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
@@ -203,6 +203,6 @@ private File getTestOutputResourceDir() {
     }
 
     private SourceSet getTestSourceSet() {
-        return Boilerplate.getJavaSourceSets(getProject()).findByName("test");
+        return GradleUtils.getJavaSourceSets(getProject()).findByName("test");
     }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index 4c86b86b56f5b..0d067b40edb29 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -20,7 +20,7 @@
 
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.info.BuildParams;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
@@ -146,6 +146,6 @@ void copy() {
     }
 
     private SourceSet getTestSourceSet() {
-        return Boilerplate.getJavaSourceSets(getProject()).findByName("test");
+        return GradleUtils.getJavaSourceSets(getProject()).findByName("test");
     }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java
index 5cd88ea01a32a..62427bc1d0a66 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RestTestRunnerTask.java
@@ -1,6 +1,6 @@
 package org.elasticsearch.gradle.testclusters;
 
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.services.internal.BuildServiceRegistryInternal;
 import org.gradle.api.tasks.CacheableTask;
@@ -62,7 +62,7 @@ public Collection<ElasticsearchCluster> getClusters() {
     public List<ResourceLock> getSharedResources() {
         List<ResourceLock> locks = new ArrayList<>(super.getSharedResources());
         BuildServiceRegistryInternal serviceRegistry = getServices().get(BuildServiceRegistryInternal.class);
-        Provider<TestClustersThrottle> throttleProvider = Boilerplate.getBuildService(serviceRegistry, THROTTLE_SERVICE_NAME);
+        Provider<TestClustersThrottle> throttleProvider = GradleUtils.getBuildService(serviceRegistry, THROTTLE_SERVICE_NAME);
         SharedResource resource = serviceRegistry.forService(throttleProvider);
 
         int nodeCount = clusters.stream().mapToInt(cluster -> cluster.getNodes().size()).sum();
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
index ebdf24d0a318f..48c436c42e03b 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.gradle.DistributionDownloadPlugin;
 import org.elasticsearch.gradle.ReaperPlugin;
 import org.elasticsearch.gradle.ReaperService;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -36,7 +36,7 @@
 
 import java.io.File;
 
-import static org.elasticsearch.gradle.tool.Boilerplate.noop;
+import static org.elasticsearch.gradle.util.GradleUtils.noop;
 
 public class TestClustersPlugin implements Plugin<Project> {
 
@@ -102,7 +102,7 @@ public void apply(Project project) {
                 throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
             }
 
-            Provider<TestClustersRegistry> registryProvider = Boilerplate.getBuildService(
+            Provider<TestClustersRegistry> registryProvider = GradleUtils.getBuildService(
                 project.getGradle().getSharedServices(),
                 REGISTRY_SERVICE_NAME
             );
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java
index 64ea96e9f8827..88e744043b747 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.gradle.docker.DockerSupportService;
 import org.elasticsearch.gradle.info.BuildParams;
 import org.elasticsearch.gradle.precommit.TestingConventionsTasks;
-import org.elasticsearch.gradle.tool.Boilerplate;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Action;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Plugin;
@@ -67,7 +67,7 @@ public void apply(Project project) {
             .getSharedServices()
             .registerIfAbsent(DOCKER_COMPOSE_THROTTLE, DockerComposeThrottle.class, spec -> spec.getMaxParallelUsages().set(1));
 
-        Provider<DockerSupportService> dockerSupport = Boilerplate.getBuildService(
+        Provider<DockerSupportService> dockerSupport = GradleUtils.getBuildService(
             project.getGradle().getSharedServices(),
             DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
         );
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java b/buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java
similarity index 98%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java
rename to buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java
index 85002b9c49821..81de7ea8176c3 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle.tool;
+package org.elasticsearch.gradle.util;
 
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
@@ -36,7 +36,7 @@
 
 import java.util.Optional;
 
-public abstract class Boilerplate {
+public abstract class GradleUtils {
 
     public static <T> Action<T> noop() {
         return t -> {};
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Util.java b/buildSrc/src/main/java/org/elasticsearch/gradle/util/Util.java
similarity index 60%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/Util.java
rename to buildSrc/src/main/java/org/elasticsearch/gradle/util/Util.java
index 9ca4efc8b9f37..03ebe906625b6 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/Util.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/util/Util.java
@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -17,10 +17,15 @@
  * under the License.
  */
 
-package org.elasticsearch.gradle;
+package org.elasticsearch.gradle.util;
 
+import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
 import org.gradle.api.GradleException;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
 import java.util.Locale;
 
 public class Util {
@@ -39,6 +44,24 @@ public static boolean getBooleanProperty(String property, boolean defaultValue)
         }
     }
 
+    public static String getResourceContents(String resourcePath) {
+        try (
+            BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)))
+        ) {
+            StringBuilder b = new StringBuilder();
+            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
+                if (b.length() != 0) {
+                    b.append('\n');
+                }
+                b.append(line);
+            }
+
+            return b.toString();
+        } catch (IOException e) {
+            throw new UncheckedIOException("Error trying to read classpath resource: " + resourcePath, e);
+        }
+    }
+
     public static String capitalize(String s) {
         return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
     }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java
index 945a5d4ed0374..507b977569c0c 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.gradle.LoggedExec;
 import org.elasticsearch.gradle.LoggingOutputStream;
 import org.elasticsearch.gradle.ReaperService;
-import org.elasticsearch.gradle.Util;
+import org.elasticsearch.gradle.util.Util;
 import org.gradle.api.Action;
 import org.gradle.api.Project;
 import org.gradle.internal.logging.progress.ProgressLogger;
diff --git a/buildSrc/src/main/resources/minimumGradleVersion b/buildSrc/src/main/resources/minimumGradleVersion
index 30bc70ba1bdc2..1f2be2a26c7b7 100644
--- a/buildSrc/src/main/resources/minimumGradleVersion
+++ b/buildSrc/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-6.2.2
\ No newline at end of file
+6.3
\ No newline at end of file
diff --git a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java
index a76413d6b47d4..624a3e6f5af61 100644
--- a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java
+++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java
@@ -3,11 +3,6 @@
 import org.gradle.api.JavaVersion;
 
 import java.io.File;
-import java.lang.annotation.Documented;
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
 import java.lang.reflect.Modifier;
 import java.time.ZonedDateTime;
 import java.util.Arrays;
@@ -75,12 +70,10 @@ public static JavaVersion getGradleJavaVersion() {
         return value(gradleJavaVersion);
     }
 
-    @ExecutionTime
     public static JavaVersion getCompilerJavaVersion() {
         return value(compilerJavaVersion);
     }
 
-    @ExecutionTime
     public static JavaVersion getRuntimeJavaVersion() {
         return value(runtimeJavaVersion);
     }
@@ -120,22 +113,13 @@ public static boolean isSnapshotBuild() {
     private static <T> T value(T object) {
         if (object == null) {
             String callingMethod = Thread.currentThread().getStackTrace()[2].getMethodName();
-            boolean executionTime;
-            try {
-                executionTime = BuildParams.class.getMethod(callingMethod).getAnnotation(ExecutionTime.class) != null;
-            } catch (NoSuchMethodException e) {
-                throw new RuntimeException(e);
-            }
 
-            String message = "Build parameter '" + propertyName(callingMethod) + "' has not been initialized. ";
-            if (executionTime) {
-                message += "This property is initialized at execution time, "
-                    + "please ensure you are not attempting to access it during project configuration.";
-            } else {
-                message += "Perhaps the plugin responsible for initializing this property has not been applied.";
-            }
-
-            throw new IllegalStateException(message);
+            throw new IllegalStateException(
+                "Build parameter '"
+                    + propertyName(callingMethod)
+                    + "' has not been initialized.\n"
+                    + "Perhaps the plugin responsible for initializing this property has not been applied."
+            );
         }
 
         return object;
@@ -236,14 +220,4 @@ public void setIsSnapshotBuild(final boolean isSnapshotBuild) {
         }
 
     }
-
-    /**
-     * Indicates that a build parameter is initialized at task execution time and is not available at project configuration time.
-     * Attempts to read an uninitialized parameter wil result in an {@link IllegalStateException}.
-     */
-    @Target({ ElementType.METHOD, ElementType.FIELD })
-    @Retention(RetentionPolicy.RUNTIME)
-    @Documented
-    public @interface ExecutionTime {
-    }
 }
diff --git a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/JavaHome.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/JavaHome.java
index 29ca2bafc79dc..1fe376545eaac 100644
--- a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/JavaHome.java
+++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/JavaHome.java
@@ -1,35 +1,27 @@
 package org.elasticsearch.gradle.info;
 
-import org.gradle.api.tasks.Input;
-import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.Optional;
-import org.gradle.api.tasks.PathSensitive;
-import org.gradle.api.tasks.PathSensitivity;
+import org.gradle.api.provider.Provider;
 
 import java.io.File;
 
 public class JavaHome {
     private Integer version;
-    private File javaHome;
+    private Provider<File> javaHome;
 
-    private JavaHome(int version, File javaHome) {
+    private JavaHome(int version, Provider<File> javaHome) {
         this.version = version;
         this.javaHome = javaHome;
     }
 
-    public static JavaHome of(int version, File javaHome) {
+    public static JavaHome of(int version, Provider<File> javaHome) {
         return new JavaHome(version, javaHome);
     }
 
-    @Input
     public Integer getVersion() {
         return version;
     }
 
-    @InputDirectory
-    @Optional
-    @PathSensitive(PathSensitivity.RELATIVE)
-    public File getJavaHome() {
+    public Provider<File> getJavaHome() {
         return javaHome;
     }
 }
diff --git a/buildSrc/src/testKit/jdk-download/reuse/build.gradle b/buildSrc/src/testKit/jdk-download/reuse/build.gradle
index c8b08171091b1..39a5c3372d5f5 100644
--- a/buildSrc/src/testKit/jdk-download/reuse/build.gradle
+++ b/buildSrc/src/testKit/jdk-download/reuse/build.gradle
@@ -7,6 +7,6 @@ jdks {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "linux"
-    architeecture = "x64"
+    architecture = "x64"
   }
 }
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index ee2ce1b8d67b3..2ffc9101b9d80 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.5.0-snapshot-7f057455901
+lucene            = 8.5.0
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 14+36
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java
index 745f247ad1ba5..0dcb2effe571c 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java
@@ -52,6 +52,6 @@ protected void doExecute(Task task, SearchRequest request, ActionListener<Search
             new InternalAggregations(Collections.emptyList()),
             new Suggest(Collections.emptyList()),
             new SearchProfileShardResults(Collections.emptyMap()), false, false, 1),
-            "", 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY));
+            "", 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null));
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java
index ab47b1c5e792a..bbf9c15e20cdf 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchClient.java
@@ -21,7 +21,10 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.asyncsearch.AsyncSearchResponse;
+import org.elasticsearch.client.asyncsearch.DeleteAsyncSearchRequest;
+import org.elasticsearch.client.asyncsearch.GetAsyncSearchRequest;
 import org.elasticsearch.client.asyncsearch.SubmitAsyncSearchRequest;
+import org.elasticsearch.client.core.AcknowledgedResponse;
 
 import java.io.IOException;
 
@@ -42,7 +45,7 @@ public class AsyncSearchClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request, RequestOptions options) throws IOException {
+    public AsyncSearchResponse submit(SubmitAsyncSearchRequest request, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(request, AsyncSearchRequestConverters::submitAsyncSearch, options,
                 AsyncSearchResponse::fromXContent, emptySet());
     }
@@ -57,10 +60,61 @@ public AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request, R
      * @param listener the listener to be notified upon request completion
      * @return cancellable that may be used to cancel the request
      */
-    public Cancellable submitAsyncSearchAsync(SubmitAsyncSearchRequest request, RequestOptions options,
+    public Cancellable submitAsync(SubmitAsyncSearchRequest request, RequestOptions options,
                                                ActionListener<AsyncSearchResponse> listener) {
         return restHighLevelClient.performRequestAsyncAndParseEntity(request, AsyncSearchRequestConverters::submitAsyncSearch, options,
                 AsyncSearchResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Get an async search request.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> the docs</a> for more.
+     *
+     */
+    public AsyncSearchResponse get(GetAsyncSearchRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, AsyncSearchRequestConverters::getAsyncSearch, options,
+                AsyncSearchResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously get an async search request.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     * @return cancellable that may be used to cancel the request
+     */
+    public Cancellable getAsync(GetAsyncSearchRequest request, RequestOptions options,
+            ActionListener<AsyncSearchResponse> listener) {
+        return restHighLevelClient.performRequestAsyncAndParseEntity(request, AsyncSearchRequestConverters::getAsyncSearch, options,
+                AsyncSearchResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Delete an async search request.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse delete(DeleteAsyncSearchRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, AsyncSearchRequestConverters::deleteAsyncSearch, options,
+                AcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously delete an async search request.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html"> the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     * @return cancellable that may be used to cancel the request
+     */
+    public Cancellable deleteAsync(DeleteAsyncSearchRequest request, RequestOptions options,
+                                               ActionListener<AcknowledgedResponse> listener) {
+        return restHighLevelClient.performRequestAsyncAndParseEntity(request, AsyncSearchRequestConverters::deleteAsyncSearch, options,
+                AcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java
index 2d91bd926ef4d..c5e70815752fd 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/AsyncSearchRequestConverters.java
@@ -19,8 +19,12 @@
 
 package org.elasticsearch.client;
 
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.elasticsearch.client.RequestConverters.Params;
+import org.elasticsearch.client.asyncsearch.DeleteAsyncSearchRequest;
+import org.elasticsearch.client.asyncsearch.GetAsyncSearchRequest;
 import org.elasticsearch.client.asyncsearch.SubmitAsyncSearchRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 
@@ -43,14 +47,14 @@ static Request submitAsyncSearch(SubmitAsyncSearchRequest asyncSearchRequest) th
             request.setEntity(RequestConverters.createEntity(asyncSearchRequest.getSearchSource(), REQUEST_BODY_CONTENT_TYPE));
         }
         // set async search submit specific parameters
-        if (asyncSearchRequest.isCleanOnCompletion() != null) {
-            params.putParam("clean_on_completion", asyncSearchRequest.isCleanOnCompletion().toString());
+        if (asyncSearchRequest.isKeepOnCompletion() != null) {
+            params.putParam("keep_on_completion", asyncSearchRequest.isKeepOnCompletion().toString());
         }
         if (asyncSearchRequest.getKeepAlive() != null) {
             params.putParam("keep_alive", asyncSearchRequest.getKeepAlive().getStringRep());
         }
-        if (asyncSearchRequest.getWaitForCompletion() != null) {
-            params.putParam("wait_for_completion", asyncSearchRequest.getWaitForCompletion().getStringRep());
+        if (asyncSearchRequest.getWaitForCompletionTimeout() != null) {
+            params.putParam("wait_for_completion_timeout", asyncSearchRequest.getWaitForCompletionTimeout().getStringRep());
         }
         request.addParameters(params.asMap());
         return request;
@@ -71,4 +75,29 @@ static void addSearchRequestParams(Params params, SubmitAsyncSearchRequest reque
         }
         params.withBatchedReduceSize(request.getBatchedReduceSize());
     }
+
+    static Request getAsyncSearch(GetAsyncSearchRequest asyncSearchRequest) {
+        String endpoint = new RequestConverters.EndpointBuilder()
+                .addPathPartAsIs("_async_search")
+                .addPathPart(asyncSearchRequest.getId())
+                .build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Params params = new RequestConverters.Params();
+        if (asyncSearchRequest.getKeepAlive() != null) {
+            params.putParam("keep_alive", asyncSearchRequest.getKeepAlive().getStringRep());
+        }
+        if (asyncSearchRequest.getWaitForCompletion() != null) {
+            params.putParam("wait_for_completion_timeout", asyncSearchRequest.getWaitForCompletion().getStringRep());
+        }
+        request.addParameters(params.asMap());
+        return request;
+    }
+
+    static Request deleteAsyncSearch(DeleteAsyncSearchRequest deleteAsyncSearchRequest) {
+        String endpoint = new RequestConverters.EndpointBuilder()
+                .addPathPartAsIs("_async_search")
+                .addPathPart(deleteAsyncSearchRequest.getId())
+                .build();
+        return new Request(HttpDelete.METHOD_NAME, endpoint);
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
index bf220d63b3c63..54dd11bf6caff 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.client.ml.DeleteJobRequest;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
 import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
+import org.elasticsearch.client.ml.EstimateModelMemoryRequest;
 import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
 import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.FindFileStructureRequest;
@@ -593,6 +594,17 @@ static Request deleteCalendarEvent(DeleteCalendarEventRequest deleteCalendarEven
         return new Request(HttpDelete.METHOD_NAME, endpoint);
     }
 
+    static Request estimateModelMemory(EstimateModelMemoryRequest estimateModelMemoryRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml")
+            .addPathPartAsIs("anomaly_detectors")
+            .addPathPartAsIs("_estimate_model_memory")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(estimateModelMemoryRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request putDataFrameAnalytics(PutDataFrameAnalyticsRequest putRequest) throws IOException {
         String endpoint = new EndpointBuilder()
             .addPathPartAsIs("_ml", "data_frame", "analytics")
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
index 504cbc541f073..61d4b52db2d6d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
@@ -23,6 +23,8 @@
 import org.elasticsearch.client.ml.CloseJobRequest;
 import org.elasticsearch.client.ml.CloseJobResponse;
 import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
+import org.elasticsearch.client.ml.EstimateModelMemoryRequest;
+import org.elasticsearch.client.ml.EstimateModelMemoryResponse;
 import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse;
 import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
@@ -1951,6 +1953,48 @@ public Cancellable setUpgradeModeAsync(SetUpgradeModeRequest request, RequestOpt
             Collections.emptySet());
     }
 
+    /**
+     * Estimate the model memory an analysis config is likely to need given supplied field cardinalities
+     * <p>
+     * For additional info
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-estimate-model-memory.html">Estimate Model Memory</a>
+     *
+     * @param request The {@link EstimateModelMemoryRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return {@link EstimateModelMemoryResponse} response object
+     */
+    public EstimateModelMemoryResponse estimateModelMemory(EstimateModelMemoryRequest request,
+                                                           RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::estimateModelMemory,
+            options,
+            EstimateModelMemoryResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Estimate the model memory an analysis config is likely to need given supplied field cardinalities and notifies listener upon
+     * completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-estimate-model-memory.html">Estimate Model Memory</a>
+     *
+     * @param request The {@link EstimateModelMemoryRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     * @return cancellable that may be used to cancel the request
+     */
+    public Cancellable estimateModelMemoryAsync(EstimateModelMemoryRequest request,
+                                                RequestOptions options,
+                                                ActionListener<EstimateModelMemoryResponse> listener) {
+        return restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::estimateModelMemory,
+            options,
+            EstimateModelMemoryResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
     /**
      * Creates a new Data Frame Analytics config
      * <p>
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index 0bb0eca83a389..d0cd3ea5a0091 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -410,7 +410,9 @@ static void addSearchRequestParams(Params params, SearchRequest searchRequest) {
         params.withIndicesOptions(searchRequest.indicesOptions());
         params.withSearchType(searchRequest.searchType().name().toLowerCase(Locale.ROOT));
         params.putParam("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips()));
-        params.putParam("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
+        if (searchRequest.getPreFilterShardSize() != null) {
+            params.putParam("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
+        }
         params.withMaxConcurrentShardRequests(searchRequest.getMaxConcurrentShardRequests());
         if (searchRequest.requestCache() != null) {
             params.withRequestCache(searchRequest.requestCache());
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java
new file mode 100644
index 0000000000000..3b37293212da0
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.client.Validatable;
+
+import java.util.Objects;
+
+public class DeleteAsyncSearchRequest implements Validatable {
+
+    private final String id;
+
+    public DeleteAsyncSearchRequest(String id) {
+        this.id = id;
+}
+
+    public String getId() {
+        return this.id;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        DeleteAsyncSearchRequest request = (DeleteAsyncSearchRequest) o;
+        return Objects.equals(getId(), request.getId());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getId());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java
new file mode 100644
index 0000000000000..11ad059349481
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.util.Objects;
+import java.util.Optional;
+
+public class GetAsyncSearchRequest implements Validatable {
+
+    private TimeValue waitForCompletion;
+    private TimeValue keepAlive;
+
+    public static final long MIN_KEEPALIVE = TimeValue.timeValueMinutes(1).millis();
+
+    private final String id;
+
+    public GetAsyncSearchRequest(String id) {
+        this.id = id;
+    }
+
+    public String getId() {
+        return this.id;
+    }
+
+    public TimeValue getWaitForCompletion() {
+        return waitForCompletion;
+    }
+
+    public void setWaitForCompletion(TimeValue waitForCompletion) {
+        this.waitForCompletion = waitForCompletion;
+    }
+
+    public TimeValue getKeepAlive() {
+        return keepAlive;
+    }
+
+    public void setKeepAlive(TimeValue keepAlive) {
+        this.keepAlive = keepAlive;
+    }
+
+    @Override
+    public Optional<ValidationException> validate() {
+        final ValidationException validationException = new ValidationException();
+        if (keepAlive != null && keepAlive.getMillis() < MIN_KEEPALIVE) {
+            validationException.addValidationError("keep_alive must be greater than 1 minute, got: " + keepAlive.toString());
+        }
+        if (validationException.validationErrors().isEmpty()) {
+            return Optional.empty();
+        }
+        return Optional.of(validationException);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GetAsyncSearchRequest request = (GetAsyncSearchRequest) o;
+        return Objects.equals(getId(), request.getId())
+                && Objects.equals(getKeepAlive(), request.getKeepAlive())
+                && Objects.equals(getWaitForCompletion(), request.getWaitForCompletion());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getId(), getKeepAlive(), getWaitForCompletion());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java
index 1b0a07c4dea41..65892fb562dd8 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java
@@ -43,8 +43,8 @@ public class SubmitAsyncSearchRequest implements Validatable {
 
     public static long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis();
 
-    private TimeValue waitForCompletion;
-    private Boolean cleanOnCompletion;
+    private TimeValue waitForCompletionTimeout;
+    private Boolean keepOnCompletion;
     private TimeValue keepAlive;
     private final SearchRequest searchRequest;
 
@@ -70,29 +70,29 @@ public String[] getIndices() {
     /**
      * Get the minimum time that the request should wait before returning a partial result (defaults to 1 second).
      */
-    public TimeValue getWaitForCompletion() {
-        return waitForCompletion;
+    public TimeValue getWaitForCompletionTimeout() {
+        return waitForCompletionTimeout;
     }
 
     /**
      * Sets the minimum time that the request should wait before returning a partial result (defaults to 1 second).
      */
-    public void setWaitForCompletion(TimeValue waitForCompletion) {
-        this.waitForCompletion = waitForCompletion;
+    public void setWaitForCompletionTimeout(TimeValue waitForCompletionTimeout) {
+        this.waitForCompletionTimeout = waitForCompletionTimeout;
     }
 
     /**
      * Returns whether the resource resource should be removed on completion or failure (defaults to true).
      */
-    public Boolean isCleanOnCompletion() {
-        return cleanOnCompletion;
+    public Boolean isKeepOnCompletion() {
+        return keepOnCompletion;
     }
 
     /**
      * Determines if the resource should be removed on completion or failure (defaults to true).
      */
-    public void setCleanOnCompletion(boolean cleanOnCompletion) {
-        this.cleanOnCompletion = cleanOnCompletion;
+    public void setKeepOnCompletion(boolean keepOnCompletion) {
+        this.keepOnCompletion = keepOnCompletion;
     }
 
     /**
@@ -273,12 +273,12 @@ public boolean equals(Object o) {
         SubmitAsyncSearchRequest request = (SubmitAsyncSearchRequest) o;
         return Objects.equals(searchRequest, request.searchRequest)
                 && Objects.equals(getKeepAlive(), request.getKeepAlive())
-                && Objects.equals(getWaitForCompletion(), request.getWaitForCompletion())
-                && Objects.equals(isCleanOnCompletion(), request.isCleanOnCompletion());
+                && Objects.equals(getWaitForCompletionTimeout(), request.getWaitForCompletionTimeout())
+                && Objects.equals(isKeepOnCompletion(), request.isKeepOnCompletion());
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(searchRequest, getKeepAlive(), getWaitForCompletion(), isCleanOnCompletion());
+        return Objects.hash(searchRequest, getKeepAlive(), getWaitForCompletionTimeout(), isKeepOnCompletion());
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java
index 5e59b4b19dbbe..dc332fa8a4ab0 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java
@@ -31,8 +31,10 @@ public abstract class IndexerJobStats {
     public static ParseField NUM_INVOCATIONS = new ParseField("trigger_count");
     public static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
     public static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
+    public static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
     public static ParseField INDEX_TOTAL = new ParseField("index_total");
     public static ParseField SEARCH_TOTAL = new ParseField("search_total");
+    public static ParseField PROCESSING_TOTAL = new ParseField("processing_total");
     public static ParseField SEARCH_FAILURES = new ParseField("search_failures");
     public static ParseField INDEX_FAILURES = new ParseField("index_failures");
 
@@ -44,11 +46,14 @@ public abstract class IndexerJobStats {
     protected final long indexTotal;
     protected final long searchTime;
     protected final long searchTotal;
+    protected final long processingTime;
+    protected final long processingTotal;
     protected final long indexFailures;
     protected final long searchFailures;
 
     public IndexerJobStats(long numPages, long numInputDocuments, long numOutputDocuments, long numInvocations,
-                           long indexTime, long searchTime, long indexTotal, long searchTotal, long indexFailures, long searchFailures) {
+                           long indexTime, long searchTime, long processingTime, long indexTotal, long searchTotal, long processingTotal,
+                           long indexFailures, long searchFailures) {
         this.numPages = numPages;
         this.numInputDocuments = numInputDocuments;
         this.numOuputDocuments = numOutputDocuments;
@@ -57,6 +62,8 @@ public IndexerJobStats(long numPages, long numInputDocuments, long numOutputDocu
         this.indexTotal = indexTotal;
         this.searchTime = searchTime;
         this.searchTotal = searchTotal;
+        this.processingTime = processingTime;
+        this.processingTotal = processingTotal;
         this.indexFailures = indexFailures;
         this.searchFailures = searchFailures;
     }
@@ -117,6 +124,13 @@ public long getSearchTime() {
         return searchTime;
     }
 
+    /**
+     * Returns the time spent processing (cumulative) in milliseconds
+     */
+    public long getProcessingTime() {
+        return processingTime;
+    }
+
     /**
      * Returns the total number of indexing requests that have been processed
      * (Note: this is not the number of _documents_ that have been indexed)
@@ -132,6 +146,14 @@ public long getSearchTotal() {
         return searchTotal;
     }
 
+    /**
+     * Returns the total number of processing runs that have been made
+     */
+    public long getProcessingTotal() {
+        return processingTotal;
+    }
+
+
     @Override
     public boolean equals(Object other) {
         if (this == other) {
@@ -149,16 +171,19 @@ public boolean equals(Object other) {
                 && Objects.equals(this.numInvocations, that.numInvocations)
                 && Objects.equals(this.indexTime, that.indexTime)
                 && Objects.equals(this.searchTime, that.searchTime)
+                && Objects.equals(this.processingTime, that.processingTime)
                 && Objects.equals(this.indexFailures, that.indexFailures)
                 && Objects.equals(this.searchFailures, that.searchFailures)
                 && Objects.equals(this.searchTotal, that.searchTotal)
+                && Objects.equals(this.processingTotal, that.processingTotal)
                 && Objects.equals(this.indexTotal, that.indexTotal);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
-                indexTime, searchTime, indexFailures, searchFailures, searchTotal, indexTotal);
+                indexTime, searchTime, processingTime, indexFailures, searchFailures, searchTotal,
+                indexTotal, processingTotal);
     }
 
     @Override
@@ -172,6 +197,8 @@ public final String toString() {
                 + ", index_time_in_ms=" + indexTime
                 + ", index_total=" + indexTotal
                 + ", search_time_in_ms=" + searchTime
-                + ", search_total=" + searchTotal+ "}";
+                + ", search_total=" + searchTotal
+                + ", processing_time_in_ms=" + processingTime
+                + ", processing_total=" + processingTotal + "}";
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateModelMemoryRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateModelMemoryRequest.java
new file mode 100644
index 0000000000000..b0dc8bb7c294e
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateModelMemoryRequest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.client.ml.job.config.AnalysisConfig;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Request to estimate the model memory an analysis config is likely to need given supplied field cardinalities.
+ */
+public class EstimateModelMemoryRequest implements Validatable, ToXContentObject {
+
+    public static final String ANALYSIS_CONFIG = "analysis_config";
+    public static final String OVERALL_CARDINALITY = "overall_cardinality";
+    public static final String MAX_BUCKET_CARDINALITY = "max_bucket_cardinality";
+
+    private final AnalysisConfig analysisConfig;
+    private Map<String, Long> overallCardinality = Collections.emptyMap();
+    private Map<String, Long> maxBucketCardinality = Collections.emptyMap();
+
+    @Override
+    public Optional<ValidationException> validate() {
+        return Optional.empty();
+    }
+
+    public EstimateModelMemoryRequest(AnalysisConfig analysisConfig) {
+        this.analysisConfig = Objects.requireNonNull(analysisConfig);
+    }
+
+    public AnalysisConfig getAnalysisConfig() {
+        return analysisConfig;
+    }
+
+    public Map<String, Long> getOverallCardinality() {
+        return overallCardinality;
+    }
+
+    public void setOverallCardinality(Map<String, Long> overallCardinality) {
+        this.overallCardinality = Collections.unmodifiableMap(overallCardinality);
+    }
+
+    public Map<String, Long> getMaxBucketCardinality() {
+        return maxBucketCardinality;
+    }
+
+    public void setMaxBucketCardinality(Map<String, Long> maxBucketCardinality) {
+        this.maxBucketCardinality = Collections.unmodifiableMap(maxBucketCardinality);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ANALYSIS_CONFIG, analysisConfig);
+        if (overallCardinality.isEmpty() == false) {
+            builder.field(OVERALL_CARDINALITY, overallCardinality);
+        }
+        if (maxBucketCardinality.isEmpty() == false) {
+            builder.field(MAX_BUCKET_CARDINALITY, maxBucketCardinality);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(analysisConfig, overallCardinality, maxBucketCardinality);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        EstimateModelMemoryRequest that = (EstimateModelMemoryRequest) other;
+        return Objects.equals(analysisConfig, that.analysisConfig) &&
+            Objects.equals(overallCardinality, that.overallCardinality) &&
+            Objects.equals(maxBucketCardinality, that.maxBucketCardinality);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateModelMemoryResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateModelMemoryResponse.java
new file mode 100644
index 0000000000000..02b5c03d9b44a
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateModelMemoryResponse.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class EstimateModelMemoryResponse {
+
+    public static final ParseField MODEL_MEMORY_ESTIMATE = new ParseField("model_memory_estimate");
+
+    static final ConstructingObjectParser<EstimateModelMemoryResponse, Void> PARSER =
+        new ConstructingObjectParser<>(
+            "estimate_model_memory",
+            true,
+            args -> new EstimateModelMemoryResponse((String) args[0]));
+
+    static {
+        PARSER.declareString(constructorArg(), MODEL_MEMORY_ESTIMATE);
+    }
+
+    public static EstimateModelMemoryResponse fromXContent(final XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    private final ByteSizeValue modelMemoryEstimate;
+
+    public EstimateModelMemoryResponse(String modelMemoryEstimate) {
+        this.modelMemoryEstimate = ByteSizeValue.parseBytesSizeValue(modelMemoryEstimate, MODEL_MEMORY_ESTIMATE.getPreferredName());
+    }
+
+    /**
+     * @return An estimate of the model memory the supplied analysis config is likely to need given the supplied field cardinalities.
+     */
+    public ByteSizeValue getModelMemoryEstimate() {
+        return modelMemoryEstimate;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        EstimateModelMemoryResponse other = (EstimateModelMemoryResponse) o;
+        return Objects.equals(this.modelMemoryEstimate, other.modelMemoryEstimate);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(modelMemoryEstimate);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
index 70169ce09b278..acdb9cccca1eb 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.client.ml.NodeAttributes;
 import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.client.ml.dataframe.stats.common.DataCounts;
 import org.elasticsearch.client.ml.dataframe.stats.common.MemoryUsage;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -47,6 +48,7 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
     static final ParseField STATE = new ParseField("state");
     static final ParseField FAILURE_REASON = new ParseField("failure_reason");
     static final ParseField PROGRESS = new ParseField("progress");
+    static final ParseField DATA_COUNTS = new ParseField("data_counts");
     static final ParseField MEMORY_USAGE = new ParseField("memory_usage");
     static final ParseField ANALYSIS_STATS = new ParseField("analysis_stats");
     static final ParseField NODE = new ParseField("node");
@@ -60,10 +62,11 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
                 (DataFrameAnalyticsState) args[1],
                 (String) args[2],
                 (List<PhaseProgress>) args[3],
-                (MemoryUsage) args[4],
-                (AnalysisStats) args[5],
-                (NodeAttributes) args[6],
-                (String) args[7]));
+                (DataCounts) args[4],
+                (MemoryUsage) args[5],
+                (AnalysisStats) args[6],
+                (NodeAttributes) args[7],
+                (String) args[8]));
 
     static {
         PARSER.declareString(constructorArg(), ID);
@@ -75,6 +78,7 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
         }, STATE, ObjectParser.ValueType.STRING);
         PARSER.declareString(optionalConstructorArg(), FAILURE_REASON);
         PARSER.declareObjectArray(optionalConstructorArg(), PhaseProgress.PARSER, PROGRESS);
+        PARSER.declareObject(optionalConstructorArg(), DataCounts.PARSER, DATA_COUNTS);
         PARSER.declareObject(optionalConstructorArg(), MemoryUsage.PARSER, MEMORY_USAGE);
         PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseAnalysisStats(p), ANALYSIS_STATS);
         PARSER.declareObject(optionalConstructorArg(), NodeAttributes.PARSER, NODE);
@@ -93,19 +97,21 @@ private static AnalysisStats parseAnalysisStats(XContentParser parser) throws IO
     private final DataFrameAnalyticsState state;
     private final String failureReason;
     private final List<PhaseProgress> progress;
+    private final DataCounts dataCounts;
     private final MemoryUsage memoryUsage;
     private final AnalysisStats analysisStats;
     private final NodeAttributes node;
     private final String assignmentExplanation;
 
     public DataFrameAnalyticsStats(String id, DataFrameAnalyticsState state, @Nullable String failureReason,
-                                   @Nullable List<PhaseProgress> progress, @Nullable MemoryUsage memoryUsage,
-                                   @Nullable AnalysisStats analysisStats, @Nullable NodeAttributes node,
+                                   @Nullable List<PhaseProgress> progress, @Nullable DataCounts dataCounts,
+                                   @Nullable MemoryUsage memoryUsage, @Nullable AnalysisStats analysisStats, @Nullable NodeAttributes node,
                                    @Nullable String assignmentExplanation) {
         this.id = id;
         this.state = state;
         this.failureReason = failureReason;
         this.progress = progress;
+        this.dataCounts = dataCounts;
         this.memoryUsage = memoryUsage;
         this.analysisStats = analysisStats;
         this.node = node;
@@ -128,6 +134,11 @@ public List<PhaseProgress> getProgress() {
         return progress;
     }
 
+    @Nullable
+    public DataCounts getDataCounts() {
+        return dataCounts;
+    }
+
     @Nullable
     public MemoryUsage getMemoryUsage() {
         return memoryUsage;
@@ -156,6 +167,7 @@ public boolean equals(Object o) {
             && Objects.equals(state, other.state)
             && Objects.equals(failureReason, other.failureReason)
             && Objects.equals(progress, other.progress)
+            && Objects.equals(dataCounts, other.dataCounts)
             && Objects.equals(memoryUsage, other.memoryUsage)
             && Objects.equals(analysisStats, other.analysisStats)
             && Objects.equals(node, other.node)
@@ -164,7 +176,7 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, state, failureReason, progress, memoryUsage, analysisStats, node, assignmentExplanation);
+        return Objects.hash(id, state, failureReason, progress, dataCounts, memoryUsage, analysisStats, node, assignmentExplanation);
     }
 
     @Override
@@ -174,6 +186,7 @@ public String toString() {
             .add("state", state)
             .add("failureReason", failureReason)
             .add("progress", progress)
+            .add("dataCounts", dataCounts)
             .add("memoryUsage", memoryUsage)
             .add("analysisStats", analysisStats)
             .add("node", node)
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCounts.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCounts.java
new file mode 100644
index 0000000000000..b7a90b1f0b5c6
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCounts.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe.stats.common;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.inject.internal.ToStringBuilder;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class DataCounts implements ToXContentObject {
+
+    public static final String TYPE_VALUE = "analytics_data_counts";
+
+    public static final ParseField TRAINING_DOCS_COUNT = new ParseField("training_docs_count");
+    public static final ParseField TEST_DOCS_COUNT = new ParseField("test_docs_count");
+    public static final ParseField SKIPPED_DOCS_COUNT = new ParseField("skipped_docs_count");
+
+    public static final ConstructingObjectParser<DataCounts, Void> PARSER = new ConstructingObjectParser<>(TYPE_VALUE, true,
+        a -> {
+            Long trainingDocsCount = (Long) a[0];
+            Long testDocsCount = (Long) a[1];
+            Long skippedDocsCount = (Long) a[2];
+            return new DataCounts(
+                getOrDefault(trainingDocsCount, 0L),
+                getOrDefault(testDocsCount, 0L),
+                getOrDefault(skippedDocsCount, 0L)
+            );
+        });
+
+    static {
+        PARSER.declareLong(optionalConstructorArg(), TRAINING_DOCS_COUNT);
+        PARSER.declareLong(optionalConstructorArg(), TEST_DOCS_COUNT);
+        PARSER.declareLong(optionalConstructorArg(), SKIPPED_DOCS_COUNT);
+    }
+
+    private final long trainingDocsCount;
+    private final long testDocsCount;
+    private final long skippedDocsCount;
+
+    public DataCounts(long trainingDocsCount, long testDocsCount, long skippedDocsCount) {
+        this.trainingDocsCount = trainingDocsCount;
+        this.testDocsCount = testDocsCount;
+        this.skippedDocsCount = skippedDocsCount;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(TRAINING_DOCS_COUNT.getPreferredName(), trainingDocsCount);
+        builder.field(TEST_DOCS_COUNT.getPreferredName(), testDocsCount);
+        builder.field(SKIPPED_DOCS_COUNT.getPreferredName(), skippedDocsCount);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DataCounts that = (DataCounts) o;
+        return trainingDocsCount == that.trainingDocsCount
+            && testDocsCount == that.testDocsCount
+            && skippedDocsCount == that.skippedDocsCount;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(trainingDocsCount, testDocsCount, skippedDocsCount);
+    }
+
+    @Override
+    public String toString() {
+        return new ToStringBuilder(getClass())
+            .add(TRAINING_DOCS_COUNT.getPreferredName(), trainingDocsCount)
+            .add(TEST_DOCS_COUNT.getPreferredName(), testDocsCount)
+            .add(SKIPPED_DOCS_COUNT.getPreferredName(), skippedDocsCount)
+            .toString();
+    }
+
+    public long getTrainingDocsCount() {
+        return trainingDocsCount;
+    }
+
+    public long getTestDocsCount() {
+        return testDocsCount;
+    }
+
+    public long getSkippedDocsCount() {
+        return skippedDocsCount;
+    }
+
+    private static <T> T getOrDefault(@Nullable T value, T defaultValue) {
+        return value != null ? value : defaultValue;
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
index 15098e3770f3b..e3236dad0cd26 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
@@ -36,7 +36,7 @@ public class OutlierDetectionStats implements AnalysisStats {
 
     public static final ParseField TIMESTAMP = new ParseField("timestamp");
     public static final ParseField PARAMETERS = new ParseField("parameters");
-    public static final ParseField TIMING_STATS = new ParseField("timings_stats");
+    public static final ParseField TIMING_STATS = new ParseField("timing_stats");
 
     public static final ConstructingObjectParser<OutlierDetectionStats, Void> PARSER = new ConstructingObjectParser<>(
         NAME.getPreferredName(), true,
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupJobResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupJobResponse.java
index e63daf5949002..9ded34aa05670 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupJobResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupJobResponse.java
@@ -177,16 +177,18 @@ public final String toString() {
     public static class RollupIndexerJobStats extends IndexerJobStats {
 
         RollupIndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations,
-                              long indexTime, long indexTotal, long searchTime, long searchTotal, long indexFailures, long searchFailures) {
+                              long indexTime, long indexTotal, long searchTime, long searchTotal, long processingTime,
+                              long processingTotal, long indexFailures, long searchFailures) {
             super(numPages, numInputDocuments, numOuputDocuments, numInvocations,
-                    indexTime, searchTime, indexTotal, searchTotal, indexFailures, searchFailures);
+                    indexTime, searchTime, processingTime, indexTotal, searchTotal, processingTotal, indexFailures, searchFailures);
         }
 
         private static final ConstructingObjectParser<RollupIndexerJobStats, Void> PARSER = new ConstructingObjectParser<>(
                 STATS.getPreferredName(),
                 true,
                 args -> new RollupIndexerJobStats((long) args[0], (long) args[1], (long) args[2], (long) args[3],
-                    (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9]));
+                    (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9],
+                    (long) args[10], (long) args[11]));
         static {
             PARSER.declareLong(constructorArg(), NUM_PAGES);
             PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
@@ -196,6 +198,8 @@ public static class RollupIndexerJobStats extends IndexerJobStats {
             PARSER.declareLong(constructorArg(), INDEX_TOTAL);
             PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
             PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
+            PARSER.declareLong(constructorArg(), PROCESSING_TIME_IN_MS);
+            PARSER.declareLong(constructorArg(), PROCESSING_TOTAL);
             PARSER.declareLong(constructorArg(), INDEX_FAILURES);
             PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
         }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java
index 2a04c6ea45eb5..e3a0032e55b0d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java
@@ -27,7 +27,6 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class TransformIndexerStats extends IndexerJobStats {
@@ -39,21 +38,38 @@ public class TransformIndexerStats extends IndexerJobStats {
     public static final ConstructingObjectParser<TransformIndexerStats, Void> LENIENT_PARSER = new ConstructingObjectParser<>(
         NAME,
         true,
-        args -> new TransformIndexerStats((long) args[0], (long) args[1], (long) args[2],
-            (long) args[3], (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9],
-            (Double) args[10], (Double) args[11], (Double) args[12]));
+        args -> new TransformIndexerStats(
+            unboxSafe(args[0], 0L),
+            unboxSafe(args[1], 0L),
+            unboxSafe(args[2], 0L),
+            unboxSafe(args[3], 0L),
+            unboxSafe(args[4], 0L),
+            unboxSafe(args[5], 0L),
+            unboxSafe(args[6], 0L),
+            unboxSafe(args[7], 0L),
+            unboxSafe(args[8], 0L),
+            unboxSafe(args[9], 0L),
+            unboxSafe(args[10], 0L),
+            unboxSafe(args[11], 0L),
+            unboxSafe(args[12], 0.0),
+            unboxSafe(args[13], 0.0),
+            unboxSafe(args[14], 0.0)
+        )
+    );
 
     static {
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_PAGES);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_INVOCATIONS);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_TOTAL);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_FAILURES);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_PAGES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INPUT_DOCUMENTS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_OUTPUT_DOCUMENTS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INVOCATIONS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_FAILURES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_FAILURES);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_INDEXED);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_PROCESSED);
@@ -67,16 +83,40 @@ public static TransformIndexerStats fromXContent(XContentParser parser) throws I
     private final double expAvgDocumentsIndexed;
     private final double expAvgDocumentsProcessed;
 
-    public TransformIndexerStats(long numPages, long numInputDocuments, long numOuputDocuments,
-                                 long numInvocations, long indexTime, long searchTime,
-                                 long indexTotal, long searchTotal, long indexFailures, long searchFailures,
-                                 Double expAvgCheckpointDurationMs, Double expAvgDocumentsIndexed,
-                                 Double expAvgDocumentsProcessed) {
-        super(numPages, numInputDocuments, numOuputDocuments, numInvocations, indexTime, searchTime,
-                indexTotal, searchTotal, indexFailures, searchFailures);
-        this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs == null ? 0.0 : expAvgCheckpointDurationMs;
-        this.expAvgDocumentsIndexed = expAvgDocumentsIndexed == null ? 0.0 : expAvgDocumentsIndexed;
-        this.expAvgDocumentsProcessed = expAvgDocumentsProcessed == null ? 0.0 : expAvgDocumentsProcessed;
+    public TransformIndexerStats(
+        long numPages,
+        long numInputDocuments,
+        long numOuputDocuments,
+        long numInvocations,
+        long indexTime,
+        long searchTime,
+        long processingTime,
+        long indexTotal,
+        long searchTotal,
+        long processingTotal,
+        long indexFailures,
+        long searchFailures,
+        double expAvgCheckpointDurationMs,
+        double expAvgDocumentsIndexed,
+        double expAvgDocumentsProcessed
+    ) {
+        super(
+            numPages,
+            numInputDocuments,
+            numOuputDocuments,
+            numInvocations,
+            indexTime,
+            searchTime,
+            processingTime,
+            indexTotal,
+            searchTotal,
+            processingTotal,
+            indexFailures,
+            searchFailures
+        );
+        this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs;
+        this.expAvgDocumentsIndexed = expAvgDocumentsIndexed;
+        this.expAvgDocumentsProcessed = expAvgDocumentsProcessed;
     }
 
     public double getExpAvgCheckpointDurationMs() {
@@ -109,10 +149,12 @@ public boolean equals(Object other) {
             && Objects.equals(this.numInvocations, that.numInvocations)
             && Objects.equals(this.indexTime, that.indexTime)
             && Objects.equals(this.searchTime, that.searchTime)
+            && Objects.equals(this.processingTime, that.processingTime)
             && Objects.equals(this.indexFailures, that.indexFailures)
             && Objects.equals(this.searchFailures, that.searchFailures)
             && Objects.equals(this.indexTotal, that.indexTotal)
             && Objects.equals(this.searchTotal, that.searchTotal)
+            && Objects.equals(this.processingTotal, that.processingTotal)
             && Objects.equals(this.expAvgCheckpointDurationMs, that.expAvgCheckpointDurationMs)
             && Objects.equals(this.expAvgDocumentsIndexed, that.expAvgDocumentsIndexed)
             && Objects.equals(this.expAvgDocumentsProcessed, that.expAvgDocumentsProcessed);
@@ -120,8 +162,31 @@ public boolean equals(Object other) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
-            indexTime, searchTime, indexFailures, searchFailures, indexTotal, searchTotal,
-            expAvgCheckpointDurationMs, expAvgDocumentsIndexed, expAvgDocumentsProcessed);
+        return Objects.hash(
+            numPages,
+            numInputDocuments,
+            numOuputDocuments,
+            numInvocations,
+            indexTime,
+            searchTime,
+            processingTime,
+            indexFailures,
+            searchFailures,
+            indexTotal,
+            searchTotal,
+            processingTotal,
+            expAvgCheckpointDurationMs,
+            expAvgDocumentsIndexed,
+            expAvgDocumentsProcessed
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T> T unboxSafe(Object l, T default_value) {
+        if (l == null) {
+            return default_value;
+        } else {
+            return (T) l;
+        }
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java
index df8fc65c751e6..9a863c0ef66ac 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/AsyncSearchRequestConvertersTests.java
@@ -19,8 +19,12 @@
 
 package org.elasticsearch.client;
 
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.client.asyncsearch.DeleteAsyncSearchRequest;
+import org.elasticsearch.client.asyncsearch.GetAsyncSearchRequest;
 import org.elasticsearch.client.asyncsearch.SubmitAsyncSearchRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -52,9 +56,9 @@ public void testSubmitAsyncSearch() throws Exception {
         setRandomIndicesOptions(submitRequest::setIndicesOptions, submitRequest::getIndicesOptions, expectedParams);
 
         if (randomBoolean()) {
-            boolean cleanOnCompletion = randomBoolean();
-            submitRequest.setCleanOnCompletion(cleanOnCompletion);
-            expectedParams.put("clean_on_completion", Boolean.toString(cleanOnCompletion));
+            boolean keepOnCompletion = randomBoolean();
+            submitRequest.setKeepOnCompletion(keepOnCompletion);
+            expectedParams.put("keep_on_completion", Boolean.toString(keepOnCompletion));
         }
         if (randomBoolean()) {
             TimeValue keepAlive = TimeValue.parseTimeValue(randomTimeValue(), "test");
@@ -62,9 +66,9 @@ public void testSubmitAsyncSearch() throws Exception {
             expectedParams.put("keep_alive", keepAlive.getStringRep());
         }
         if (randomBoolean()) {
-            TimeValue waitForCompletion = TimeValue.parseTimeValue(randomTimeValue(), "test");
-            submitRequest.setWaitForCompletion(waitForCompletion);
-            expectedParams.put("wait_for_completion", waitForCompletion.getStringRep());
+            TimeValue waitForCompletionTimeout = TimeValue.parseTimeValue(randomTimeValue(), "test");
+            submitRequest.setWaitForCompletionTimeout(waitForCompletionTimeout);
+            expectedParams.put("wait_for_completion_timeout", waitForCompletionTimeout.getStringRep());
         }
 
         Request request = AsyncSearchRequestConverters.submitAsyncSearch(submitRequest);
@@ -112,4 +116,35 @@ private static void setRandomSearchParams(SubmitAsyncSearchRequest request, Map<
         expectedParams.put("max_concurrent_shard_requests", Integer.toString(request.getMaxConcurrentShardRequests()));
     }
 
+    public void testGetAsyncSearch() throws Exception {
+        String id = randomAlphaOfLengthBetween(5, 10);
+        Map<String, String> expectedParams = new HashMap<>();
+        GetAsyncSearchRequest submitRequest = new GetAsyncSearchRequest(id);
+        if (randomBoolean()) {
+            TimeValue keepAlive = TimeValue.parseTimeValue(randomTimeValue(), "test");
+            submitRequest.setKeepAlive(keepAlive);
+            expectedParams.put("keep_alive", keepAlive.getStringRep());
+        }
+        if (randomBoolean()) {
+            TimeValue waitForCompletion = TimeValue.parseTimeValue(randomTimeValue(), "test");
+            submitRequest.setWaitForCompletion(waitForCompletion);
+            expectedParams.put("wait_for_completion_timeout", waitForCompletion.getStringRep());
+        }
+
+        Request request = AsyncSearchRequestConverters.getAsyncSearch(submitRequest);
+        String endpoint = "/_async_search/" + id;
+        assertEquals(HttpGet.METHOD_NAME, request.getMethod());
+        assertEquals(endpoint.toString(), request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+    }
+
+    public void testDeleteAsyncSearch() throws Exception {
+        String id = randomAlphaOfLengthBetween(5, 10);
+        DeleteAsyncSearchRequest deleteRequest = new DeleteAsyncSearchRequest(id);
+
+        Request request = AsyncSearchRequestConverters.deleteAsyncSearch(deleteRequest);
+        assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
+        assertEquals("/_async_search/" + id, request.getEndpoint());
+        assertTrue(request.getParameters().isEmpty());
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
index 7f8ed9e61bd1f..86b14a7e8a745 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.client.ml.DeleteJobRequest;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
 import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
+import org.elasticsearch.client.ml.EstimateModelMemoryRequest;
 import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
 import org.elasticsearch.client.ml.EvaluateDataFrameRequestTests;
 import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
@@ -107,6 +108,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -695,6 +697,25 @@ public void testDeleteCalendarEvent() {
         assertEquals("/_ml/calendars/" + calendarId + "/events/" + eventId, request.getEndpoint());
     }
 
+    public void testEstimateModelMemory() throws Exception {
+        String byFieldName = randomAlphaOfLength(10);
+        String influencerFieldName = randomAlphaOfLength(10);
+        AnalysisConfig analysisConfig = AnalysisConfig.builder(
+            Collections.singletonList(
+                Detector.builder().setFunction("count").setByFieldName(byFieldName).build()
+            )).setInfluencers(Collections.singletonList(influencerFieldName)).build();
+        EstimateModelMemoryRequest estimateModelMemoryRequest = new EstimateModelMemoryRequest(analysisConfig);
+        estimateModelMemoryRequest.setOverallCardinality(Collections.singletonMap(byFieldName, randomNonNegativeLong()));
+        estimateModelMemoryRequest.setMaxBucketCardinality(Collections.singletonMap(influencerFieldName, randomNonNegativeLong()));
+        Request request = MLRequestConverters.estimateModelMemory(estimateModelMemoryRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/anomaly_detectors/_estimate_model_memory", request.getEndpoint());
+
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder = estimateModelMemoryRequest.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals(Strings.toString(builder), requestEntityToString(request));
+    }
+
     public void testPutDataFrameAnalytics() throws IOException {
         PutDataFrameAnalyticsRequest putRequest = new PutDataFrameAnalyticsRequest(randomDataFrameAnalyticsConfig());
         Request request = MLRequestConverters.putDataFrameAnalytics(putRequest);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
index 17a9c0cd1cfca..f66f237d775cf 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -46,6 +46,8 @@
 import org.elasticsearch.client.ml.DeleteJobResponse;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
 import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
+import org.elasticsearch.client.ml.EstimateModelMemoryRequest;
+import org.elasticsearch.client.ml.EstimateModelMemoryResponse;
 import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
 import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
 import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
@@ -874,7 +876,7 @@ private  String createExpiredData(String jobId) throws Exception {
         {
             // Index a randomly named unused state document
             String docId = "non_existing_job_" + randomFrom("model_state_1234567#1", "quantiles", "categorizer_state#1");
-            IndexRequest indexRequest = new IndexRequest(".ml-state").id(docId);
+            IndexRequest indexRequest = new IndexRequest(".ml-state-000001").id(docId);
             indexRequest.source(Collections.emptyMap(), XContentType.JSON);
             indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
             highLevelClient().index(indexRequest, RequestOptions.DEFAULT);
@@ -944,8 +946,8 @@ public void testDeleteExpiredData() throws Exception {
         assertTrue(forecastExists(jobId, forecastId));
 
         {
-            // Verify .ml-state contains the expected unused state document
-            Iterable<SearchHit> hits = searchAll(".ml-state");
+            // Verify .ml-state* contains the expected unused state document
+            Iterable<SearchHit> hits = searchAll(".ml-state*");
             List<SearchHit> target = new ArrayList<>();
             hits.forEach(target::add);
             long numMatches = target.stream()
@@ -974,8 +976,8 @@ public void testDeleteExpiredData() throws Exception {
         assertFalse(forecastExists(jobId, forecastId));
 
         {
-            // Verify .ml-state doesn't contain unused state documents
-            Iterable<SearchHit> hits = searchAll(".ml-state");
+            // Verify .ml-state* doesn't contain unused state documents
+            Iterable<SearchHit> hits = searchAll(".ml-state*");
             List<SearchHit> hitList = new ArrayList<>();
             hits.forEach(hitList::add);
             long numMatches = hitList.stream()
@@ -1244,6 +1246,27 @@ public void testDeleteCalendarEvent() throws IOException {
         assertThat(remainingIds, not(hasItem(deletedEvent)));
     }
 
+    public void testEstimateModelMemory() throws Exception {
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+
+        String byFieldName = randomAlphaOfLength(10);
+        String influencerFieldName = randomAlphaOfLength(10);
+        AnalysisConfig analysisConfig = AnalysisConfig.builder(
+            Collections.singletonList(
+                Detector.builder().setFunction("count").setByFieldName(byFieldName).build()
+            )).setInfluencers(Collections.singletonList(influencerFieldName)).build();
+        EstimateModelMemoryRequest estimateModelMemoryRequest = new EstimateModelMemoryRequest(analysisConfig);
+        estimateModelMemoryRequest.setOverallCardinality(Collections.singletonMap(byFieldName, randomNonNegativeLong()));
+        estimateModelMemoryRequest.setMaxBucketCardinality(Collections.singletonMap(influencerFieldName, randomNonNegativeLong()));
+
+        EstimateModelMemoryResponse estimateModelMemoryResponse = execute(
+            estimateModelMemoryRequest,
+            machineLearningClient::estimateModelMemory, machineLearningClient::estimateModelMemoryAsync);
+
+        ByteSizeValue modelMemoryEstimate = estimateModelMemoryResponse.getModelMemoryEstimate();
+        assertThat(modelMemoryEstimate.getBytes(), greaterThanOrEqualTo(10000000L));
+    }
+
     public void testPutDataFrameAnalyticsConfig_GivenOutlierDetectionAnalysis() throws Exception {
         MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
         String configId = "test-put-df-analytics-outlier-detection";
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index a7291b3dbbcf3..346a6310c5d75 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -1875,7 +1875,9 @@ private static void setRandomSearchParams(SearchRequest searchRequest,
         if (randomBoolean()) {
             searchRequest.setPreFilterShardSize(randomIntBetween(2, Integer.MAX_VALUE));
         }
-        expectedParams.put("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
+        if (searchRequest.getPreFilterShardSize() != null) {
+            expectedParams.put("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
+        }
     }
 
     public static void setRandomIndicesOptions(Consumer<IndicesOptions> setter, Supplier<IndicesOptions> getter,
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index e35adbc7aa5e4..b5ca32db68c2c 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -234,7 +234,7 @@ public void testInfo() throws IOException {
     public void testSearchScroll() throws IOException {
         SearchResponse mockSearchResponse = new SearchResponse(new SearchResponseSections(SearchHits.empty(), InternalAggregations.EMPTY,
                 null, false, false, null, 1), randomAlphaOfLengthBetween(5, 10), 5, 5, 0, 100, ShardSearchFailure.EMPTY_ARRAY,
-                SearchResponse.Clusters.EMPTY);
+                SearchResponse.Clusters.EMPTY, null);
         mockResponse(mockSearchResponse);
         SearchResponse searchResponse = restHighLevelClient.scroll(
                 new SearchScrollRequest(randomAlphaOfLengthBetween(5, 10)), RequestOptions.DEFAULT);
@@ -801,7 +801,8 @@ public void testApiNamingConventions() throws Exception {
             "cluster.delete_component_template",
             "indices.create_data_stream",
             "indices.get_data_streams",
-            "indices.delete_data_stream"
+            "indices.delete_data_stream",
+            "clear_reader"
         };
         //These API are not required for high-level client feature completeness
         String[] notRequiredApi = new String[] {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
index f9679cf5eb61c..44562c2c16156 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
@@ -146,7 +146,7 @@ public void testCleanupRepository() throws IOException {
         assertThat(response.result().blobs(), equalTo(0L));
     }
 
-    public void testCreateSnapshot() throws IOException {
+    public void testCreateSnapshot() throws Exception {
         String repository = "test_repository";
         assertTrue(createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}").isAcknowledged());
 
@@ -163,12 +163,23 @@ public void testCreateSnapshot() throws IOException {
         CreateSnapshotResponse response = createTestSnapshot(request);
         assertEquals(waitForCompletion ? RestStatus.OK : RestStatus.ACCEPTED, response.status());
         if (waitForCompletion == false) {
-            // If we don't wait for the snapshot to complete we have to cancel it to not leak the snapshot task
-            AcknowledgedResponse deleteResponse = execute(
-                    new DeleteSnapshotRequest(repository, snapshot),
-                    highLevelClient().snapshot()::delete, highLevelClient().snapshot()::deleteAsync
-            );
-            assertTrue(deleteResponse.isAcknowledged());
+            // busy assert on the delete because a known race condition could cause the delete request to not see
+            // the snapshot if delete and snapshot finalization happen at the same time
+            // See https://github.com/elastic/elasticsearch/issues/53509#issuecomment-603899620 for details
+            // TODO: Remove busy assert in 7.x+ once this race is fixed
+            assertBusy(() -> {
+                // If we don't wait for the snapshot to complete we have to cancel it to not leak the snapshot task
+                AcknowledgedResponse deleteResponse;
+                try {
+                    deleteResponse = execute(
+                        new DeleteSnapshotRequest(repository, snapshot),
+                        highLevelClient().snapshot()::delete, highLevelClient().snapshot()::deleteAsync
+                    );
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                }
+                assertTrue(deleteResponse.isAcknowledged());
+            });
         }
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java
index 3f93806aca779..94341c41685f0 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java
@@ -441,6 +441,8 @@ public void testGetStats() throws Exception {
             0L,
             0L,
             0L,
+            0L,
+            0L,
             0.0,
             0.0,
             0.0);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
index d0f6ddc93080f..f22b25fda5632 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchIT.java
@@ -21,36 +21,50 @@
 
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.core.AcknowledgedResponse;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
-import java.util.concurrent.TimeUnit;
 
 public class AsyncSearchIT extends ESRestHighLevelClientTestCase {
 
-    public void testSubmitAsyncSearchRequest() throws IOException {
+    public void testAsyncSearch() throws IOException {
         String index = "test-index";
         createIndex(index, Settings.EMPTY);
 
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery());
-        SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(sourceBuilder, index);
-        // 15 sec should be enough to make sure we always complete right away
-        request.setWaitForCompletion(new TimeValue(15, TimeUnit.SECONDS));
-        AsyncSearchResponse response = highLevelClient().asyncSearch().submitAsyncSearch(request, RequestOptions.DEFAULT);
-        assertFalse(response.isPartial());
-        assertTrue(response.getStartTime() > 0);
-        assertTrue(response.getExpirationTime() > 0);
-        assertNotNull(response.getSearchResponse());
-        if (response.isRunning() == false) {
-            assertNull(response.getId());
-            assertFalse(response.isPartial());
+        SubmitAsyncSearchRequest submitRequest = new SubmitAsyncSearchRequest(sourceBuilder, index);
+        submitRequest.setKeepOnCompletion(true);
+        AsyncSearchResponse submitResponse = highLevelClient().asyncSearch().submit(submitRequest, RequestOptions.DEFAULT);
+        assertNotNull(submitResponse.getId());
+        assertFalse(submitResponse.isPartial());
+        assertTrue(submitResponse.getStartTime() > 0);
+        assertTrue(submitResponse.getExpirationTime() > 0);
+        assertNotNull(submitResponse.getSearchResponse());
+        if (submitResponse.isRunning() == false) {
+            assertFalse(submitResponse.isPartial());
         } else {
-            assertTrue(response.isPartial());
-            assertNotNull(response.getId());
+            assertTrue(submitResponse.isPartial());
+        }
+
+        GetAsyncSearchRequest getRequest = new GetAsyncSearchRequest(submitResponse.getId());
+        AsyncSearchResponse getResponse = highLevelClient().asyncSearch().get(getRequest, RequestOptions.DEFAULT);
+        while (getResponse.isRunning()) {
+            getResponse = highLevelClient().asyncSearch().get(getRequest, RequestOptions.DEFAULT);
         }
-    }
 
+        assertFalse(getResponse.isRunning());
+        assertFalse(getResponse.isPartial());
+        assertTrue(getResponse.getStartTime() > 0);
+        assertTrue(getResponse.getExpirationTime() > 0);
+        assertNotNull(getResponse.getSearchResponse());
+
+        DeleteAsyncSearchRequest deleteRequest = new DeleteAsyncSearchRequest(submitResponse.getId());
+        AcknowledgedResponse deleteAsyncSearchResponse = highLevelClient().asyncSearch().delete(deleteRequest,
+                RequestOptions.DEFAULT);
+        assertNotNull(deleteAsyncSearchResponse);
+        assertNotNull(deleteAsyncSearchResponse.isAcknowledged());
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java
index 08c0da25e5bb6..dca2c48b94dbc 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/AsyncSearchResponseTests.java
@@ -45,7 +45,7 @@ protected org.elasticsearch.xpack.core.search.action.AsyncSearchResponse createS
         // add search response, minimal object is okay since the full randomization of parsing is tested in SearchResponseTests
         SearchResponse searchResponse = randomBoolean() ? null
                 : new SearchResponse(InternalSearchResponse.empty(), randomAlphaOfLength(10), 1, 1, 0, randomIntBetween(0, 10000),
-                        ShardSearchFailure.EMPTY_ARRAY, Clusters.EMPTY);
+                        ShardSearchFailure.EMPTY_ARRAY, Clusters.EMPTY, null);
         org.elasticsearch.xpack.core.search.action.AsyncSearchResponse testResponse =
                 new org.elasticsearch.xpack.core.search.action.AsyncSearchResponse(id, searchResponse, error, isPartial, isRunning,
                         startTimeMillis, expirationTimeMillis);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequestTests.java
new file mode 100644
index 0000000000000..b6861b218cd28
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequestTests.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.asyncsearch;
+
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.concurrent.TimeUnit;
+
+public class GetAsyncSearchRequestTests extends ESTestCase {
+
+    public void testValidation() {
+        GetAsyncSearchRequest getAsyncSearchRequest = new GetAsyncSearchRequest(randomAlphaOfLength(10));
+        getAsyncSearchRequest.setKeepAlive(new TimeValue(0));
+        assertTrue(getAsyncSearchRequest.validate().isPresent());
+        ValidationException validationException = getAsyncSearchRequest.validate().get();
+        assertEquals(1, validationException.validationErrors().size());
+        assertEquals("Validation Failed: 1: keep_alive must be greater than 1 minute, got: 0s;", validationException.getMessage());
+
+        getAsyncSearchRequest.setKeepAlive(new TimeValue(1, TimeUnit.MINUTES));
+        assertFalse(getAsyncSearchRequest.validate().isPresent());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
index adf78daacc7d5..bfb3d6d5bee4f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
@@ -49,6 +49,8 @@
 import org.elasticsearch.client.ml.DeleteJobResponse;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
 import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
+import org.elasticsearch.client.ml.EstimateModelMemoryRequest;
+import org.elasticsearch.client.ml.EstimateModelMemoryResponse;
 import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
 import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
 import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
@@ -4133,6 +4135,65 @@ public void onFailure(Exception e) {
         }
     }
 
+    public void testEstimateModelMemory() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        {
+            // tag::estimate-model-memory-request
+            Detector.Builder detectorBuilder = new Detector.Builder()
+                .setFunction("count")
+                .setPartitionFieldName("status");
+            AnalysisConfig.Builder analysisConfigBuilder =
+                new AnalysisConfig.Builder(Collections.singletonList(detectorBuilder.build()))
+                .setBucketSpan(TimeValue.timeValueMinutes(10))
+                .setInfluencers(Collections.singletonList("src_ip"));
+            EstimateModelMemoryRequest request = new EstimateModelMemoryRequest(analysisConfigBuilder.build()); // <1>
+            request.setOverallCardinality(Collections.singletonMap("status", 50L));                             // <2>
+            request.setMaxBucketCardinality(Collections.singletonMap("src_ip", 30L));                           // <3>
+            // end::estimate-model-memory-request
+
+            // tag::estimate-model-memory-execute
+            EstimateModelMemoryResponse estimateModelMemoryResponse =
+                client.machineLearning().estimateModelMemory(request, RequestOptions.DEFAULT);
+            // end::estimate-model-memory-execute
+
+            // tag::estimate-model-memory-response
+            ByteSizeValue modelMemoryEstimate = estimateModelMemoryResponse.getModelMemoryEstimate(); // <1>
+            long estimateInBytes = modelMemoryEstimate.getBytes();
+            // end::estimate-model-memory-response
+            assertThat(estimateInBytes, greaterThan(10000000L));
+        }
+        {
+            AnalysisConfig analysisConfig =
+                AnalysisConfig.builder(Collections.singletonList(Detector.builder().setFunction("count").build())).build();
+            EstimateModelMemoryRequest request = new EstimateModelMemoryRequest(analysisConfig);
+
+            // tag::estimate-model-memory-execute-listener
+            ActionListener<EstimateModelMemoryResponse> listener = new ActionListener<EstimateModelMemoryResponse>() {
+                @Override
+                public void onResponse(EstimateModelMemoryResponse estimateModelMemoryResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::estimate-model-memory-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::estimate-model-memory-execute-async
+            client.machineLearning()
+                .estimateModelMemoryAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::estimate-model-memory-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
     private String createFilter(RestHighLevelClient client) throws IOException {
         MlFilter.Builder filterBuilder = MlFilter.builder("my_safe_domains")
             .setDescription("A list of safe domains")
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
index 25345181982af..d251f568dfa79 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
 import org.elasticsearch.client.ml.dataframe.stats.AnalysisStatsNamedXContentProvider;
 import org.elasticsearch.client.ml.dataframe.stats.classification.ClassificationStatsTests;
+import org.elasticsearch.client.ml.dataframe.stats.common.DataCountsTests;
 import org.elasticsearch.client.ml.dataframe.stats.common.MemoryUsageTests;
 import org.elasticsearch.client.ml.dataframe.stats.outlierdetection.OutlierDetectionStatsTests;
 import org.elasticsearch.client.ml.dataframe.stats.regression.RegressionStatsTests;
@@ -68,6 +69,7 @@ public static DataFrameAnalyticsStats randomDataFrameAnalyticsStats() {
             randomFrom(DataFrameAnalyticsState.values()),
             randomBoolean() ? null : randomAlphaOfLength(10),
             randomBoolean() ? null : createRandomProgress(),
+            randomBoolean() ? null : DataCountsTests.createRandom(),
             randomBoolean() ? null : MemoryUsageTests.createRandom(),
             analysisStats,
             randomBoolean() ? null : NodeAttributesTests.createRandom(),
@@ -93,6 +95,9 @@ public static void toXContent(DataFrameAnalyticsStats stats, XContentBuilder bui
         if (stats.getProgress() != null) {
             builder.field(DataFrameAnalyticsStats.PROGRESS.getPreferredName(), stats.getProgress());
         }
+        if (stats.getDataCounts() != null) {
+            builder.field(DataFrameAnalyticsStats.DATA_COUNTS.getPreferredName(), stats.getDataCounts());
+        }
         if (stats.getMemoryUsage() != null) {
             builder.field(DataFrameAnalyticsStats.MEMORY_USAGE.getPreferredName(), stats.getMemoryUsage());
         }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCountsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCountsTests.java
new file mode 100644
index 0000000000000..5e877e2d40f7b
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/stats/common/DataCountsTests.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe.stats.common;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class DataCountsTests extends AbstractXContentTestCase<DataCounts> {
+
+    @Override
+    protected DataCounts createTestInstance() {
+        return createRandom();
+    }
+
+    public static DataCounts createRandom() {
+        return new DataCounts(
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong()
+        );
+    }
+
+    @Override
+    protected DataCounts doParseInstance(XContentParser parser) throws IOException {
+        return DataCounts.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupJobResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupJobResponseTests.java
index b866420a44c01..122f156986d71 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupJobResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupJobResponseTests.java
@@ -64,8 +64,9 @@ private GetRollupJobResponse createTestInstance() {
 
     private RollupIndexerJobStats randomStats() {
         return new RollupIndexerJobStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+            randomNonNegativeLong());
     }
 
     private RollupJobStatus randomStatus() {
@@ -120,6 +121,8 @@ public void toXContent(RollupIndexerJobStats stats, XContentBuilder builder, ToX
         builder.field(IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
         builder.field(IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
         builder.field(IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
+        builder.field(IndexerJobStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
+        builder.field(IndexerJobStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
         builder.endObject();
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java
index 018cab89b0fc9..e06a7cddb93e9 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java
@@ -31,41 +31,103 @@ public class TransformIndexerStatsTests extends ESTestCase {
 
     public void testFromXContent() throws IOException {
         xContentTester(
-                this::createParser,
-                TransformIndexerStatsTests::randomStats,
-                TransformIndexerStatsTests::toXContent,
-                TransformIndexerStats::fromXContent)
-                .supportsUnknownFields(true)
-                .test();
+            this::createParser,
+            TransformIndexerStatsTests::randomStats,
+            TransformIndexerStatsTests::toXContent,
+            TransformIndexerStats::fromXContent
+        ).supportsUnknownFields(true).test();
     }
 
     public static TransformIndexerStats randomStats() {
-        return new TransformIndexerStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-                randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-                randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble());
+        return new TransformIndexerStats(
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomDouble(),
+            randomDouble(),
+            randomDouble()
+        );
     }
 
     public static void toXContent(TransformIndexerStats stats, XContentBuilder builder) throws IOException {
         builder.startObject();
-        builder.field(IndexerJobStats.NUM_PAGES.getPreferredName(), stats.getNumPages());
-        builder.field(IndexerJobStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments());
-        builder.field(IndexerJobStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments());
-        builder.field(IndexerJobStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations());
-        builder.field(IndexerJobStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
-        builder.field(IndexerJobStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
-        builder.field(IndexerJobStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
-        builder.field(IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
-        builder.field(IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
-        builder.field(IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
-        builder.field(TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
-            stats.getExpAvgCheckpointDurationMs());
-        builder.field(TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(),
-            stats.getExpAvgDocumentsIndexed());
-        builder.field(TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(),
-            stats.getExpAvgDocumentsProcessed());
+        if (randomBoolean()) {
+            builder.field(IndexerJobStats.NUM_PAGES.getPreferredName(), stats.getNumPages());
+            builder.field(IndexerJobStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments());
+            builder.field(IndexerJobStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments());
+            builder.field(IndexerJobStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations());
+            builder.field(IndexerJobStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
+            builder.field(IndexerJobStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
+            builder.field(IndexerJobStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
+            builder.field(IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
+            builder.field(IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
+            builder.field(IndexerJobStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
+            builder.field(IndexerJobStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
+            builder.field(IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
+            builder.field(
+                TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
+                stats.getExpAvgCheckpointDurationMs()
+            );
+            builder.field(TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(), stats.getExpAvgDocumentsIndexed());
+            builder.field(
+                TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(),
+                stats.getExpAvgDocumentsProcessed()
+            );
+        } else {
+            // a toXContent version which leaves out field with value 0 (simulating the case that an older version misses a field)
+            xContentFieldIfNotZero(builder, IndexerJobStats.NUM_PAGES.getPreferredName(), stats.getNumPages());
+            xContentFieldIfNotZero(builder, IndexerJobStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments());
+            xContentFieldIfNotZero(builder, IndexerJobStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments());
+            xContentFieldIfNotZero(builder, IndexerJobStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations());
+            xContentFieldIfNotZero(builder, IndexerJobStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
+            xContentFieldIfNotZero(builder, IndexerJobStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
+            xContentFieldIfNotZero(builder, IndexerJobStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
+            xContentFieldIfNotZero(builder, IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
+            xContentFieldIfNotZero(builder, IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
+            xContentFieldIfNotZero(builder, IndexerJobStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
+            xContentFieldIfNotZero(builder, IndexerJobStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
+            xContentFieldIfNotZero(builder, IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
+            xContentFieldIfNotZero(
+                builder,
+                TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
+                stats.getExpAvgCheckpointDurationMs()
+            );
+            xContentFieldIfNotZero(
+                builder,
+                TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(),
+                stats.getExpAvgDocumentsIndexed()
+            );
+            xContentFieldIfNotZero(
+                builder,
+                TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(),
+                stats.getExpAvgDocumentsProcessed()
+            );
+        }
         builder.endObject();
     }
+
+    private static XContentBuilder xContentFieldIfNotZero(XContentBuilder builder, String name, long value) throws IOException {
+        if (value > 0) {
+            builder.field(name, value);
+        }
+
+        return builder;
+    }
+
+    private static XContentBuilder xContentFieldIfNotZero(XContentBuilder builder, String name, double value) throws IOException {
+        if (value > 0.0) {
+            builder.field(name, value);
+        }
+
+        return builder;
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java
index 50c98bcd8f0b6..eb74164be5364 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java
@@ -30,17 +30,31 @@
 
 public class TransformIndexerStatsTests extends AbstractResponseTestCase<
     org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats,
-        TransformIndexerStats> {
+    TransformIndexerStats> {
+
+    public static org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats randomStats() {
+        return new org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats(
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomDouble(),
+            randomDouble(),
+            randomDouble()
+        );
+    }
 
     @Override
     protected org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats createServerTestInstance(XContentType xContentType) {
-        return new org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats(randomLongBetween(10L, 10000L),
-            randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble());
+        return randomStats();
     }
 
     @Override
@@ -49,8 +63,10 @@ protected TransformIndexerStats doParseToClientInstance(XContentParser parser) t
     }
 
     @Override
-    protected void assertInstances(org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats serverTestInstance,
-                                   TransformIndexerStats clientInstance) {
+    protected void assertInstances(
+        org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats serverTestInstance,
+        TransformIndexerStats clientInstance
+    ) {
         assertTransformIndexerStats(serverTestInstance, clientInstance);
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java
index 63f6bffdf6321..c732c425fe427 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java
@@ -36,40 +36,36 @@
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class TransformStatsTests extends AbstractResponseTestCase<org.elasticsearch.xpack.core.transform.transforms.TransformStats,
+public class TransformStatsTests extends AbstractResponseTestCase<
+    org.elasticsearch.xpack.core.transform.transforms.TransformStats,
     org.elasticsearch.client.transform.transforms.TransformStats> {
 
     public static org.elasticsearch.xpack.core.transform.transforms.NodeAttributes randomNodeAttributes() {
         int numberOfAttributes = randomIntBetween(1, 10);
         Map<String, String> attributes = new HashMap<>(numberOfAttributes);
-        for(int i = 0; i < numberOfAttributes; i++) {
+        for (int i = 0; i < numberOfAttributes; i++) {
             String val = randomAlphaOfLength(10);
-            attributes.put("key-"+i, val);
+            attributes.put("key-" + i, val);
         }
-        return new org.elasticsearch.xpack.core.transform.transforms.NodeAttributes(randomAlphaOfLength(10),
+        return new org.elasticsearch.xpack.core.transform.transforms.NodeAttributes(
             randomAlphaOfLength(10),
             randomAlphaOfLength(10),
             randomAlphaOfLength(10),
-            attributes);
+            randomAlphaOfLength(10),
+            attributes
+        );
     }
 
-    public static org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats randomStats() {
-        return new org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats(randomLongBetween(10L, 10000L),
-            randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble(),
-            randomBoolean() ? null : randomDouble());
-    }
     @Override
     protected org.elasticsearch.xpack.core.transform.transforms.TransformStats createServerTestInstance(XContentType xContentType) {
-        return new org.elasticsearch.xpack.core.transform.transforms.TransformStats(randomAlphaOfLength(10),
+        return new org.elasticsearch.xpack.core.transform.transforms.TransformStats(
+            randomAlphaOfLength(10),
             randomFrom(org.elasticsearch.xpack.core.transform.transforms.TransformStats.State.values()),
             randomBoolean() ? null : randomAlphaOfLength(100),
             randomBoolean() ? null : randomNodeAttributes(),
-            randomStats(),
-            TransformCheckpointingInfoTests.randomTransformCheckpointingInfo());
+            TransformIndexerStatsTests.randomStats(),
+            TransformCheckpointingInfoTests.randomTransformCheckpointingInfo()
+        );
     }
 
     @Override
@@ -78,8 +74,10 @@ protected TransformStats doParseToClientInstance(XContentParser parser) throws I
     }
 
     @Override
-    protected void assertInstances(org.elasticsearch.xpack.core.transform.transforms.TransformStats serverTestInstance,
-                                   TransformStats clientInstance) {
+    protected void assertInstances(
+        org.elasticsearch.xpack.core.transform.transforms.TransformStats serverTestInstance,
+        TransformStats clientInstance
+    ) {
         assertThat(serverTestInstance.getId(), equalTo(clientInstance.getId()));
         assertThat(serverTestInstance.getState().value(), equalTo(clientInstance.getState().value()));
         assertTransformIndexerStats(serverTestInstance.getIndexerStats(), clientInstance.getIndexerStats());
@@ -88,8 +86,10 @@ protected void assertInstances(org.elasticsearch.xpack.core.transform.transforms
         assertThat(serverTestInstance.getReason(), equalTo(clientInstance.getReason()));
     }
 
-    private void assertNodeAttributes(org.elasticsearch.xpack.core.transform.transforms.NodeAttributes serverTestInstance,
-                                      NodeAttributes clientInstance) {
+    private void assertNodeAttributes(
+        org.elasticsearch.xpack.core.transform.transforms.NodeAttributes serverTestInstance,
+        NodeAttributes clientInstance
+    ) {
         if (serverTestInstance == null || clientInstance == null) {
             assertNull(serverTestInstance);
             assertNull(clientInstance);
@@ -102,8 +102,10 @@ private void assertNodeAttributes(org.elasticsearch.xpack.core.transform.transfo
         assertThat(serverTestInstance.getTransportAddress(), equalTo(clientInstance.getTransportAddress()));
     }
 
-    public static void assertTransformProgress(org.elasticsearch.xpack.core.transform.transforms.TransformProgress serverTestInstance,
-                                         TransformProgress clientInstance) {
+    public static void assertTransformProgress(
+        org.elasticsearch.xpack.core.transform.transforms.TransformProgress serverTestInstance,
+        TransformProgress clientInstance
+    ) {
         if (serverTestInstance == null || clientInstance == null) {
             assertNull(serverTestInstance);
             assertNull(clientInstance);
@@ -115,16 +117,18 @@ public static void assertTransformProgress(org.elasticsearch.xpack.core.transfor
         assertThat(serverTestInstance.getDocumentsIndexed(), equalTo(clientInstance.getDocumentsIndexed()));
     }
 
-    public static void assertPosition(org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition serverTestInstance,
-                                TransformIndexerPosition clientInstance) {
+    public static void assertPosition(
+        org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition serverTestInstance,
+        TransformIndexerPosition clientInstance
+    ) {
         assertThat(serverTestInstance.getIndexerPosition(), equalTo(clientInstance.getIndexerPosition()));
         assertThat(serverTestInstance.getBucketsPosition(), equalTo(clientInstance.getBucketsPosition()));
     }
 
-
     public static void assertTransformCheckpointStats(
-            org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointStats serverTestInstance,
-            TransformCheckpointStats clientInstance) {
+        org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointStats serverTestInstance,
+        TransformCheckpointStats clientInstance
+    ) {
         assertTransformProgress(serverTestInstance.getCheckpointProgress(), clientInstance.getCheckpointProgress());
         assertThat(serverTestInstance.getCheckpoint(), equalTo(clientInstance.getCheckpoint()));
         assertPosition(serverTestInstance.getPosition(), clientInstance.getPosition());
@@ -133,8 +137,9 @@ public static void assertTransformCheckpointStats(
     }
 
     public static void assertTransformCheckpointInfo(
-            org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo serverTestInstance,
-            TransformCheckpointingInfo clientInstance) {
+        org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo serverTestInstance,
+        TransformCheckpointingInfo clientInstance
+    ) {
         assertTransformCheckpointStats(serverTestInstance.getNext(), clientInstance.getNext());
         assertTransformCheckpointStats(serverTestInstance.getLast(), clientInstance.getLast());
         assertThat(serverTestInstance.getChangesLastDetectedAt(), equalTo(clientInstance.getChangesLastDetectedAt()));
@@ -142,8 +147,9 @@ public static void assertTransformCheckpointInfo(
     }
 
     public static void assertTransformIndexerStats(
-            org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats serverTestInstance,
-            TransformIndexerStats clientInstance) {
+        org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats serverTestInstance,
+        TransformIndexerStats clientInstance
+    ) {
         assertThat(serverTestInstance.getExpAvgCheckpointDurationMs(), equalTo(clientInstance.getExpAvgCheckpointDurationMs()));
         assertThat(serverTestInstance.getExpAvgDocumentsProcessed(), equalTo(clientInstance.getExpAvgDocumentsProcessed()));
         assertThat(serverTestInstance.getExpAvgDocumentsIndexed(), equalTo(clientInstance.getExpAvgDocumentsIndexed()));
diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
index 17e9cd36d0212..969e92fda28ce 100644
--- a/distribution/archives/build.gradle
+++ b/distribution/archives/build.gradle
@@ -188,8 +188,8 @@ task buildLinuxTar(type: SymbolicLinkPreservingTar) {
 
 task buildOssLinuxAarch64Tar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
-  archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, true)
+  archiveClassifier = 'linux-aarch64'
+  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'aarch64', true, true)
 }
 
 task buildOssLinuxTar(type: SymbolicLinkPreservingTar) {
diff --git a/distribution/docker/aarch64-docker-build-context/build.gradle b/distribution/docker/aarch64-docker-build-context/build.gradle
new file mode 100644
index 0000000000000..19b0bc3646c60
--- /dev/null
+++ b/distribution/docker/aarch64-docker-build-context/build.gradle
@@ -0,0 +1,11 @@
+apply plugin: 'base'
+
+task buildDockerBuildContext(type: Tar) {
+  extension = 'tar.gz'
+  compression = Compression.GZIP
+  archiveClassifier = "docker-build-context"
+  archiveBaseName = "elasticsearch-aarch64"
+  with dockerBuildContext("aarch64", false, false)
+}
+
+assemble.dependsOn buildDockerBuildContext
diff --git a/distribution/docker/aarch64-docker-export/build.gradle b/distribution/docker/aarch64-docker-export/build.gradle
new file mode 100644
index 0000000000000..537b5a093683e
--- /dev/null
+++ b/distribution/docker/aarch64-docker-export/build.gradle
@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// export is done in the parent project.
diff --git a/distribution/docker/aarch64-oss-docker-build-context/build.gradle b/distribution/docker/aarch64-oss-docker-build-context/build.gradle
new file mode 100644
index 0000000000000..bea7d156803fd
--- /dev/null
+++ b/distribution/docker/aarch64-oss-docker-build-context/build.gradle
@@ -0,0 +1,11 @@
+apply plugin: 'base'
+
+task buildOssDockerBuildContext(type: Tar) {
+  extension = 'tar.gz'
+  compression = Compression.GZIP
+  archiveClassifier = "docker-build-context"
+  archiveBaseName = "elasticsearch-aarch64-oss"
+  with dockerBuildContext("aarch64", true, false)
+}
+
+assemble.dependsOn buildOssDockerBuildContext
diff --git a/distribution/docker/aarch64-oss-docker-export/build.gradle b/distribution/docker/aarch64-oss-docker-export/build.gradle
new file mode 100644
index 0000000000000..537b5a093683e
--- /dev/null
+++ b/distribution/docker/aarch64-oss-docker-export/build.gradle
@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// export is done in the parent project.
diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle
index c33f74501da95..f67e520ce76a4 100644
--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@@ -1,3 +1,4 @@
+import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.VersionProperties
@@ -13,38 +14,51 @@ testFixtures.useFixture()
 
 configurations {
   dockerPlugins
+  aarch64DockerSource
   dockerSource
+  aarch64OssDockerSource
   ossDockerSource
 }
 
 dependencies {
+  aarch64DockerSource project(path: ":distribution:archives:linux-aarch64-tar")
   dockerSource project(path: ":distribution:archives:linux-tar")
+  aarch64OssDockerSource project(path: ":distribution:archives:oss-linux-aarch64-tar")
   ossDockerSource project(path: ":distribution:archives:oss-linux-tar")
 }
 
-ext.expansions = { oss, local ->
-  final String classifier = 'linux-x86_64'
+ext.expansions = { architecture, oss, local ->
+  switch (architecture) {
+    case "aarch64":
+    case "x64":
+      break;
+    default:
+      throw new IllegalArgumentException("unrecongized architecture [" + architecture + "], must be one of (aarch64|x64)")
+  }
+  final String classifier = "aarch64".equals(architecture) ? "linux-aarch64" : "linux-x86_64"
   final String elasticsearch = oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz"
   return [
+    'base_image'          : "aarch64".equals(architecture) ? "arm64v8/centos:7" : "centos:7",
     'build_date'          : BuildParams.buildDate,
     'elasticsearch'       : elasticsearch,
     'git_revision'        : BuildParams.gitRevision,
     'license'             : oss ? 'Apache-2.0' : 'Elastic-License',
     'source_elasticsearch': local ? "COPY $elasticsearch /opt/" : "RUN cd /opt && curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/elasticsearch/${elasticsearch} && cd -",
+    'tini_suffix'         : "aarch64".equals(architecture) ? "-arm64" : "",
     'version'             : VersionProperties.elasticsearch
   ]
 }
 
-private static String buildPath(final boolean oss) {
-  return "build/${oss ? 'oss-' : ''}docker"
+private static String buildPath(final String architecture, final boolean oss) {
+  return "build/${"aarch64".equals(architecture) ? 'aarch64-' : ''}${oss ? 'oss-' : ''}docker"
 }
 
-private static String taskName(final String prefix, final boolean oss, final String suffix) {
-  return "${prefix}${oss ? 'Oss' : ''}${suffix}"
+private static String taskName(final String prefix, final String architecture, final boolean oss, final String suffix) {
+  return "${prefix}${"aarch64".equals(architecture) ? 'Aarch64' : ''}${oss ? 'Oss' : ''}${suffix}"
 }
 
 project.ext {
-  dockerBuildContext = { boolean oss, boolean local ->
+  dockerBuildContext = { String architecture, boolean oss, boolean local ->
     copySpec {
       into('bin') {
         from project.projectDir.toPath().resolve("src/docker/bin")
@@ -62,25 +76,33 @@ project.ext {
       }
 
       from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) {
-        expand(expansions(oss, local))
+        expand(expansions(architecture, oss, local))
       }
     }
   }
 }
 
-void addCopyDockerContextTask(final boolean oss) {
-  task(taskName("copy", oss, "DockerContext"), type: Sync) {
-    expansions(oss, true).findAll { it.key != 'build_date' }.each { k, v ->
+void addCopyDockerContextTask(final String architecture, final boolean oss) {
+  task(taskName("copy", architecture, oss, "DockerContext"), type: Sync) {
+    expansions(architecture, oss, true).findAll { it.key != 'build_date' }.each { k, v ->
       inputs.property(k, { v.toString() })
     }
-    into buildPath(oss)
+    into buildPath(architecture, oss)
 
-    with dockerBuildContext(oss, true)
+    with dockerBuildContext(architecture, oss, true)
 
-    if (oss) {
-      from configurations.ossDockerSource
+    if ("aarch64".equals(architecture)) {
+      if (oss) {
+        from configurations.aarch64OssDockerSource
+      } else {
+        from configurations.aarch64DockerSource
+      }
     } else {
-      from configurations.dockerSource
+      if (oss) {
+        from configurations.ossDockerSource
+      } else {
+        from configurations.dockerSource
+      }
     }
 
     from configurations.dockerPlugins
@@ -149,9 +171,9 @@ task integTest(type: Test) {
 
 check.dependsOn integTest
 
-void addBuildDockerImage(final boolean oss) {
-  final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: DockerBuildTask) {
-    TaskProvider<Sync> copyContextTask = tasks.named(taskName("copy", oss, "DockerContext"))
+void addBuildDockerImage(final String architecture, final boolean oss) {
+  final Task buildDockerImageTask = task(taskName("build", architecture, oss, "DockerImage"), type: DockerBuildTask) {
+    TaskProvider<Sync> copyContextTask = tasks.named(taskName("copy", architecture, oss, "DockerContext"))
     dependsOn(copyContextTask)
     dockerContext.fileProvider(copyContextTask.map { it.destinationDir })
 
@@ -169,12 +191,15 @@ void addBuildDockerImage(final boolean oss) {
       ]
     }
   }
+  buildDockerImageTask.onlyIf { Architecture.current().name().toLowerCase().equals(architecture) }
   assemble.dependsOn(buildDockerImageTask)
 }
 
-for (final boolean oss : [false, true]) {
-  addCopyDockerContextTask(oss)
-  addBuildDockerImage(oss)
+for (final String architecture : ["aarch64", "x64"]) {
+  for (final boolean oss : [false, true]) {
+    addCopyDockerContextTask(architecture, oss)
+    addBuildDockerImage(architecture, oss)
+  }
 }
 
 // We build the images used in compose locally, but the pull command insists on using a repository
@@ -192,11 +217,12 @@ subprojects { Project subProject ->
   if (subProject.name.contains('docker-export')) {
     apply plugin: 'distribution'
 
+    final String architecture = subProject.name.contains('aarch64-') ? 'aarch64' : 'x64'
     final boolean oss = subProject.name.contains('oss-')
 
-    def exportTaskName = taskName("export", oss, "DockerImage")
-    def buildTaskName = taskName("build", oss, "DockerImage")
-    def tarFile = "${parent.projectDir}/build/elasticsearch${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar"
+    def exportTaskName = taskName("export", architecture, oss, "DockerImage")
+    def buildTaskName = taskName("build", architecture, oss, "DockerImage")
+    def tarFile = "${parent.projectDir}/build/elasticsearch${"aarch64".equals(architecture) ? '-aarch64' : ''}${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar"
 
     final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) {
       inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker")
@@ -212,7 +238,7 @@ subprojects { Project subProject ->
 
     artifacts.add('default', file(tarFile)) {
       type 'tar'
-      name "elasticsearch${oss ? '-oss' : ''}"
+      name "elasticsearch${"aarch64".equals(architecture) ? '-aarch64' : ''}${oss ? '-oss' : ''}"
       builtBy exportTaskName
     }
 
diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle
index 50be407e566bc..2dd28329d7ba5 100644
--- a/distribution/docker/docker-build-context/build.gradle
+++ b/distribution/docker/docker-build-context/build.gradle
@@ -5,7 +5,7 @@ task buildDockerBuildContext(type: Tar) {
   compression = Compression.GZIP
   archiveClassifier = "docker-build-context"
   archiveBaseName = "elasticsearch"
-  with dockerBuildContext(false, false)
+  with dockerBuildContext("x64", false, false)
 }
 
 assemble.dependsOn buildDockerBuildContext
diff --git a/distribution/docker/oss-docker-build-context/build.gradle b/distribution/docker/oss-docker-build-context/build.gradle
index b69f7dc620f53..0a29c2a2b7274 100644
--- a/distribution/docker/oss-docker-build-context/build.gradle
+++ b/distribution/docker/oss-docker-build-context/build.gradle
@@ -5,7 +5,7 @@ task buildOssDockerBuildContext(type: Tar) {
   compression = Compression.GZIP
   archiveClassifier = "docker-build-context"
   archiveBaseName = "elasticsearch-oss"
-  with dockerBuildContext(true, false)
+  with dockerBuildContext("x64", true, false)
 }
 
 assemble.dependsOn buildOssDockerBuildContext
diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile
index b12a3c7518265..7eb62b7d8ad1b 100644
--- a/distribution/docker/src/docker/Dockerfile
+++ b/distribution/docker/src/docker/Dockerfile
@@ -11,7 +11,7 @@
 # Set gid=0 and make group perms==owner perms
 ################################################################################
 
-FROM centos:7 AS builder
+FROM ${base_image} AS builder
 
 RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \
     yum install --setopt=tsflags=nodocs -y gzip shadow-utils tar && \
@@ -42,8 +42,8 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
 # gpg, but the keyservers are slow to return the key and this can fail the
 # build. Instead, we check the binary against a checksum that we have
 # computed.
-ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini /tini
-COPY config/tini.sha512 /tini.sha512
+ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini${tini_suffix} /tini
+COPY config/tini${tini_suffix}.sha512 /tini.sha512
 RUN sha512sum -c /tini.sha512 && chmod +x /tini
 
 ################################################################################
@@ -52,7 +52,7 @@ RUN sha512sum -c /tini.sha512 && chmod +x /tini
 # Add entrypoint
 ################################################################################
 
-FROM centos:7
+FROM ${base_image}
 
 ENV ELASTIC_CONTAINER true
 
diff --git a/distribution/docker/src/docker/config/tini-arm64.sha512 b/distribution/docker/src/docker/config/tini-arm64.sha512
new file mode 100644
index 0000000000000..274eaa28cff08
--- /dev/null
+++ b/distribution/docker/src/docker/config/tini-arm64.sha512
@@ -0,0 +1 @@
+6ae5147e522e484b9d59b0caa04e6dadf0efe332b272039c7cf5951e39f5028e9852c3c4bcdd46b98977329108d555ee7ea55f9eca99765d05922ec7aff837d8  /tini
diff --git a/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml b/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml
index c5b355e9b83f3..acda1a3dcedf7 100644
--- a/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml
+++ b/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml
@@ -7,7 +7,7 @@
   - match:
       $body: |
         /  #ip                          heap.percent        ram.percent     cpu         load_1m                load_5m                load_15m               node.role           master          name
-        ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
+        ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilmrt]{1,6}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
@@ -16,7 +16,7 @@
   - match:
       $body: |
         /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role      \s+  master   \s+   name  \n
-           ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
+           ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilmrt]{1,6}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/AddFileKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/AddFileKeyStoreCommand.java
index a36c6fb262550..2217eda8cde49 100644
--- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/AddFileKeyStoreCommand.java
+++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/AddFileKeyStoreCommand.java
@@ -19,11 +19,6 @@
 
 package org.elasticsearch.common.settings;
 
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.List;
-
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.elasticsearch.cli.ExitCodes;
@@ -33,6 +28,11 @@
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.env.Environment;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * A subcommand for the keystore cli which adds a file setting.
  */
@@ -49,38 +49,39 @@ class AddFileKeyStoreCommand extends BaseKeyStoreCommand {
         // jopt simple has issue with multiple non options, so we just get one set of them here
         // and convert to File when necessary
         // see https://github.com/jopt-simple/jopt-simple/issues/103
-        this.arguments = parser.nonOptions("setting [filepath]");
+        this.arguments = parser.nonOptions("(setting path)+");
     }
 
     @Override
     protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception {
-        List<String> argumentValues = arguments.values(options);
+        final List<String> argumentValues = arguments.values(options);
         if (argumentValues.size() == 0) {
             throw new UserException(ExitCodes.USAGE, "Missing setting name");
         }
-        String setting = argumentValues.get(0);
+        if (argumentValues.size() % 2 != 0) {
+            throw new UserException(ExitCodes.USAGE, "settings and filenames must come in pairs");
+        }
+
         final KeyStoreWrapper keyStore = getKeyStore();
-        if (keyStore.getSettingNames().contains(setting) && options.has(forceOption) == false) {
-            if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
-                terminal.println("Exiting without modifying keystore.");
-                return;
+
+        for (int i = 0; i < argumentValues.size(); i += 2) {
+            final String setting = argumentValues.get(i);
+
+            if (keyStore.getSettingNames().contains(setting) && options.has(forceOption) == false) {
+                if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
+                    terminal.println("Exiting without modifying keystore.");
+                    return;
+                }
             }
-        }
 
-        if (argumentValues.size() == 1) {
-            throw new UserException(ExitCodes.USAGE, "Missing file name");
-        }
-        Path file = getPath(argumentValues.get(1));
-        if (Files.exists(file) == false) {
-            throw new UserException(ExitCodes.IO_ERROR, "File [" + file.toString() + "] does not exist");
-        }
-        if (argumentValues.size() > 2) {
-            throw new UserException(
-                ExitCodes.USAGE,
-                "Unrecognized extra arguments [" + String.join(", ", argumentValues.subList(2, argumentValues.size())) + "] after filepath"
-            );
+            final Path file = getPath(argumentValues.get(i + 1));
+            if (Files.exists(file) == false) {
+                throw new UserException(ExitCodes.IO_ERROR, "File [" + file.toString() + "] does not exist");
+            }
+
+            keyStore.setFile(setting, Files.readAllBytes(file));
         }
-        keyStore.setFile(setting, Files.readAllBytes(file));
+
         keyStore.save(env.configFile(), getKeyStorePassword().getChars());
     }
 
@@ -88,4 +89,5 @@ protected void executeCommand(Terminal terminal, OptionSet options, Environment
     private Path getPath(String file) {
         return PathUtils.get(file);
     }
+
 }
diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java
index 421ea6eaee066..a4c0de633cf1b 100644
--- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java
+++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java
@@ -19,20 +19,24 @@
 
 package org.elasticsearch.common.settings;
 
-import java.io.BufferedReader;
-import java.io.CharArrayWriter;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.env.Environment;
 
+import java.io.BufferedReader;
+import java.io.CharArrayWriter;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * A subcommand for the keystore cli which adds a string setting.
  */
@@ -42,13 +46,13 @@ class AddStringKeyStoreCommand extends BaseKeyStoreCommand {
     private final OptionSpec<String> arguments;
 
     AddStringKeyStoreCommand() {
-        super("Add a string setting to the keystore", false);
-        this.stdinOption = parser.acceptsAll(Arrays.asList("x", "stdin"), "Read setting value from stdin");
+        super("Add string settings to the keystore", false);
+        this.stdinOption = parser.acceptsAll(Arrays.asList("x", "stdin"), "Read setting values from stdin");
         this.forceOption = parser.acceptsAll(
             Arrays.asList("f", "force"),
             "Overwrite existing setting without prompting, creating keystore if necessary"
         );
-        this.arguments = parser.nonOptions("setting name");
+        this.arguments = parser.nonOptions("setting names");
     }
 
     // pkg private so tests can manipulate
@@ -58,43 +62,53 @@ InputStream getStdin() {
 
     @Override
     protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception {
-        String setting = arguments.value(options);
-        if (setting == null) {
-            throw new UserException(ExitCodes.USAGE, "The setting name can not be null");
+        final List<String> settings = arguments.values(options);
+        if (settings.isEmpty()) {
+            throw new UserException(ExitCodes.USAGE, "the setting names can not be empty");
         }
+
         final KeyStoreWrapper keyStore = getKeyStore();
-        if (keyStore.getSettingNames().contains(setting) && options.has(forceOption) == false) {
-            if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
-                terminal.println("Exiting without modifying keystore.");
-                return;
-            }
-        }
 
-        final char[] value;
+        final Closeable closeable;
+        final CheckedFunction<String, char[], IOException> valueSupplier;
         if (options.has(stdinOption)) {
-            try (
-                BufferedReader stdinReader = new BufferedReader(new InputStreamReader(getStdin(), StandardCharsets.UTF_8));
-                CharArrayWriter writer = new CharArrayWriter()
-            ) {
-                int charInt;
-                while ((charInt = stdinReader.read()) != -1) {
-                    if ((char) charInt == '\r' || (char) charInt == '\n') {
-                        break;
+            final BufferedReader stdinReader = new BufferedReader(new InputStreamReader(getStdin(), StandardCharsets.UTF_8));
+            valueSupplier = s -> {
+                try (CharArrayWriter writer = new CharArrayWriter()) {
+                    int c;
+                    while ((c = stdinReader.read()) != -1) {
+                        if ((char) c == '\r' || (char) c == '\n') {
+                            break;
+                        }
+                        writer.write((char) c);
                     }
-                    writer.write((char) charInt);
+                    return writer.toCharArray();
                 }
-                value = writer.toCharArray();
-            }
+            };
+            closeable = stdinReader;
         } else {
-            value = terminal.readSecret("Enter value for " + setting + ": ");
+            valueSupplier = s -> terminal.readSecret("Enter value for " + s + ": ");
+            closeable = () -> {};
         }
 
-        try {
-            keyStore.setString(setting, value);
-        } catch (IllegalArgumentException e) {
-            throw new UserException(ExitCodes.DATA_ERROR, e.getMessage());
+        try (closeable) {
+            for (final String setting : settings) {
+                if (keyStore.getSettingNames().contains(setting) && options.has(forceOption) == false) {
+                    if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
+                        terminal.println("Exiting without modifying keystore.");
+                        return;
+                    }
+                }
+
+                try {
+                    keyStore.setString(setting, valueSupplier.apply(setting));
+                } catch (final IllegalArgumentException e) {
+                    throw new UserException(ExitCodes.DATA_ERROR, e.getMessage());
+                }
+            }
         }
-        keyStore.save(env.configFile(), getKeyStorePassword().getChars());
 
+        keyStore.save(env.configFile(), getKeyStorePassword().getChars());
     }
+
 }
diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/AddFileKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/AddFileKeyStoreCommandTests.java
index 8d6a2d7e4383d..50e897da8548c 100644
--- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/AddFileKeyStoreCommandTests.java
+++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/AddFileKeyStoreCommandTests.java
@@ -19,16 +19,20 @@
 
 package org.elasticsearch.common.settings;
 
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Map;
-
 import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.env.Environment;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 
@@ -49,7 +53,7 @@ private Path createRandomFile() throws IOException {
         for (int i = 0; i < length; ++i) {
             bytes[i] = randomByte();
         }
-        Path file = env.configFile().resolve("randomfile");
+        Path file = env.configFile().resolve(randomAlphaOfLength(16));
         Files.write(file, bytes);
         return file;
     }
@@ -164,7 +168,7 @@ public void testMissingFileName() throws Exception {
         terminal.addSecretInput(password);
         UserException e = expectThrows(UserException.class, () -> execute("foo"));
         assertEquals(ExitCodes.USAGE, e.exitCode);
-        assertThat(e.getMessage(), containsString("Missing file name"));
+        assertThat(e.getMessage(), containsString("settings and filenames must come in pairs"));
     }
 
     public void testFileDNE() throws Exception {
@@ -183,7 +187,7 @@ public void testExtraArguments() throws Exception {
         terminal.addSecretInput(password);
         UserException e = expectThrows(UserException.class, () -> execute("foo", file.toString(), "bar"));
         assertEquals(e.getMessage(), ExitCodes.USAGE, e.exitCode);
-        assertThat(e.getMessage(), containsString("Unrecognized extra arguments [bar]"));
+        assertThat(e.getMessage(), containsString("settings and filenames must come in pairs"));
     }
 
     public void testIncorrectPassword() throws Exception {
@@ -216,4 +220,20 @@ public void testAddToUnprotectedKeystore() throws Exception {
         execute("foo", "path/dne");
         assertSecureFile("foo", file, password);
     }
+
+    public void testAddMultipleFiles() throws Exception {
+        final String password = "keystorepassword";
+        createKeystore(password);
+        final int n = randomIntBetween(1, 8);
+        final List<Tuple<String, Path>> settingFilePairs = new ArrayList<>(n);
+        for (int i = 0; i < n; i++) {
+            settingFilePairs.add(Tuple.tuple("foo" + i, createRandomFile()));
+        }
+        terminal.addSecretInput(password);
+        execute(settingFilePairs.stream().flatMap(t -> Stream.of(t.v1(), t.v2().toString())).toArray(String[]::new));
+        for (int i = 0; i < n; i++) {
+            assertSecureFile(settingFilePairs.get(i).v1(), settingFilePairs.get(i).v2(), password);
+        }
+    }
+
 }
diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java
index f40d212a9b3d1..15a8aa20b851f 100644
--- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java
+++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java
@@ -19,17 +19,17 @@
 
 package org.elasticsearch.common.settings;
 
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
 import java.io.ByteArrayInputStream;
 import java.io.CharArrayWriter;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
 
-import org.elasticsearch.cli.Command;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.UserException;
-import org.elasticsearch.env.Environment;
-
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasToString;
@@ -155,6 +155,19 @@ public void testPromptForValue() throws Exception {
         assertSecureString("foo", "secret value", password);
     }
 
+    public void testPromptForMultipleValues() throws Exception {
+        final String password = "keystorepassword";
+        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        terminal.addSecretInput(password);
+        terminal.addSecretInput("bar1");
+        terminal.addSecretInput("bar2");
+        terminal.addSecretInput("bar3");
+        execute("foo1", "foo2", "foo3");
+        assertSecureString("foo1", "bar1", password);
+        assertSecureString("foo2", "bar2", password);
+        assertSecureString("foo3", "bar3", password);
+    }
+
     public void testStdinShort() throws Exception {
         String password = "keystorepassword";
         KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
@@ -200,6 +213,17 @@ public void testStdinInputWithCarriageReturn() throws Exception {
         assertSecureString("foo", "Typedthisandhitenter", password);
     }
 
+    public void testStdinWithMultipleValues() throws Exception {
+        final String password = "keystorepassword";
+        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        terminal.addSecretInput(password);
+        setInput("bar1\nbar2\nbar3");
+        execute(randomFrom("-x", "--stdin"), "foo1", "foo2", "foo3");
+        assertSecureString("foo1", "bar1", password);
+        assertSecureString("foo2", "bar2", password);
+        assertSecureString("foo3", "bar3", password);
+    }
+
     public void testAddUtf8String() throws Exception {
         String password = "keystorepassword";
         KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
@@ -223,7 +247,7 @@ public void testMissingSettingName() throws Exception {
         terminal.addTextInput("");
         UserException e = expectThrows(UserException.class, this::execute);
         assertEquals(ExitCodes.USAGE, e.exitCode);
-        assertThat(e.getMessage(), containsString("The setting name can not be null"));
+        assertThat(e.getMessage(), containsString("the setting names can not be empty"));
     }
 
     public void testSpecialCharacterInName() throws Exception {
diff --git a/docs/java-rest/high-level/ml/estimate-model-memory.asciidoc b/docs/java-rest/high-level/ml/estimate-model-memory.asciidoc
new file mode 100644
index 0000000000000..8e8b5f1befa34
--- /dev/null
+++ b/docs/java-rest/high-level/ml/estimate-model-memory.asciidoc
@@ -0,0 +1,42 @@
+--
+:api: estimate-model-memory
+:request: EstimateModelMemoryRequest
+:response: EstimateModelMemoryResponse
+--
+[role="xpack"]
+[id="{upid}-{api}"]
+=== Estimate {anomaly-job} model memory API
+
+Estimate the model memory an analysis config is likely to need for
+the given cardinality of the fields it references.
+
+[id="{upid}-{api}-request"]
+==== Estimate {anomaly-job} model memory request
+
+A +{request}+ can be set up as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+<1> Pass an `AnalysisConfig` to the constructor.
+<2> For any `by_field_name`, `over_field_name` or
+    `partition_field_name` fields referenced by the
+    detectors, supply overall cardinality estimates
+    in a `Map`.
+<3> For any `influencers`, supply a `Map` containing
+    estimates of the highest cardinality expected in
+    any single bucket.
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Estimate {anomaly-job} model memory response
+
+The returned +{response}+ contains the model memory estimate:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
+<1> The model memory estimate.
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
index efe7b6650640c..4999d0627a9eb 100644
--- a/docs/java-rest/high-level/supported-apis.asciidoc
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -295,6 +295,7 @@ The Java High Level REST Client supports the following Machine Learning APIs:
 * <<{upid}-put-calendar-job>>
 * <<{upid}-delete-calendar-job>>
 * <<{upid}-delete-calendar>>
+* <<{upid}-estimate-model-memory>>
 * <<{upid}-get-data-frame-analytics>>
 * <<{upid}-get-data-frame-analytics-stats>>
 * <<{upid}-put-data-frame-analytics>>
@@ -351,6 +352,7 @@ include::ml/delete-calendar-event.asciidoc[]
 include::ml/put-calendar-job.asciidoc[]
 include::ml/delete-calendar-job.asciidoc[]
 include::ml/delete-calendar.asciidoc[]
+include::ml/estimate-model-memory.asciidoc[]
 include::ml/get-data-frame-analytics.asciidoc[]
 include::ml/get-data-frame-analytics-stats.asciidoc[]
 include::ml/put-data-frame-analytics.asciidoc[]
diff --git a/docs/plugins/analysis-nori.asciidoc b/docs/plugins/analysis-nori.asciidoc
index 7cc04c9c3de75..1e5f998a72da4 100644
--- a/docs/plugins/analysis-nori.asciidoc
+++ b/docs/plugins/analysis-nori.asciidoc
@@ -54,6 +54,10 @@ It can be set to:
     가곡역 => 가곡역, 가곡, 역
 --
 
+`discard_punctuation`::
+
+    Whether punctuation should be discarded from the output. Defaults to `true`.
+
 `user_dictionary`::
 +
 --
@@ -99,6 +103,7 @@ PUT nori_sample
           "nori_user_dict": {
             "type": "nori_tokenizer",
             "decompound_mode": "mixed",
+            "discard_punctuation": "false",
             "user_dictionary": "userdict_ko.txt"
           }
         },
@@ -434,3 +439,107 @@ Which responds with:
 --------------------------------------------------
 
 <1> The Hanja form is replaced by the Hangul translation.
+
+
+[[analysis-nori-number]]
+==== `nori_number` token filter
+
+The `nori_number` token filter normalizes Korean numbers
+to regular Arabic decimal numbers in half-width characters.
+
+Korean numbers are often written using a combination of Hangul and Arabic numbers with various kinds punctuation.
+For example, ３．２천 means 3200.
+This filter does this kind of normalization and allows a search for 3200 to match ３．２천 in text,
+but can also be used to make range facets based on the normalized numbers and so on.
+
+[NOTE]
+====
+Notice that this analyzer uses a token composition scheme and relies on punctuation tokens
+being found in the token stream.
+Please make sure your `nori_tokenizer` has `discard_punctuation` set to false.
+In case punctuation characters, such as U+FF0E(．), is removed from the token stream,
+this filter would find input tokens ３ and ２천 and give outputs 3 and 2000 instead of 3200,
+which is likely not the intended result.
+
+If you want to remove punctuation characters from your index that are not part of normalized numbers,
+add a `stop` token filter with the punctuation you wish to remove after `nori_number` in your analyzer chain.
+====
+Below are some examples of normalizations this filter supports.
+The input is untokenized text and the result is the single term attribute emitted for the input.
+
+- 영영칠 -> 7
+- 일영영영 -> 1000
+- 삼천2백2십삼 -> 3223
+- 조육백만오천일 -> 1000006005001
+- ３.２천 ->  3200
+- １.２만３４５.６７ -> 12345.67
+- 4,647.100 -> 4647.1
+- 15,7 -> 157 (be aware of this weakness)
+
+For example:
+
+[source,console]
+--------------------------------------------------
+PUT nori_sample
+{
+  "settings": {
+    "index": {
+      "analysis": {
+        "analyzer": {
+          "my_analyzer": {
+            "tokenizer": "tokenizer_discard_puncuation_false",
+            "filter": [
+              "part_of_speech_stop_sp", "nori_number"
+            ]
+          }
+        },
+        "tokenizer": {
+          "tokenizer_discard_puncuation_false": {
+            "type": "nori_tokenizer",
+            "discard_punctuation": "false"
+          }
+        },
+        "filter": {
+            "part_of_speech_stop_sp": {
+                "type": "nori_part_of_speech",
+                "stoptags": ["SP"]
+            }
+        }
+      }
+    }
+  }
+}
+
+GET nori_sample/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "십만이천오백과 ３.２천"
+}
+--------------------------------------------------
+
+Which results in:
+
+[source,console-result]
+--------------------------------------------------
+{
+  "tokens" : [{
+    "token" : "102500",
+    "start_offset" : 0,
+    "end_offset" : 6,
+    "type" : "word",
+    "position" : 0
+  }, {
+    "token" : "과",
+    "start_offset" : 6,
+    "end_offset" : 7,
+    "type" : "word",
+    "position" : 1
+  }, {
+    "token" : "3200",
+    "start_offset" : 8,
+    "end_offset" : 12,
+    "type" : "word",
+    "position" : 2
+  }]
+}
+--------------------------------------------------
diff --git a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
index 6dbc8adfd3ff8..e0ab0afde2f3a 100644
--- a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc
@@ -50,7 +50,7 @@ faster.
 
 The `sort` field in the metric request functions exactly the same as the `sort` field in the
 <<request-body-search-sort, search>> request except:
-* It can't be used on <<binary,binary>>, <<flattened,flattened>, <<ip,ip>>,
+* It can't be used on <<binary,binary>>, <<flattened,flattened>>, <<ip,ip>>,
 <<keyword,keyword>>, or <<text,text>> fields.
 * It only supports a single sort value so which document wins ties is not specified.
 
diff --git a/docs/reference/analysis/tokenfilters/keyword-marker-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/keyword-marker-tokenfilter.asciidoc
index 6ee9f41f777d1..a3d6b82625c38 100644
--- a/docs/reference/analysis/tokenfilters/keyword-marker-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/keyword-marker-tokenfilter.asciidoc
@@ -4,137 +4,385 @@
 <titleabbrev>Keyword marker</titleabbrev>
 ++++
 
-Protects words from being modified by stemmers. Must be placed before
-any stemming filters.
+Marks specified tokens as keywords, which are not stemmed.
 
-[cols="<,<",options="header",]
-|=======================================================================
-|Setting |Description
-|`keywords` |A list of words to use.
+The `keyword_marker` assigns specified tokens a `keyword` attribute of `true`.
+Stemmer token filters, such as <<analysis-stemmer-tokenfilter,`stemmer`>> or
+<<analysis-porterstem-tokenfilter,`porter_stem`>>, skip tokens with a `keyword`
+attribute of `true`.
 
-|`keywords_path` |A path (either relative to `config` location, or
-absolute) to a list of words.
+[IMPORTANT]
+====
+To work properly, the `keyword_marker` filter must be listed before any stemmer
+token filters in the <<analysis-custom-analyzer,analyzer configuration>>.
+====
 
-|`keywords_pattern` |A regular expression pattern to match against words
-in the text.
+The `keyword_marker` filter uses Lucene's
+{lucene-analysis-docs}/miscellaneous/KeywordMarkerFilter.html[KeywordMarkerFilter].
 
-|`ignore_case` |Set to `true` to lower case all words first. Defaults to
-`false`.
-|=======================================================================
+[[analysis-keyword-marker-tokenfilter-analyze-ex]]
+==== Example
 
-You can configure it like:
+To see how the `keyword_marker` filter works, you first need to produce a token
+stream containing stemmed tokens.
 
-[source,console]
---------------------------------------------------
-PUT /keyword_marker_example
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "protect_cats": {
-          "type": "custom",
-          "tokenizer": "standard",
-          "filter": ["lowercase", "protect_cats", "porter_stem"]
-        },
-        "normal": {
-          "type": "custom",
-          "tokenizer": "standard",
-          "filter": ["lowercase", "porter_stem"]
-        }
-      },
-      "filter": {
-        "protect_cats": {
-          "type": "keyword_marker",
-          "keywords": ["cats"]
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-
-And test it with:
+The following <<indices-analyze,analyze API>> request uses the
+<<analysis-stemmer-tokenfilter,`stemmer`>> filter to create stemmed tokens for
+`fox running and jumping`.
 
 [source,console]
---------------------------------------------------
-POST /keyword_marker_example/_analyze
+----
+GET /_analyze
 {
-  "analyzer" : "protect_cats",
-  "text" : "I like cats"
+  "tokenizer": "whitespace",
+  "filter": [ "stemmer" ],
+  "text": "fox running and jumping"
 }
---------------------------------------------------
-// TEST[continued]
+----
+
+The request produces the following tokens. Note that `running` was stemmed to
+`run` and `jumping` was stemmed to `jump`.
 
-And it'd respond:
+[source,text]
+----
+[ fox, run, and, jump ]
+----
 
+////
 [source,console-result]
---------------------------------------------------
+----
 {
   "tokens": [
     {
-      "token": "i",
+      "token": "fox",
       "start_offset": 0,
-      "end_offset": 1,
-      "type": "<ALPHANUM>",
+      "end_offset": 3,
+      "type": "word",
       "position": 0
     },
     {
-      "token": "like",
-      "start_offset": 2,
-      "end_offset": 6,
-      "type": "<ALPHANUM>",
+      "token": "run",
+      "start_offset": 4,
+      "end_offset": 11,
+      "type": "word",
       "position": 1
     },
     {
-      "token": "cats",
-      "start_offset": 7,
-      "end_offset": 11,
-      "type": "<ALPHANUM>",
+      "token": "and",
+      "start_offset": 12,
+      "end_offset": 15,
+      "type": "word",
       "position": 2
+    },
+    {
+      "token": "jump",
+      "start_offset": 16,
+      "end_offset": 23,
+      "type": "word",
+      "position": 3
     }
   ]
 }
---------------------------------------------------
+----
+////
 
-As compared to the `normal` analyzer which has `cats` stemmed to `cat`:
+To prevent `jumping` from being stemmed, add the `keyword_marker` filter before
+the `stemmer` filter in the previous analyze API request. Specify `jumping` in
+the `keywords` parameter of the `keyword_marker` filter.
 
 [source,console]
---------------------------------------------------
-POST /keyword_marker_example/_analyze
+----
+GET /_analyze
 {
-  "analyzer" : "normal",
-  "text" : "I like cats"
+  "tokenizer": "whitespace",
+  "filter": [
+    {
+      "type": "keyword_marker",
+      "keywords": [ "jumping" ]
+    },
+    "stemmer"
+  ],
+  "text": "fox running and jumping"
 }
---------------------------------------------------
-// TEST[continued]
+----
 
-Response:
+The request produces the following tokens. `running` is still stemmed to `run`,
+but `jumping` is not stemmed.
 
+[source,text]
+----
+[ fox, run, and, jumping ]
+----
+
+////
 [source,console-result]
---------------------------------------------------
+----
 {
   "tokens": [
     {
-      "token": "i",
+      "token": "fox",
       "start_offset": 0,
-      "end_offset": 1,
-      "type": "<ALPHANUM>",
+      "end_offset": 3,
+      "type": "word",
       "position": 0
     },
     {
-      "token": "like",
-      "start_offset": 2,
-      "end_offset": 6,
-      "type": "<ALPHANUM>",
+      "token": "run",
+      "start_offset": 4,
+      "end_offset": 11,
+      "type": "word",
       "position": 1
     },
     {
-      "token": "cat",
-      "start_offset": 7,
-      "end_offset": 11,
-      "type": "<ALPHANUM>",
+      "token": "and",
+      "start_offset": 12,
+      "end_offset": 15,
+      "type": "word",
       "position": 2
+    },
+    {
+      "token": "jumping",
+      "start_offset": 16,
+      "end_offset": 23,
+      "type": "word",
+      "position": 3
     }
   ]
 }
---------------------------------------------------
+----
+////
+
+To see the `keyword` attribute for these tokens, add the following arguments to
+the analyze API request:
+
+* `explain`:  `true`
+* `attributes`: `keyword`
+
+[source,console]
+----
+GET /_analyze
+{
+  "tokenizer": "whitespace",
+  "filter": [
+    {
+      "type": "keyword_marker",
+      "keywords": [ "jumping" ]
+    },
+    "stemmer"
+  ],
+  "text": "fox running and jumping",
+  "explain": true,
+  "attributes": "keyword"
+}
+----
+
+The API returns the following response.  Note the `jumping` token has a
+`keyword` attribute of `true`.
+
+[source,console-result]
+----
+{
+  "detail": {
+    "custom_analyzer": true,
+    "charfilters": [],
+    "tokenizer": {
+      "name": "whitespace",
+      "tokens": [
+        {
+          "token": "fox",
+          "start_offset": 0,
+          "end_offset": 3,
+          "type": "word",
+          "position": 0
+        },
+        {
+          "token": "running",
+          "start_offset": 4,
+          "end_offset": 11,
+          "type": "word",
+          "position": 1
+        },
+        {
+          "token": "and",
+          "start_offset": 12,
+          "end_offset": 15,
+          "type": "word",
+          "position": 2
+        },
+        {
+          "token": "jumping",
+          "start_offset": 16,
+          "end_offset": 23,
+          "type": "word",
+          "position": 3
+        }
+      ]
+    },
+    "tokenfilters": [
+      {
+        "name": "__anonymous__keyword_marker",
+        "tokens": [
+          {
+            "token": "fox",
+            "start_offset": 0,
+            "end_offset": 3,
+            "type": "word",
+            "position": 0,
+            "keyword": false
+          },
+          {
+            "token": "running",
+            "start_offset": 4,
+            "end_offset": 11,
+            "type": "word",
+            "position": 1,
+            "keyword": false
+          },
+          {
+            "token": "and",
+            "start_offset": 12,
+            "end_offset": 15,
+            "type": "word",
+            "position": 2,
+            "keyword": false
+          },
+          {
+            "token": "jumping",
+            "start_offset": 16,
+            "end_offset": 23,
+            "type": "word",
+            "position": 3,
+            "keyword": true
+          }
+        ]
+      },
+      {
+        "name": "stemmer",
+        "tokens": [
+          {
+            "token": "fox",
+            "start_offset": 0,
+            "end_offset": 3,
+            "type": "word",
+            "position": 0,
+            "keyword": false
+          },
+          {
+            "token": "run",
+            "start_offset": 4,
+            "end_offset": 11,
+            "type": "word",
+            "position": 1,
+            "keyword": false
+          },
+          {
+            "token": "and",
+            "start_offset": 12,
+            "end_offset": 15,
+            "type": "word",
+            "position": 2,
+            "keyword": false
+          },
+          {
+            "token": "jumping",
+            "start_offset": 16,
+            "end_offset": 23,
+            "type": "word",
+            "position": 3,
+            "keyword": true
+          }
+        ]
+      }
+    ]
+  }
+}
+----
+
+[[analysis-keyword-marker-tokenfilter-configure-parms]]
+==== Configurable parameters
+
+`ignore_case`::
+(Optional, boolean)
+If `true`, matching for the `keywords` and `keywords_path` parameters ignores
+letter case. Defaults to `false`.
+
+`keywords`::
+(Required*, array of strings)
+Array of keywords. Tokens that match these keywords are not stemmed.
++
+This parameter, `keywords_path`, or `keywords_pattern` must be specified.
+You cannot specify this parameter and `keywords_pattern`.
+
+`keywords_path`::
++
+--
+(Required*, array of strings)
+Path to a file that contains a list of keywords. Tokens that match these
+keywords are not stemmed.
+
+This path must be absolute or relative to the `config` location, and the file
+must be UTF-8 encoded. Each word in the file must be separated by a line break.
+
+This parameter, `keywords`, or `keywords_pattern` must be specified.
+You cannot specify this parameter and `keywords_pattern`.
+--
+
+`keywords_pattern`::
++
+--
+(Required*, string)
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java
+regular expression] used to match tokens. Tokens that match this expression are
+marked as keywords and not stemmed.
+
+This parameter, `keywords`, or `keywords_path` must be specified. You
+cannot specify this parameter and `keywords` or `keywords_pattern`.
+
+[WARNING]
+====
+Poorly written regular expressions can cause {es} to run slowly or result
+in stack overflow errors, causing the running node to suddenly exit.
+====
+--
+
+[[analysis-keyword-marker-tokenfilter-customize]]
+==== Customize and add to an analyzer
+
+To customize the `keyword_marker` filter, duplicate it to create the basis for a
+new custom token filter. You can modify the filter using its configurable
+parameters.
+
+For example, the following <<indices-create-index,create index API>> request
+uses a custom `keyword_marker` filter and the `porter_stem`
+filter to configure a new <<analysis-custom-analyzer,custom analyzer>>.
+
+The custom `keyword_marker` filter marks tokens specified in the
+`analysis/example_word_list.txt` file as keywords. The `porter_stem` filter does
+not stem these tokens.
+
+[source,console]
+----
+PUT /my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_custom_analyzer": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": [
+            "my_custom_keyword_marker_filter",
+            "porter_stem"
+          ]
+        }
+      },
+      "filter": {
+        "my_custom_keyword_marker_filter": {
+          "type": "keyword_marker",
+          "keywords": "analysis/example_word_list.txt"
+        }
+      }
+    }
+  }
+}
+----
\ No newline at end of file
diff --git a/docs/reference/analysis/tokenfilters/uppercase-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/uppercase-tokenfilter.asciidoc
index 780e09fa951cc..84c9ebd186595 100644
--- a/docs/reference/analysis/tokenfilters/uppercase-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/uppercase-tokenfilter.asciidoc
@@ -16,7 +16,8 @@ Depending on the language, an uppercase character can map to multiple
 lowercase characters. Using the `uppercase` filter could result in the loss of
 lowercase character information.
 
-To avoid this loss but still have a consistent lettercase, use the <<analysis-lowercase-tokenfilter,`lowercase`>> filter instead.
+To avoid this loss but still have a consistent letter case, use the
+<<analysis-lowercase-tokenfilter,`lowercase`>> filter instead.
 ====
 
 [[analysis-uppercase-tokenfilter-analyze-ex]]
diff --git a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
index 1f2f61a5071dc..2fa9c41ad79b6 100644
--- a/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/word-delimiter-graph-tokenfilter.asciidoc
@@ -429,7 +429,7 @@ PUT /my_index
 [[analysis-word-delimiter-graph-differences]]
 ==== Differences between `word_delimiter_graph` and `word_delimiter` 
 
-Both the  `word_delimiter_graph` and
+Both the `word_delimiter_graph` and
 <<analysis-word-delimiter-tokenfilter,`word_delimiter`>> filters produce tokens
 that span multiple positions when any of the following parameters are `true`:
 
diff --git a/docs/reference/autoscaling/apis/get-autoscaling-decision.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-decision.asciidoc
index aa66e3a0d034f..dfa14ac180636 100644
--- a/docs/reference/autoscaling/apis/get-autoscaling-decision.asciidoc
+++ b/docs/reference/autoscaling/apis/get-autoscaling-decision.asciidoc
@@ -47,6 +47,6 @@ The API returns the following result:
 [source,console-result]
 --------------------------------------------------
 {
-
+  decisions: []
 }
 --------------------------------------------------
diff --git a/docs/reference/cat/transforms.asciidoc b/docs/reference/cat/transforms.asciidoc
index 78013e394ef24..81c22a31d91b1 100644
--- a/docs/reference/cat/transforms.asciidoc
+++ b/docs/reference/cat/transforms.asciidoc
@@ -6,23 +6,180 @@
 
 Returns configuration and usage information about {transforms}.
 
-
 [[cat-transforms-api-request]]
 ==== {api-request-title}
 
+`GET /_cat/transforms/<transform_id>` +
+
+`GET /_cat/transforms/_all` +
+
+`GET /_cat/transforms/*` +
+
 `GET /_cat/transforms`
 
+[[cat-transforms-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_transform` 
+cluster privileges to use this API. The built-in `transform_user` role has these 
+privileges. For more information, see <<security-privileges>> and 
+<<built-in-roles>>.
 
 //[[cat-transforms-api-desc]]
 //==== {api-description-title}
 
+[[cat-transforms-api-path-params]]
+==== {api-path-parms-title}
+
+`<transform_id>`::
+(Optional, string)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard]
+
+[[cat-transforms-api-query-params]]
+==== {api-query-parms-title}
+
+`allow_no_match`::
+(Optional, boolean)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transforms1]
+
+include::{docdir}/rest-api/common-parms.asciidoc[tag=http-format]
+
+`from`::
+(Optional, integer)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=from-transforms]
+
+include::{docdir}/rest-api/common-parms.asciidoc[tag=cat-h]
++
+If you do not specify which columns to include, the API returns the default
+columns. If you explicitly specify one or more columns, it returns only the
+specified columns.
++
+Valid columns are:
+
+`changes_last_detection_time`, `cldt`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=checkpointing-changes-last-detected-at]
+
+`checkpoint_duration_time_exp_avg`, `cdtea`, `checkpointTimeExpAvg`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=exponential-avg-checkpoint-duration-ms]
+
+`create_time`, `ct`, `createTime`:::
+(Default)
+The time the {transform} was created.
+
+`description`, `d`:::
+(Default)
+The description of the {transform}.
+
+`dest_index`, `di`, `destIndex`:::
+(Default)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=dest-index]
+
+`document_total`, `dt`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=docs-processed]
+
+`frequency`, `f`:::
+(Default)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=frequency]
+
+`id`:::
+(Default)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=transform-id]
+
+`index_failure`, `if`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=index-failures]
+
+`index_time`, `itime`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=index-time-ms]
+
+`index_total`, `it`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=index-total]
+
+`indexed_documents_exp_avg`, `idea`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-indexed]
+
+`invocation_total`, `itotal`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=trigger-count]
+
+`max_page_search_size`, `mpsz`:::
+(Default)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=pivot-max-page-search-size]
+
+`page_total`, `pt`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=pages-processed]
+
+`pipeline`, `p`:::
+(Default)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=dest-pipeline]
+
+`processed_documents_exp_avg`, `pdea`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-processed]
+
+`reason`, `r`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=state-transform-reason]
+
+`search_failure`, `sf`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=search-failures]
+
+`search_time`, `stime`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=search-time-ms]
+
+`search_total`, `st`:::
+include::{docdir}/rest-api/common-parms.asciidoc[tag=search-total]
+
+`source_index`, `si`, `sourceIndex`:::
+(Default)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=source-index-transforms]
+
+`state`, `s`:::
+(Default)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=state-transform]
+
+`transform_type`, `tt`:::
+(Default)
+Indicates the type of {transform}: `batch` or `continuous`. 
+
+`version`, `v`:::
+(Default)
+The version of {es} that existed on the node when the {transform} was
+created.
+
+include::{docdir}/rest-api/common-parms.asciidoc[tag=help]
+
+include::{docdir}/rest-api/common-parms.asciidoc[tag=cat-s]
+
+`size`::
+(Optional, integer)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=size-transforms]
+
+include::{docdir}/rest-api/common-parms.asciidoc[tag=time]
 
-//[[cat-transforms-api-query-params]]
-//==== {api-query-parms-title}
+include::{docdir}/rest-api/common-parms.asciidoc[tag=cat-v]
 
+[[cat-transforms-api-examples]]
+==== {api-examples-title}
 
-//[[cat-transforms-api-response-codes]]
-//==== {api-response-codes-title}
+[source,console]
+--------------------------------------------------
+GET /_cat/transforms?v&format=json
+--------------------------------------------------
+// TEST[skip:kibana sample data]
 
-//[[cat-transforms-api-examples]]
-//==== {api-examples-title}
+[source,console-result]
+----
+[
+  {
+    "id" : "ecommerce_transform",
+    "create_time" : "2020-03-20T20:31:25.077Z",
+    "version" : "7.7.0",
+    "source_index" : "kibana_sample_data_ecommerce",
+    "dest_index" : "kibana_sample_data_ecommerce_transform",
+    "pipeline" : null,
+    "description" : "Maximum priced ecommerce data by customer_id in Asia",
+    "transform_type" : "continuous",
+    "frequency" : "5m",
+    "max_page_search_size" : "500",
+    "state" : "STARTED"
+  }
+]
+----
+// TESTRESPONSE[skip:kibana sample data]
diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc
index 8b54b5fc92ef4..ed424b9121819 100644
--- a/docs/reference/cluster/nodes-stats.asciidoc
+++ b/docs/reference/cluster/nodes-stats.asciidoc
@@ -760,83 +760,131 @@ recovery operations were delayed due to throttling.
 [%collapsible]
 ====
 `os.timestamp`::
-    Last time the operating system statistics have been refreshed.
+    (integer)
+    Last time the operating system statistics were refreshed. Recorded in
+    milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch].
 
 `os.cpu.percent`::
-    Recent CPU usage for the whole system, or -1 if not supported.
+    (integer)
+    Recent CPU usage for the whole system, or `-1` if not supported.
 
 `os.cpu.load_average.1m`::
+    (float)
     One-minute load average on the system (field is not present if one-minute 
     load average is not available).
     
 `os.cpu.load_average.5m`::
+    (float)
     Five-minute load average on the system (field is not present if five-minute 
     load average is not available).
 
 `os.cpu.load_average.15m`::
+    (float)
     Fifteen-minute load average on the system (field is not present if 
     fifteen-minute load average is not available).
 
+`os.mem.total`::
+    (<<byte-units,byte value>>)
+    Total amount of physical memory.
+
 `os.mem.total_in_bytes`::
+    (integer)
     Total amount of physical memory in bytes.
 
+`os.mem.free`::
+    (<<byte-units,byte value>>)
+    Amount of free physical memory.
+
 `os.mem.free_in_bytes`::
+    (integer)
     Amount of free physical memory in bytes.
 
-`os.mem.free_percent`::
-    Percentage of free memory.
+`os.mem.used`::
+    (<<byte-units,byte value>>)
+    Amount of used physical memory.
 
 `os.mem.used_in_bytes`::
+    (integer)
     Amount of used physical memory in bytes.
 
+`os.mem.free_percent`::
+    (integer)
+    Percentage of free memory.
+
 `os.mem.used_percent`::
+    (integer)
     Percentage of used memory.
 
+`os.swap.total`::
+    (<<byte-units,byte value>>)
+    Total amount of swap space.
+
 `os.swap.total_in_bytes`::
+    (integer)
     Total amount of swap space in bytes.
 
+`os.swap.free`::
+    (<<byte-units,byte value>>)
+    Amount of free swap space.
+
 `os.swap.free_in_bytes`::
+    (integer)
     Amount of free swap space in bytes.
 
+`os.swap.used`::
+    (<<byte-units,byte value>>)
+    Amount of used swap space.
+
 `os.swap.used_in_bytes`::
+    (integer)
     Amount of used swap space in bytes.
 
 `os.cgroup.cpuacct.control_group` (Linux only)::
+    (string)
     The `cpuacct` control group to which the {es} process belongs.
 
 `os.cgroup.cpuacct.usage_nanos` (Linux only)::
+    (integer)
     The total CPU time (in nanoseconds) consumed by all tasks in the same cgroup 
     as the {es} process.
 
 `os.cgroup.cpu.control_group` (Linux only)::
+    (string)
     The `cpu` control group to which the {es} process belongs.
 
 `os.cgroup.cpu.cfs_period_micros` (Linux only)::
+    (integer)
     The period of time (in microseconds) for how regularly all tasks in the same 
     cgroup as the {es} process should have their access to CPU resources 
     reallocated.
 
 `os.cgroup.cpu.cfs_quota_micros` (Linux only)::
+    (integer)
     The total amount of time (in microseconds) for which all tasks in
     the same cgroup as the {es} process can run during one period 
     `os.cgroup.cpu.cfs_period_micros`.
 
 `os.cgroup.cpu.stat.number_of_elapsed_periods` (Linux only)::
+    (integer)
     The number of reporting periods (as specified by
     `os.cgroup.cpu.cfs_period_micros`) that have elapsed.
 
 `os.cgroup.cpu.stat.number_of_times_throttled` (Linux only)::
+    (integer)
     The number of times all tasks in the same cgroup as the {es} process have 
     been throttled.
 
 `os.cgroup.cpu.stat.time_throttled_nanos` (Linux only)::
+    (integer)
     The total amount of time (in nanoseconds) for which all tasks in the same 
     cgroup as the {es} process have been throttled.
 
 `os.cgroup.memory.control_group` (Linux only)::
+    (string)
     The `memory` control group to which the {es} process belongs.
 
 `os.cgroup.memory.limit_in_bytes` (Linux only)::
+    (string)
     The maximum amount of user memory (including file cache) allowed for all 
     tasks in the same cgroup as the {es} process. This value can be too big to 
     store in a `long`, so is returned as a string so that the value returned can 
@@ -845,6 +893,7 @@ recovery operations were delayed due to throttling.
     limit has been set for the cgroup.
 
 `os.cgroup.memory.usage_in_bytes` (Linux only)::
+    (string)
     The total current memory usage by processes in the cgroup (in bytes) by all 
     tasks in the same cgroup as the {es} process. This value is stored as a 
     string for consistency with `os.cgroup.memory.limit_in_bytes`.
@@ -860,24 +909,41 @@ must be readable from `/sys/fs/cgroup/cpu` and `/sys/fs/cgroup/cpuacct`.
 [%collapsible]
 ====
 `process.timestamp`::
-    Last time the process statistics have been refreshed.
+    (integer)
+    Last time the process statistics were refreshed. Recorded in milliseconds
+    since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch].
 
 `process.open_file_descriptors`::
-    Number of opened file descriptors associated with the current process, or -1 
-    if not supported.
+    (integer)
+    Number of opened file descriptors associated with the current process, or
+    `-1` if not supported.
 
 `process.max_file_descriptors`::
-    Maximum number of file descriptors allowed on the system, or -1 if not 
+    (integer)
+    Maximum number of file descriptors allowed on the system, or `-1` if not 
     supported.
 
 `process.cpu.percent`::
-    CPU usage in percent, or -1 if not known at the time the stats are computed
+    (integer)
+    CPU usage in percent, or `-1` if not known at the time the stats are
+    computed.
+
+`process.cpu.total`::
+    (<<time-units,time value>>)
+    CPU time used by the process on which the Java virtual machine is running.
 
 `process.cpu.total_in_millis`::
+    (integer)
     CPU time (in milliseconds) used by the process on which the Java virtual 
-    machine is running, or -1 if not supported.
+    machine is running, or `-1` if not supported.
+
+`process.mem.total_virtual`::
+    (<<byte-units,byte value>>)
+    Size of virtual memory that is guaranteed to be available to the 
+    running process.
 
 `process.mem.total_virtual_in_bytes`::
+    (integer)
     Size in bytes of virtual memory that is guaranteed to be available to the 
     running process.
 ====
@@ -1070,85 +1136,154 @@ Number of tasks completed by the thread pool executor.
 [%collapsible]
 ====
 `fs.timestamp`::
-    Last time the file stores statistics have been refreshed.
+    (integer)
+    Last time the file stores statistics were refreshed. Recorded in
+    milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix Epoch].
+
+`fs.total.total`::
+    (<<byte-units,byte value>>)
+    Total size of all file stores.
 
 `fs.total.total_in_bytes`::
+    (integer)
     Total size (in bytes) of all file stores.
 
+`fs.total.free`::
+    (<<byte-units,byte value>>)
+    Total unallocated disk space in all file stores.
+
 `fs.total.free_in_bytes`::
+    (integer)
     Total number of unallocated bytes in all file stores.
 
+`fs.total.available`::
+    (<<byte-units,byte value>>)
+    Total disk space available to this Java virtual machine on all file 
+    stores. Depending on OS or process level restrictions, this might appear 
+    less than `fs.total.free`. This is the actual amount of free disk 
+    space the {es} node can utilise.
+
 `fs.total.available_in_bytes`::
+    (integer)
     Total number of bytes available to this Java virtual machine on all file 
     stores. Depending on OS or process level restrictions, this might appear 
     less than `fs.total.free_in_bytes`. This is the actual amount of free disk 
     space the {es} node can utilise.
 
+`fs.least_usage_estimate`::
+    (object)
+    Object containing statistics for the file store with the least estimated
+    usage. See <<cluster-nodes-stats-fs-data,`fs.data`>> for a list of child
+    parameters.
+    
+
+`fs.most_usage_estimate`::
+    (object)
+    Object containing statistics for the file store with the most estimated
+    usage. See <<cluster-nodes-stats-fs-data,`fs.data`>> for a list of child
+    parameters.
+
+[[cluster-nodes-stats-fs-data]]
 `fs.data`::
+    (array of objects)
     List of all file stores.
 
 `fs.data.path`::
+    (string)
     Path to the file store.
 
 `fs.data.mount`::
+    (string)
     Mount point of the file store (ex: /dev/sda2).
+    +
+    NOTE: This parameter is not provided for the `least_usage_estimate` or 
+    `most_usage_estimate` file stores.
 
 `fs.data.type`::
+    (string)
     Type of the file store (ex: ext4).
 
+`fs.data.total`::
+    (<<byte-units,byte value>>)
+    Total size of the file store.
+
 `fs.data.total_in_bytes`::
+    (integer)
     Total size (in bytes) of the file store.
 
+`fs.data.free`::
+    (<<byte-units,byte value>>)
+    Total amount of unallocated disk space in the file store.
+
 `fs.data.free_in_bytes`::
+    (integer)
     Total number of unallocated bytes in the file store.
 
+`fs.data.available`::
+    (<<byte-units,byte value>>)
+    Total amount of disk space available to this Java virtual machine on this file 
+    store.
+
 `fs.data.available_in_bytes`::
+    (integer)
     Total number of bytes available to this Java virtual machine on this file 
     store.
 
 `fs.io_stats.devices` (Linux only)::
+    (array)
     Array of disk metrics for each device that is backing an {es} data path. 
     These disk metrics are probed periodically and averages between the last 
     probe and the current probe are computed.
 
 `fs.io_stats.devices.device_name` (Linux only)::
+    (string)
     The Linux device name.
 
 `fs.io_stats.devices.operations` (Linux only)::
+    (integer)
     The total number of read and write operations for the device completed since 
     starting {es}.
 
 `fs.io_stats.devices.read_operations` (Linux only)::
+    (integer)
     The total number of read operations for the device completed since starting 
     {es}.
 
 `fs.io_stats.devices.write_operations` (Linux only)::
+    (integer)
     The total number of write operations for the device completed since starting 
     {es}.
 
 `fs.io_stats.devices.read_kilobytes` (Linux only)::
+    (integer)
     The total number of kilobytes read for the device since starting {es}.
 
 `fs.io_stats.devices.write_kilobytes` (Linux only)::
+    (integer)
     The total number of kilobytes written for the device since starting {es}.
 
 `fs.io_stats.operations` (Linux only)::
+    (integer)
     The total number of read and write operations across all devices used by 
     {es} completed since starting {es}.
 
 `fs.io_stats.read_operations` (Linux only)::
+    (integer)
     The total number of read operations for across all devices used by {es} 
     completed since starting {es}.
 
 `fs.io_stats.write_operations` (Linux only)::
+    (integer)
     The total number of write operations across all devices used by {es} 
     completed since starting {es}.
 
 `fs.io_stats.read_kilobytes` (Linux only)::
+    (integer)
     The total number of kilobytes read across all devices used by {es} since 
     starting {es}.
 
 `fs.io_stats.write_kilobytes` (Linux only)::
+    (integer)
     The total number of kilobytes written across all devices used by {es} since 
     starting {es}.
 ====
@@ -1167,6 +1302,10 @@ Number of open TCP connections used for internal communication between nodes.
 Total number of RX (receive) packets received by the node during internal
 cluster communication.
 
+`transport.rx_size`::
+(<<byte-units,byte value>>)
+Size of RX packets received by the node during internal cluster communication.
+
 `transport.rx_size_in_bytes`::
 (integer)
 Size, in bytes, of RX packets received by the node during internal cluster
@@ -1177,6 +1316,10 @@ communication.
 Total number of TX (transmit) packets sent by the node during internal cluster
 communication.
 
+`transport.tx_size`::
+(<<byte-units,byte value>>)
+Size of TX packets sent by the node during internal cluster communication.
+
 `transport.tx_size_in_bytes`::
 (integer)
 Size, in bytes, of TX packets sent by the node during internal cluster
@@ -1287,11 +1430,16 @@ Number of compatible differences between published cluster states.
     (integer)
     Total number of documents ingested during the lifetime of this node.
 
-`ingest.total.time_in_millis`::
-    (integer)
+`ingest.total.time`::
+    (<<time-units,time value>>)
     Total time spent preprocessing ingest documents during the lifetime of this
     node.
 
+`ingest.total.time_in_millis`::
+    (integer)
+    Total time, in milliseconds, spent preprocessing ingest documents during the
+    lifetime of this node.
+
 `ingest.total.current`::
     (integer)
     Total number of documents currently being ingested.
@@ -1304,27 +1452,36 @@ Number of compatible differences between published cluster states.
     (integer)
     Number of documents preprocessed by the ingest pipeline.
 
+`ingest.pipelines.<pipeline_id>.time`::
+    (<<time-units,time value>>)
+    Total time spent preprocessing documents in the ingest pipeline.
+
 `ingest.pipelines.<pipeline_id>.time_in_millis`::
     (integer)
-    Total time spent preprocessing documents in the ingest pipeline.
+    Total time, in milliseconds, spent preprocessing documents in the ingest
+    pipeline.
 
 `ingest.pipelines.<pipeline_id>.failed`::
     (integer)
     Total number of failed operations for the ingest pipeline.
 
-`ingest.pipelines.<pipeline_id>.<processor>.count`::
+`ingest.pipelines.<pipeline_id>.processors.<processor>.count`::
     (integer)
     Number of documents transformed by the processor.
 
-`ingest.pipelines.<pipeline_id>.<processor>.time_in_millis`::
-    (integer)
+`ingest.pipelines.<pipeline_id>.processors.<processor>.time`::
+    (<<time-units,time value>>)
     Time spent by the processor transforming documents.
 
-`ingest.pipelines.<pipeline_id>.<processor>.current`::
+`ingest.pipelines.<pipeline_id>.processors.<processor>.time_in_millis`::
+    (integer)
+    Time, in milliseconds, spent by the processor transforming documents.
+
+`ingest.pipelines.<pipeline_id>.processors.<processor>.current`::
     (integer)
     Number of documents currently being transformed by the processor.
 
-`ingest.pipelines.<pipeline_id>.<processor>.failed`::
+`ingest.pipelines.<pipeline_id>.processors.<processor>.failed`::
     (integer)
     Number of failed operations for the processor.
 ====
@@ -1335,22 +1492,37 @@ Number of compatible differences between published cluster states.
 [%collapsible]
 ====
 `adaptive_selection.outgoing_searches`::
+    (integer)
     The number of outstanding search requests from the node these stats are for 
     to the keyed node.
 
-`avg_queue_size`::
+`adaptive_selection.avg_queue_size`::
+    (integer)
     The exponentially weighted moving average queue size of search requests on 
     the keyed node.
 
-`avg_service_time_ns`::
+`adaptive_selection.avg_service_time`::
+    (<<time-units,time value>>)
     The exponentially weighted moving average service time of search requests on
     the keyed node.
 
-`avg_response_time_ns`::
+`adaptive_selection.avg_service_time_ns`::
+    (integer)
+    The exponentially weighted moving average service time, in nanoseconds, of
+    search requests on the keyed node.
+
+`adaptive_selection.avg_response_time`::
+    (<<time-units,time value>>)
     The exponentially weighted moving average response time of search requests 
     on the keyed node.
 
-`rank`::
+`adaptive_selection.avg_response_time_ns`::
+    (integer)
+    The exponentially weighted moving average response time, in nanoseconds, of
+    search requests on the keyed node.
+
+`adaptive_selection.rank`::
+    (string)
     The rank of this node; used for shard selection when routing search 
     requests.
 ====
diff --git a/docs/reference/commands/keystore.asciidoc b/docs/reference/commands/keystore.asciidoc
index 085f8acb7adcb..83441126d4824 100644
--- a/docs/reference/commands/keystore.asciidoc
+++ b/docs/reference/commands/keystore.asciidoc
@@ -11,8 +11,8 @@ in the {es} keystore.
 [source,shell]
 --------------------------------------------------
 bin/elasticsearch-keystore
-([add <setting>] [-f] [--stdin] |
-[add-file <setting> <path>] | [create] [-p] |
+([add <settings>] [-f] [--stdin] |
+[add-file (<setting> <path>)+] | [create] [-p] |
 [list] | [passwd] | [remove <setting>] | [upgrade])
 [-h, --help] ([-s, --silent] | [-v, --verbose])
 --------------------------------------------------
@@ -40,18 +40,19 @@ keystore, see the setting reference.
 [[elasticsearch-keystore-parameters]]
 === Parameters
 
-`add <setting>`:: Adds settings to the keystore. By default, you are prompted
-for the value of the setting. If the keystore is password protected, you are
-also prompted to enter the password. If the setting already exists in the
-keystore, you must confirm that you want to overwrite the current value. If the
-keystore does not exist, you must confirm that you want to create a keystore. To
-avoid these two confirmation prompts, use the `-f` parameter.
+`add <settings>`:: Adds settings to the keystore. Multiple setting names can be
+specified as arguments to the `add` command. By default, you are prompted for
+the values of the settings. If the keystore is password protected, you are also
+prompted to enter the password. If a setting already exists in the keystore, you
+must confirm that you want to overwrite the current value. If the keystore does
+not exist, you must confirm that you want to create a keystore. To avoid these
+two confirmation prompts, use the `-f` parameter.
 
-`add-file <setting> <path>`:: Adds a file to the keystore.
+`add-file (<setting> <path>)+`:: Adds files to the keystore.
 
 `create`:: Creates the keystore.
 
-`-f`:: When used with the `add` parameter, the command no longer prompts you
+`-f, --force`:: When used with the `add` parameter, the command no longer prompts you
 before overwriting existing entries in the keystore. Also, if you haven't
 created a keystore yet, it creates a keystore that is obfuscated but not
 password protected.
@@ -70,12 +71,14 @@ protected, you are prompted to enter the current password and the new one. You
 can optionally use an empty string to remove the password. If the keystore is
 not password protected, you can use this command to set a password.
 
-`remove <setting>`:: Removes a setting from the keystore.
+`remove <settings>`:: Removes settings from the keystore. Multiple setting
+names can be specified as arguments to the `add` command.
 
 `-s, --silent`:: Shows minimal output.
 
-`--stdin`:: When used with the `add` parameter, you can pass the setting value
-through standard input (stdin). See <<add-string-to-keystore>>.
+`-x, --stdin`:: When used with the `add` parameter, you can pass the settings values
+through standard input (stdin). Separate multiple values with carriage returns
+or newlines. See <<add-string-to-keystore>>.
 
 `upgrade`:: Upgrades the internal format of the keystore.
 
@@ -143,26 +146,50 @@ bin/elasticsearch-keystore add the.setting.name.to.set
 You are prompted to enter the value of the setting. If the {es} keystore is
 password protected, you are also prompted to enter the password.
 
-To pass the setting value through standard input (stdin), use the `--stdin` flag:
+You can also add multiple settings with the `add` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore add \
+  the.setting.name.to.set \
+  the.other.setting.name.to.set
+----------------------------------------------------------------
+
+You are prompted to enter the values of the settings. If the {es} keystore is
+password protected, you are also prompted to enter the password.
+
+To pass the settings values through standard input (stdin), use the `--stdin`
+flag:
 
 [source,sh]
 ----------------------------------------------------------------
 cat /file/containing/setting/value | bin/elasticsearch-keystore add --stdin the.setting.name.to.set
 ----------------------------------------------------------------
 
+Values for multiple settings must be separated by carriage returns or newlines.
+
 [discrete]
 [[add-file-to-keystore]]
 ==== Add files to the keystore
 
 You can add sensitive files, like authentication key files for Cloud plugins,
-using the `add-file` command. Be sure to include your file path as an argument
-after the setting name.
+using the `add-file` command. Settings and file paths are specified in pairs
+consisting of `setting path`.
 
 [source,sh]
 ----------------------------------------------------------------
 bin/elasticsearch-keystore add-file the.setting.name.to.set /path/example-file.json
 ----------------------------------------------------------------
 
+You can add multiple files with the `add-file` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore add-file \
+  the.setting.name.to.set /path/example-file.json \
+  the.other.setting.name.to.set /path/other-example-file.json
+----------------------------------------------------------------
+
 If the {es} keystore is password protected, you are prompted to enter the
 password.
 
diff --git a/docs/reference/eql/functions.asciidoc b/docs/reference/eql/functions.asciidoc
new file mode 100644
index 0000000000000..cc5fbba57e4ea
--- /dev/null
+++ b/docs/reference/eql/functions.asciidoc
@@ -0,0 +1,66 @@
+[[eql-function-ref]]
+== EQL function reference
+++++
+<titleabbrev>Function reference</titleabbrev>
+++++
+
+experimental::[]
+
+{es} supports the following EQL functions:
+
+* <<eql-fn-substring>>
+
+[discrete]
+[[eql-fn-substring]]
+=== `substring`
+
+Extracts a substring from a source string at provided start and end positions.
+
+If no end position is provided, the function extracts the remaining string.
+
+[%collapsible]
+====
+*Example*
+[source,eql]
+----
+substring("start regsvr32.exe", 6)        // returns "regsvr32.exe"
+substring("start regsvr32.exe", 0, 5)     // returns "start"
+substring("start regsvr32.exe", 6, 14)    // returns "regsvr32"
+substring("start regsvr32.exe", -4)       // returns ".exe"
+substring("start regsvr32.exe", -4, -1)   // returns ".ex"
+----
+
+*Syntax*
+
+[source,txt]
+----
+substring(<source>, <start_pos>[, <end_pos>])
+----
+
+*Parameters*
+
+`<source>`::
+(Required, string)
+Source string.
+
+`<start_pos>`::
++
+--
+(Required, integer)
+Starting position for extraction.
+
+If this position is higher than the `<end_pos>` position or the length of the
+`<source>` string, the function returns an empty string.
+
+Positions are zero-indexed. Negative offsets are supported.
+--
+
+`<end_pos>`::
+(Optional, integer)
+Exclusive end position for extraction. If this position is not provided, the
+function returns the remaining string.
++
+Positions are zero-indexed. Negative offsets are supported.
+
+*Returns:* string
+====
\ No newline at end of file
diff --git a/docs/reference/eql/index.asciidoc b/docs/reference/eql/index.asciidoc
index 328f40a36a557..f9168b861c9b7 100644
--- a/docs/reference/eql/index.asciidoc
+++ b/docs/reference/eql/index.asciidoc
@@ -32,9 +32,11 @@ Consider using EQL if you:
 * <<eql-requirements>>
 * <<eql-search>>
 * <<eql-syntax>>
+* <<eql-function-ref>>
 * <<eql-limitations>>
 
 include::requirements.asciidoc[]
 include::search.asciidoc[]
 include::syntax.asciidoc[]
+include::functions.asciidoc[]
 include::limitations.asciidoc[]
diff --git a/docs/reference/eql/limitations.asciidoc b/docs/reference/eql/limitations.asciidoc
index 5e6cc74c319c2..dc7a211d8bd34 100644
--- a/docs/reference/eql/limitations.asciidoc
+++ b/docs/reference/eql/limitations.asciidoc
@@ -15,7 +15,10 @@ experimental::[]
 {es} supports a subset of {eql-ref}/index.html[EQL syntax]. {es} cannot run EQL
 queries that contain:
 
-* {eql-ref}/functions.html[Functions]
+* Array functions:
+** {eql-ref}/functions.html#arrayContains[`arrayContains`]
+** {eql-ref}/functions.html#arrayCount[`arrayCount`]
+** {eql-ref}/functions.html#arraySearch[`arraySearch`]
 
 * {eql-ref}/joins.html[Joins]
 
diff --git a/docs/reference/eql/syntax.asciidoc b/docs/reference/eql/syntax.asciidoc
index 9beb9598bb9b9..a3827cfc58ab7 100644
--- a/docs/reference/eql/syntax.asciidoc
+++ b/docs/reference/eql/syntax.asciidoc
@@ -2,6 +2,9 @@
 [testenv="basic"]
 [[eql-syntax]]
 == EQL syntax reference
+++++
+<titleabbrev>Syntax reference</titleabbrev>
+++++
 
 experimental::[]
 
@@ -283,3 +286,12 @@ dots (`.`), hyphens (`-`), or spaces, must be escaped using backticks (+++`+++).
 `my-field`
 `my field`
 ----
+
+[discrete]
+[[eql-functions]]
+=== Functions
+
+{es} supports several of EQL's built-in functions. You can use these functions
+to convert data types, perform math, manipulate strings, and more.
+
+For a list of supported functions, see <<eql-function-ref>>.
\ No newline at end of file
diff --git a/docs/reference/frozen-indices.asciidoc b/docs/reference/frozen-indices.asciidoc
index b6cafad30f5dc..5a2ef125a8cc3 100644
--- a/docs/reference/frozen-indices.asciidoc
+++ b/docs/reference/frozen-indices.asciidoc
@@ -74,8 +74,8 @@ POST /twitter/_forcemerge?max_num_segments=1
 == Searching a frozen index
 
 Frozen indices are throttled in order to limit memory consumptions per node. The number of concurrently loaded frozen indices per node is
-limited by the number of threads in the <<search-throttled,search_throttled>> threadpool,  which is `1` by default. 
-Search requests will not be executed against frozen indices by default, even if a frozen index is named explicitly. This is 
+limited by the number of threads in the <<search-throttled,search_throttled>> threadpool,  which is `1` by default.
+Search requests will not be executed against frozen indices by default, even if a frozen index is named explicitly. This is
 to prevent accidental slowdowns by targeting a frozen index by mistake. To include frozen indices a search request must be executed with
 the query parameter `ignore_throttled=false`.
 
@@ -85,15 +85,6 @@ GET /twitter/_search?q=user:kimchy&ignore_throttled=false
 --------------------------------------------------
 // TEST[setup:twitter]
 
-[IMPORTANT]
-================================
-While frozen indices are slow to search, they can be pre-filtered efficiently. The request parameter `pre_filter_shard_size` specifies
-a threshold that, when exceeded, will enforce a round-trip to pre-filter search shards that cannot possibly match.
-This filter phase can limit the number of shards significantly. For instance, if a date range filter is applied, then all indices (frozen or unfrozen) that do not contain documents within the date range can be skipped efficiently.
-The default value for `pre_filter_shard_size` is `128` but it's recommended to set it to `1` when searching frozen indices. There is no
-significant overhead associated with this pre-filter phase.
-================================
-
 [role="xpack"]
 [testenv="basic"]
 [[monitoring_frozen_indices]]
diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc
index 9f6ed505eb09f..1b73bdd7a3592 100644
--- a/docs/reference/index-modules/similarity.asciidoc
+++ b/docs/reference/index-modules/similarity.asciidoc
@@ -124,7 +124,7 @@ This similarity has the following options:
     {lucene-core-javadoc}/org/apache/lucene/search/similarities/IndependenceSaturated.html[`saturated`],
     {lucene-core-javadoc}/org/apache/lucene/search/similarities/IndependenceChiSquared.html[`chisquared`].
 
-When using this similarity, it is highly recommended to remove stop words to get
+When using this similarity, it is highly recommended *not* to remove stop words to get
 good relevance. Also beware that terms whose frequency is less than the expected
 frequency will get a score equal to 0.
 
diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc
index b1b110e2b3889..a97188978dce7 100644
--- a/docs/reference/ingest/processors/inference.asciidoc
+++ b/docs/reference/ingest/processors/inference.asciidoc
@@ -34,7 +34,6 @@ include::common-options.asciidoc[]
 // NOTCONSOLE
 
 
-
 [discrete]
 [[inference-processor-regression-opt]]
 ==== {regression-cap} configuration options
@@ -51,6 +50,7 @@ Specifies the maximum number of
 importance] values per document. By default, it is zero and no feature importance
 calculation occurs.
 
+
 [discrete]
 [[inference-processor-classification-opt]]
 ==== {classification-cap} configuration options
@@ -73,8 +73,9 @@ Specifies the field to which the top classes are written. Defaults to
 (Optional, integer)
 Specifies the maximum number of
 {ml-docs}/dfa-classification.html#dfa-classification-feature-importance[feature
-importance] values per document. By default, it is zero and no feature importance
-calculation occurs.
+importance] values per document. By default, it is zero and no feature 
+importance calculation occurs.
+
 
 [discrete]
 [[inference-processor-config-example]]
@@ -116,3 +117,65 @@ categories for which the predicted probabilities are reported is 2
 (`num_top_classes`). The result is written to the `prediction` field and the top 
 classes to the `probabilities` field. Both fields are contained in the 
 `target_field` results object.
+
+
+[discrete]
+[[inference-processor-feature-importance]]
+==== {feat-imp-cap} object mapping
+
+Update your index mapping of the {feat-imp} result field as you can see below to 
+get the full benefit of aggregating and searching for 
+{ml-docs}/dfa-classification.html#dfa-classification-feature-importance[{feat-imp}].
+
+[source,js]
+--------------------------------------------------
+"ml.inference.feature_importance": {
+  "type": "nested",
+  "dynamic": true,
+  "properties": {
+    "feature_name": {
+      "type": "keyword"
+    },
+    "importance": {
+      "type": "double"
+    }
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+The mapping field name for {feat-imp} is compounded as follows:
+
+`<ml.inference.target_field>`.`<inference.tag>`.`feature_importance`
+
+If `inference.tag` is not provided in the processor definition, it is not part 
+of the field path. The `<ml.inference.target_field>` defaults to `ml.inference`.
+
+For example, you provide a tag `foo` in the definition as you can see below:
+
+[source,js]
+--------------------------------------------------
+{
+  "tag": "foo",
+  ...
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+
+The `{feat-imp}` value is written to the `ml.inference.foo.feature_importance` 
+field.
+
+You can also specify a target field as follows:
+
+[source,js]
+--------------------------------------------------
+{
+  "tag": "foo",
+  "target_field": "my_field"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+In this case, `{feat-imp}` is exposed in the 
+`my_field.foo.feature_importance` field.
\ No newline at end of file
diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc
index 3f89571ae5808..96153f5e2a355 100644
--- a/docs/reference/mapping.asciidoc
+++ b/docs/reference/mapping.asciidoc
@@ -13,29 +13,22 @@ are stored and indexed. For instance, use mappings to define:
 * custom rules to control the mapping for
   <<dynamic-mapping,dynamically added fields>>.
 
-[float]
-[[mapping-type]]
-== Mapping Type
-
-Each index has one _mapping type_ which determines how the document will be
-indexed.
-
-deprecated::[6.0.0,See <<removal-of-types>>]
-
-A mapping type has:
+A mapping definition has:
 
 <<mapping-fields,Meta-fields>>::
 
 Meta-fields are used to customize how a document's metadata associated is
 treated. Examples of meta-fields include the document's
-<<mapping-index-field,`_index`>>, <<mapping-type-field,`_type`>>,
-<<mapping-id-field,`_id`>>,  and <<mapping-source-field,`_source`>> fields.
+<<mapping-index-field,`_index`>>, <<mapping-id-field,`_id`>>, and
+<<mapping-source-field,`_source`>> fields.
 
 <<mapping-types,Fields>> or _properties_::
 
-A mapping type contains a list of fields or `properties` pertinent to the
+A mapping contains a list of fields or `properties` pertinent to the
 document.
 
+NOTE: Before 7.0.0, the 'mappings' definition used to include a type name.
+For more details, please see <<removal-of-types>>.
 
 [float]
 [[field-datatypes]]
diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc
index f6d5f00a9b5e0..0ea4b77441c42 100644
--- a/docs/reference/mapping/fields.asciidoc
+++ b/docs/reference/mapping/fields.asciidoc
@@ -15,7 +15,7 @@ can be customised when a mapping type is created.
 
 <<mapping-type-field,`_type`>>::
 
-    The document's <<mapping-type,mapping type>>.
+    The document's mapping type.
 
 <<mapping-id-field,`_id`>>::
 
diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc
index 25c3571f08e8d..46a204ccddfde 100644
--- a/docs/reference/mapping/fields/routing-field.asciidoc
+++ b/docs/reference/mapping/fields/routing-field.asciidoc
@@ -92,7 +92,7 @@ PUT my_index2/_doc/1 <2>
 ------------------------------
 // TEST[catch:bad_request]
 
-<1> Routing is required for `_doc` documents.
+<1> Routing is required for all documents.
 <2> This index request throws a `routing_missing_exception`.
 
 ==== Unique IDs with custom routing
@@ -128,4 +128,4 @@ less than `index.number_of_shards`.
 Once enabled, the partitioned index will have the following limitations:
 
 *   Mappings with <<parent-join,`join` field>> relationships cannot be created within it.
-*   All mappings within the index must have the `_routing` field marked as required.
\ No newline at end of file
+*   All mappings within the index must have the `_routing` field marked as required.
diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index 044729b84eece..4e6126cde9c9d 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -3,9 +3,9 @@
 
 deprecated[6.0.0,See <<removal-of-types>>]
 
-Each document indexed is associated with a <<mapping-type-field,`_type`>> (see
-<<mapping-type>>) and an <<mapping-id-field,`_id`>>.  The `_type` field is
-indexed in order to make searching by type name fast.
+Each document indexed is associated with a <<mapping-type-field,`_type`>> and
+an <<mapping-id-field,`_id`>>.  The `_type` field is indexed in order to make
+searching by type name fast.
 
 The value of the `_type` field is accessible in queries, aggregations,
 scripts, and when sorting:
diff --git a/docs/reference/mapping/types/date_nanos.asciidoc b/docs/reference/mapping/types/date_nanos.asciidoc
index aadae3238cdf9..c5dc9481d3191 100644
--- a/docs/reference/mapping/types/date_nanos.asciidoc
+++ b/docs/reference/mapping/types/date_nanos.asciidoc
@@ -94,6 +94,6 @@ same mapping parameters than with the `date` field can be used.
 [[date-nanos-limitations]]
 ==== Limitations
 
-Aggregations are still on millisecond resolution, even when using a
-`date_nanos` field.
+Aggregations are still on millisecond resolution, even when using a `date_nanos`
+field. This limitation also affects <<transforms,{transforms}>>.
 
diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc
index 9d01f4820a894..aa2de7db87afc 100644
--- a/docs/reference/mapping/types/wildcard.asciidoc
+++ b/docs/reference/mapping/types/wildcard.asciidoc
@@ -48,6 +48,23 @@ POST my_index/_doc/_search
 --------------------------------------------------
 
 
+[[wildcard-params]]
+==== Parameters for wildcard fields
+
+The following parameters are accepted by `wildcard` fields:
+
+[horizontal]
+
+<<ignore-above,`ignore_above`>>::
+
+    Do not index any string longer than this value.  Defaults to `2147483647`
+    so that all values would be accepted.
+
+<<normalizer,`normalizer`>>::
+
+    How to pre-process the value prior to indexing. Defaults to `null`,
+    meaning the value is kept as-is.
+
 ==== Limitations
 
 * `wildcard` fields are untokenized like keyword fields, so do not support queries that rely on word positions such as phrase queries.
diff --git a/docs/reference/migration/migrate_8_0/settings.asciidoc b/docs/reference/migration/migrate_8_0/settings.asciidoc
index 82ef5b8190adf..663133fed9488 100644
--- a/docs/reference/migration/migrate_8_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_8_0/settings.asciidoc
@@ -37,3 +37,10 @@ processors. As this leads to more context switches and more threads but without
 an increase in the number of physical CPUs on which to schedule these additional
 threads, the `node.processors` setting is now bounded by the number of available
 processors.
+
+[float]
+==== `cluster.remote.connect` is removed
+
+In Elasticsearch 7.7.0, the setting `cluster.remote.connect` was deprecated in
+favor of setting `node.remote_cluster_client`. In Elasticsearch 8.0.0, the
+setting `cluster.remote.connect` is removed.
diff --git a/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc b/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc
new file mode 100644
index 0000000000000..46a49f7cd239c
--- /dev/null
+++ b/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc
@@ -0,0 +1,87 @@
+[role="xpack"]
+[testenv="platinum"]
+[[ml-estimate-model-memory]]
+=== Estimate {anomaly-jobs} model memory API
+++++
+<titleabbrev>Estimate model memory</titleabbrev>
+++++
+
+Estimates the model memory an {anomaly-job} is likely to need based on analysis
+configuration details and cardinality estimates for the fields it references.
+
+[[ml-estimate-model-memory-request]]
+==== {api-request-title}
+
+`POST _ml/anomaly_detectors/_estimate_model_memory`
+
+[[ml-estimate-model-memory-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+<<security-privileges>>.
+
+[[ml-estimate-model-memory-request-body]]
+==== {api-request-body-title}
+
+`analysis_config`::
+(Required, object) For a list of the properties that you can specify in the
+`analysis_config` component of the body of this API, see <<put-analysisconfig>>.
+
+`max_bucket_cardinality`::
+(Optional, object) Estimates of the highest cardinality in a single bucket
+that will be observed for influencer fields over the time period that the job
+analyzes data. To produce a good answer, values must be provided for
+all influencer fields. It does not matter if values are provided for fields
+that are not listed as `influencers`. If there are no `influencers` then
+`max_bucket_cardinality` can be omitted from the request.
+
+`overall_cardinality`::
+(Optional, object) Estimates of the cardinality that will be observed for
+fields over the whole time period that the job analyzes data. To produce
+a good answer, values must be provided for fields referenced in the
+`by_field_name`, `over_field_name` and `partition_field_name` of any
+detectors. It does not matter if values are provided for other fields.
+If no detectors have a `by_field_name`, `over_field_name` or
+`partition_field_name` then `overall_cardinality` can be omitted
+from the request.
+
+[[ml-estimate-model-memory-example]]
+==== {api-examples-title}
+
+[source,console]
+--------------------------------------------------
+POST _ml/anomaly_detectors/_estimate_model_memory
+{
+    "analysis_config": {
+        "bucket_span": "5m",
+        "detectors": [
+          {
+            "function": "sum",
+            "field_name": "bytes",
+            "by_field_name": "status",
+            "partition_field_name": "app"
+          }
+        ],
+        "influencers": [ "source_ip", "dest_ip" ]
+    },
+    "overall_cardinality": {
+       "status": 10,
+       "app": 50
+    },
+    "max_bucket_cardinality": {
+       "source_ip": 300,
+       "dest_ip": 30
+    }
+}
+--------------------------------------------------
+// TEST[skip:needs-licence]
+
+The estimate returns the following result:
+
+[source,console-result]
+----
+{
+  "model_memory_estimate": "45mb"
+}
+----
diff --git a/docs/reference/ml/anomaly-detection/apis/ml-api.asciidoc b/docs/reference/ml/anomaly-detection/apis/ml-api.asciidoc
index f02312cb0ac94..e6514dff60a00 100644
--- a/docs/reference/ml/anomaly-detection/apis/ml-api.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/ml-api.asciidoc
@@ -118,6 +118,8 @@ include::delete-job.asciidoc[]
 include::delete-calendar-job.asciidoc[]
 include::delete-snapshot.asciidoc[]
 include::delete-expired-data.asciidoc[]
+//ESTIMATE
+include::estimate-model-memory.asciidoc[]
 //FIND
 include::find-file-structure.asciidoc[]
 //FLUSH
diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
index 16c9f486d8d63..1d559f85866df 100644
--- a/docs/reference/ml/ml-shared.asciidoc
+++ b/docs/reference/ml/ml-shared.asciidoc
@@ -798,8 +798,8 @@ An array of index names. Wildcards are supported. For example:
 `["it_ops_metrics", "server*"]`.
 +
 --
-NOTE: If any indices are in remote clusters then `cluster.remote.connect` must
-not be set to `false` on any {ml} nodes.
+NOTE: If any indices are in remote clusters then `node.remote_cluster_client`
+must not be set to `false` on any {ml} nodes.
 
 --
 end::indices[]
@@ -1388,7 +1388,10 @@ tag::state-datafeed[]
 The status of the {dfeed}, which can be one of the following values:
 +
 --
+* `starting`: The {dfeed} has been requested to start but has not yet started.
 * `started`: The {dfeed} is actively receiving data.
+* `stopping`: The {dfeed} has been requested to stop gracefully and is
+completing its final action.
 * `stopped`: The {dfeed} is stopped and will not receive data until it is
 re-started.
 --
diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc
index 47b64ac4bd560..380d708e80eb7 100644
--- a/docs/reference/modules/node.asciidoc
+++ b/docs/reference/modules/node.asciidoc
@@ -1,8 +1,8 @@
 [[modules-node]]
 == Node
 
-Any time that you start an instance of Elasticsearch, you are starting a _node_. 
-A collection of connected nodes is called a <<modules-cluster,cluster>>. If you 
+Any time that you start an instance of Elasticsearch, you are starting a _node_.
+A collection of connected nodes is called a <<modules-cluster,cluster>>. If you
 are running a single node of {es}, then you have a cluster of one node.
 
 Every node in the cluster can handle <<modules-http,HTTP>> and
@@ -11,7 +11,7 @@ exclusively for communication between nodes; the HTTP layer is used by REST
 clients.
 
 All nodes know about all the other nodes in the cluster and can forward client
-requests to the appropriate node. 
+requests to the appropriate node.
 
 By default, a node is all of the following types: master-eligible, data, ingest,
 and machine learning (if available).
@@ -95,22 +95,22 @@ be elected to become the master node by the <<modules-discovery,master election
 process>>.
 
 IMPORTANT: Master nodes must have access to the `data/` directory (just like
-`data` nodes) as this is where the cluster state is persisted between node 
+`data` nodes) as this is where the cluster state is persisted between node
 restarts.
 
 [float]
 [[dedicated-master-node]]
 ==== Dedicated master-eligible node
 
-It is important for the health of the cluster that the elected master node has 
-the resources it needs to fulfill its responsibilities. If the elected master 
-node is overloaded with other tasks then the cluster may not operate well. In 
-particular, indexing and searching your data can be very resource-intensive, so 
-in large or high-throughput clusters it is a good idea to avoid using the 
-master-eligible nodes for tasks such as indexing and searching. You can do this 
-by configuring three of your nodes to be dedicated master-eligible nodes. 
+It is important for the health of the cluster that the elected master node has
+the resources it needs to fulfill its responsibilities. If the elected master
+node is overloaded with other tasks then the cluster may not operate well. In
+particular, indexing and searching your data can be very resource-intensive, so
+in large or high-throughput clusters it is a good idea to avoid using the
+master-eligible nodes for tasks such as indexing and searching. You can do this
+by configuring three of your nodes to be dedicated master-eligible nodes.
 Dedicated master-eligible nodes only have the `master` role, allowing them to
-focus on managing the cluster. While master nodes can also behave as 
+focus on managing the cluster. While master nodes can also behave as
 <<coordinating-node,coordinating nodes>> and route search and indexing requests
 from clients to data nodes, it is better _not_ to use dedicated master nodes for
 this purpose.
@@ -127,7 +127,7 @@ node.ml: false <5>
 xpack.ml.enabled: true <6>
 node.transform: false <7>
 xpack.transform.enabled: true <8>
-cluster.remote.connect: false <9>
+node.remote_client_client: false <9>
 -------------------
 <1> The `node.master` role is enabled by default.
 <2> The `node.voting_only` role is disabled by default.
@@ -146,7 +146,7 @@ To create a dedicated master-eligible node in the {oss-dist}, set:
 node.master: true <1>
 node.data: false <2>
 node.ingest: false <3>
-cluster.remote.connect: false <4>
+node.remote_cluster_client: false <4>
 -------------------
 <1> The `node.master` role is enabled by default.
 <2> Disable the `node.data` role (enabled by default).
@@ -210,7 +210,7 @@ node.ml: false <5>
 xpack.ml.enabled: true <6>
 node.transform: false <7>
 xpack.transform.enabled: true <8>
-cluster.remote.connect: false <9>
+node.remote_cluster_client: false <9>
 -------------------
 <1> The `node.master` role is enabled by default.
 <2> Enable the `node.voting_only` role (disabled by default).
@@ -243,7 +243,7 @@ node.data: true <3>
 node.ingest: false <4>
 node.ml: false <5>
 node.transform: false <6>
-cluster.remote.connect: false <7>
+node.remote_cluster_client: false <7>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
@@ -259,7 +259,7 @@ To create a dedicated data node in the {oss-dist}, set:
 node.master: false <1>
 node.data: true <2>
 node.ingest: false <3>
-cluster.remote.connect: false <4>
+node.remote_cluster_client: false <4>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.data` role is enabled by default.
@@ -285,7 +285,7 @@ node.data: false <3>
 node.ingest: true <4>
 node.ml: false <5>
 node.transform: false <6>
-cluster.remote.connect: false <7>
+node.remote_cluster_client: false <7>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
@@ -302,7 +302,7 @@ To create a dedicated ingest node in the {oss-dist}, set:
 node.master: false <1>
 node.data: false <2>
 node.ingest: true <3>
-cluster.remote.connect: false <4>
+node.remote_cluster_client: false <4>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> Disable the `node.data` role (enabled by default).
@@ -340,7 +340,7 @@ node.data: false <3>
 node.ingest: false <4>
 node.ml: false <5>
 node.transform: false <6>
-cluster.remote.connect: false <7>
+node.remote_cluster_client: false <7>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
@@ -357,7 +357,7 @@ To create a dedicated coordinating node in the {oss-dist}, set:
 node.master: false <1>
 node.data: false <2>
 node.ingest: false <3>
-cluster.remote.connect: false <4>
+node.remote_cluster_client: false <4>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> Disable the `node.data` role (enabled by default).
@@ -390,7 +390,7 @@ node.ml: true <5>
 xpack.ml.enabled: true <6>
 node.transform: false <7>
 xpack.transform.enabled: true <8>
-cluster.remote.connect: false <9>
+node.remote_cluster_client: false <9>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
@@ -425,7 +425,7 @@ node.ingest: false <4>
 node.ml: false <5>
 node.transform: true <6>
 xpack.transform.enabled: true <7>
-cluster.remote.connect: false <8>
+node.remote_cluster_client: false <8>
 -------------------
 <1> Disable the `node.master` role.
 <2> Disable the `node.voting_only`.
diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc
index 07b9fd23f96dc..426f79f0f4f65 100644
--- a/docs/reference/modules/remote-clusters.asciidoc
+++ b/docs/reference/modules/remote-clusters.asciidoc
@@ -254,12 +254,12 @@ and <<remote-cluster-proxy-settings,proxy mode settings>> are described below.
   The time to wait for remote connections to be established when the node
   starts. The default is `30s`.
 
-`cluster.remote.connect`::
+`node.remote_cluster_client`::
 
   By default, any node in the cluster can act as a cross-cluster client and
-  connect to remote clusters. The `cluster.remote.connect` setting can be set to
-  `false` (defaults to `true`) to prevent certain nodes from connecting to
-  remote clusters. Remote cluster requests must be sent to a node that is
+  connect to remote clusters. The `node.remote_cluster_client` setting can be
+  set to `false` (defaults to `true`) to prevent certain nodes from connecting
+  to remote clusters. Remote cluster requests must be sent to a node that is
   allowed to act as a cross-cluster client.
 
 `cluster.remote.<cluster_alias>.skip_unavailable`::
diff --git a/docs/reference/monitoring/how-monitoring-works.asciidoc b/docs/reference/monitoring/how-monitoring-works.asciidoc
index dc1c54ce9b865..2a89f0aec6524 100644
--- a/docs/reference/monitoring/how-monitoring-works.asciidoc
+++ b/docs/reference/monitoring/how-monitoring-works.asciidoc
@@ -22,7 +22,7 @@ To learn how to collect monitoring data, see:
 * <<collecting-monitoring-data>>
 * <<configuring-metricbeat>>
 * {kibana-ref}/xpack-monitoring.html[Monitoring {kib}]
-* {logstash-ref}/monitoring-logstash.html[Monitoring {ls}]
+* {logstash-ref}/configuring-logstash.html[Monitoring {ls}]
 * Monitoring Beats:
 ** {auditbeat-ref}/monitoring.html[{auditbeat}]
 ** {filebeat-ref}/monitoring.html[{filebeat}]
diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index 03d54def6a054..4425b079d0551 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -385,8 +385,3 @@ coming::[7.x]
 === Data stream APIs
 
 coming::[7.x]
-
-[role="exclude",id="cat-transform"]
-=== cat transform API
-
-See <<cat-transforms>>.
\ No newline at end of file
diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc
index d297b70fa5553..af19d8efeda96 100644
--- a/docs/reference/rest-api/common-parms.asciidoc
+++ b/docs/reference/rest-api/common-parms.asciidoc
@@ -99,6 +99,10 @@ tag::bytes[]
 (Optional, <<byte-units,byte size units>>) Unit used to display byte values.
 end::bytes[]
 
+tag::checkpointing-changes-last-detected-at[]
+The timestamp when changes were last detected in the source indices.
+end::checkpointing-changes-last-detected-at[]
+
 tag::cluster-health-status[]
 (string)
 Health status of the cluster, based on the state of its primary and replica
@@ -178,6 +182,11 @@ is based on Lucene documents. {es} reclaims the disk space of deleted Lucene
 documents when a segment is merged.
 end::docs-deleted[]
 
+tag::docs-processed[]
+The number of documents that have been processed from the source index of
+the {transform}.
+end::docs-processed[]
+
 tag::enrich-policy[]
 Enrich policy name
 used to limit the request.
@@ -207,6 +216,20 @@ Wildcard expressions are not accepted.
 --
 end::expand-wildcards[]
 
+tag::exponential-avg-checkpoint-duration-ms[]
+Exponential moving average of the duration of the checkpoint, in milliseconds.
+end::exponential-avg-checkpoint-duration-ms[]
+
+tag::exponential-avg-documents-indexed[]
+Exponential moving average of the number of new documents that have been
+indexed.
+end::exponential-avg-documents-indexed[]
+
+tag::exponential-avg-documents-processed[]
+Exponential moving average of the number of documents that have been
+processed.
+end::exponential-avg-documents-processed[]
+
 tag::field_statistics[]
 `field_statistics`::
 (Optional, boolean) If `true`, the response includes the document count, sum of document frequencies,
@@ -380,6 +403,18 @@ tag::index[]
 used to limit the request.
 end::index[]
 
+tag::index-failures[]
+The number of indexing failures.
+end::index-failures[]
+
+tag::index-time-ms[]
+The amount of time spent indexing, in milliseconds.
+end::index-time-ms[]
+
+tag::index-total[]
+The number of indices created.
+end::index-total[]
+
 tag::bulk-index[]
 `_index`::
 (Optional, string)
@@ -574,6 +609,11 @@ tag::pipeline[]
 (Optional, string) ID of the pipeline to use to preprocess incoming documents.
 end::pipeline[]
 
+tag::pages-processed[]
+The number of search or bulk index operations processed. Documents are
+processed in batches instead of individually.
+end::pages-processed[]
+
 tag::path-pipeline[]
 `<pipeline>`::
 (Optional, string) Comma-separated list or wildcard expression of pipeline IDs
@@ -708,6 +748,14 @@ tag::scroll_size[]
 Defaults to 100.
 end::scroll_size[]
 
+tag::search-failures[]
+The number of search failures.
+end::search-failures[]
+
+tag::search-time-ms[]
+The amount of time spent searching, in milliseconds.
+end::search-time-ms[]
+
 tag::search_timeout[]
 `timeout`::
 (Optional, <<time-units, time units>>)
@@ -715,6 +763,10 @@ Explicit timeout for each search request.
 Defaults to no timeout.
 end::search_timeout[]
 
+tag::search-total[]
+The number of search operations on the source index for the {transform}.
+end::search-total[]
+
 tag::search_type[]
 `search_type`::
 (Optional, string) The type of the search operation. Available options:
@@ -804,6 +856,26 @@ A query clause that retrieves a subset of data from the source index. See
 <<query-dsl>>.
 end::source-query-transforms[]
 
+tag::state-transform[]
+The status of the {transform}, which can be one of the following values:
++
+--
+* `aborting`: The {transform} is aborting.
+* `failed`: The {transform} failed. For more information about the failure,
+check the reason field.
+* `indexing`: The {transform} is actively processing data and creating new
+documents.
+* `started`: The {transform} is running but not actively indexing data.
+* `stopped`: The {transform} is stopped.
+* `stopping`: The {transform} is stopping.
+--
+end::state-transform[]
+
+tag::state-transform-reason[]
+If a {transform} has a `failed` state, this property provides details about the
+reason for the failure.
+end::state-transform-reason[]
+
 tag::stats[]
 `stats`::
 (Optional, string) Specific `tag` of the request for logging and statistical
@@ -899,6 +971,13 @@ expression. If you do not specify one of these options, the API returns
 information for all {transforms}.
 end::transform-id-wildcard[]
 
+tag::trigger-count[]
+The number of times the {transform} has been triggered by the scheduler. For 
+example, the scheduler triggers the {transform} indexer to check for updates
+or ingest new data at an interval specified in the
+<<put-transform-request-body,`frequency` property>>.
+end::trigger-count[]
+
 tag::cat-v[]
 `v`::
 (Optional, boolean) If `true`, the response includes column headings.
diff --git a/docs/reference/rollup/apis/get-job.asciidoc b/docs/reference/rollup/apis/get-job.asciidoc
index de45f96549f7f..a13ac77bbb884 100644
--- a/docs/reference/rollup/apis/get-job.asciidoc
+++ b/docs/reference/rollup/apis/get-job.asciidoc
@@ -142,14 +142,16 @@ The API yields the following response:
             "index_total": 0,
             "search_failures": 0,
             "search_time_in_ms": 0,
-            "search_total": 0
+            "search_total": 0,
+            "processing_time_in_ms": 0,
+            "processing_total": 0
           }
         }
     ]
 }
 ----
 
-The `jobs` array contains a single job (`id: sensor`) since we requested a single job in the endpoint's URL. 
+The `jobs` array contains a single job (`id: sensor`) since we requested a single job in the endpoint's URL.
 If we add another job, we can see how multi-job responses are handled:
 
 [source,console]
@@ -245,7 +247,9 @@ Which will yield the following response:
             "index_total": 0,
             "search_failures": 0,
             "search_time_in_ms": 0,
-            "search_total": 0
+            "search_total": 0,
+            "processing_time_in_ms": 0,
+            "processing_total": 0
           }
         },
         {
@@ -299,7 +303,9 @@ Which will yield the following response:
             "index_total": 0,
             "search_failures": 0,
             "search_time_in_ms": 0,
-            "search_total": 0
+            "search_total": 0,
+            "processing_time_in_ms": 0,
+            "processing_total": 0
           }
         }
     ]
diff --git a/docs/reference/search/async-search.asciidoc b/docs/reference/search/async-search.asciidoc
index 7f5957a94e41c..b01e3d19f49f2 100644
--- a/docs/reference/search/async-search.asciidoc
+++ b/docs/reference/search/async-search.asciidoc
@@ -31,7 +31,7 @@ POST /sales*/_async_search?size=0
 }
 --------------------------------------------------
 // TEST[setup:sales]
-// TEST[s/size=0/size=0&wait_for_completion=10s&clean_on_completion=false/]
+// TEST[s/size=0/size=0&wait_for_completion_timeout=10s&keep_on_completion=true/]
 
 The response contains an identifier of the search being executed.
 You can use this ID to later retrieve the search's final results.
@@ -88,8 +88,8 @@ results are returned as part of the <<search-api-response-body,`response`>> obje
 <6> How many documents are currently matching the query, which belong to the shards that have already completed the search
 
 It is possible to block and wait until the search is completed up to a certain
-timeout by providing the `wait_for_completion` parameter, which defaults to
-`1` second.
+timeout by providing the `wait_for_completion_timeout` parameter, which
+defaults to `1` second.
 
 You can also specify how long the async search needs to be
 available through the `keep_alive` parameter, which defaults to `5d` (five days).
@@ -109,9 +109,10 @@ become available, which happens whenever shard results are reduced. A partial
 reduction is performed every time the coordinating node has received a certain
 number of new shard responses (`5` by default).
 * `request_cache` defaults to `true`
-* `pre_filter_shard_size` defaults to `1`: this is to enforce the execution of
-a pre-filter roundtrip to retrieve statistics from each shard so that the ones
-that surely don't hold any document matching the query get skipped.
+* `pre_filter_shard_size` defaults to `1` and cannot be changed: this is to
+enforce the execution of a pre-filter roundtrip to retrieve statistics from
+each shard so that the ones that surely don't hold any document matching the
+query get skipped.
 * `ccs_minimize_roundtrips` defaults to `false`, which is also the only
 supported value
 
@@ -192,11 +193,12 @@ first.
 <6> Partial aggregations results, coming from the shards that have already
 completed the execution of the query.
 
-The `wait_for_completion` parameter, which defaults to `1`, can also be provided
-when calling the Get Async Search API, in order to wait for the search to be
-completed up until the provided timeout. Final results will be returned if
-available before the timeout expires, otherwise the currently available results
-will be returned once the timeout expires.
+The `wait_for_completion_timeout` parameter can also be provided when calling
+the Get Async Search API, in order to wait for the search to be completed up
+until the provided timeout. Final results will be returned if available before
+the timeout expires, otherwise the currently available results will be
+returned once the timeout expires. By default no timeout is set meaning that
+the currently available results will be returned without any additional wait.
 
 The `keep_alive` parameter specifies how long the async search should be
 available in the cluster. When not specified, the `keep_alive` set with the
diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc
index f24d963f41ac3..d3751af655df4 100644
--- a/docs/reference/search/multi-search.asciidoc
+++ b/docs/reference/search/multi-search.asciidoc
@@ -23,7 +23,7 @@ GET twitter/_msearch
 ==== {api-description-title}
 
 The multi search API executes several searches from a single API request.
-The format of the request is similar to the bulk API format and makes use 
+The format of the request is similar to the bulk API format and makes use
 of the newline delimited JSON (NDJSON) format.
 
 The structure is as follows:
@@ -85,7 +85,7 @@ Maximum number of concurrent searches the multi search API can execute.
 --
 (Optional, integer)
 Maximum number of concurrent shard requests that each sub-search request
-executes per node.  Defaults to `5`. 
+executes per node.  Defaults to `5`.
 
 You can use this parameter to prevent a request from overloading a cluster. For
 example, a default request hits all indices in a cluster. This could cause shard
@@ -103,8 +103,13 @@ Defines a threshold that enforces a pre-filter roundtrip to prefilter search
 shards based on query rewriting if the number of shards the search request
 expands to exceeds the threshold. This filter roundtrip can limit the number of
 shards significantly if for instance a shard can not match any documents based
-on it's rewrite method i.e., if date filters are mandatory to match but the
-shard bounds and the query are disjoint. Defaults to `128`.
+on its rewrite method i.e., if date filters are mandatory to match but the
+shard bounds and the query are disjoint.
+When unspecified, the pre-filter phase is executed if any of these
+conditions is met:
+  - The request targets more than `128` shards.
+  - The request targets one or more read-only index.
+  - The primary sort of the query targets an indexed field.
 
 `rest_total_hits_as_int`::
 (Optional, boolean)
@@ -121,7 +126,7 @@ to a specific shard.
 --
 (Optional, string)
 Indicates whether global term and document frequencies should be used when
-scoring returned documents. 
+scoring returned documents.
 
 Options are:
 
@@ -134,7 +139,7 @@ This is usually faster but less accurate.
 Documents are scored using global term and document frequencies across all
 shards. This is usually slower but more accurate.
 --
-  
+
 `typed_keys`::
 (Optional, boolean)
 Specifies whether aggregation and suggester names should be prefixed by their
@@ -196,7 +201,7 @@ to a specific shard.
 --
 (Optional, string)
 Indicates whether global term and document frequencies should be used when
-scoring returned documents. 
+scoring returned documents.
 
 Options are:
 
@@ -234,18 +239,18 @@ Number of hits to return. Defaults to `10`.
 ==== {api-response-body-title}
 
 `responses`::
-  (array) Includes the search response and status code for each search request 
-  matching its order in the original multi search request. If there was a  
-  complete failure for a specific search request, an object with `error` message 
-  and corresponding status code will be returned in place of the actual search 
+  (array) Includes the search response and status code for each search request
+  matching its order in the original multi search request. If there was a
+  complete failure for a specific search request, an object with `error` message
+  and corresponding status code will be returned in place of the actual search
   response.
 
 
 [[search-multi-search-api-example]]
 ==== {api-examples-title}
 
-The header part includes which index / indices to search on, the `search_type`, 
-`preference`, and `routing`. The body includes the typical search body request 
+The header part includes which index / indices to search on, the `search_type`,
+`preference`, and `routing`. The body includes the typical search body request
 (including the `query`, `aggregations`, `from`, `size`, and so on).
 
 [source,js]
@@ -308,7 +313,7 @@ See <<url-access-control>>
 ==== Template support
 
 Much like described in <<search-template>> for the _search resource, _msearch
-also provides support for templates. Submit them like follows for inline 
+also provides support for templates. Submit them like follows for inline
 templates:
 
 [source,console]
@@ -377,6 +382,6 @@ GET _msearch/template
 [[multi-search-partial-responses]]
 ==== Partial responses
 
-To ensure fast responses, the multi search API will respond with partial results 
-if one or more shards fail. See <<shard-failures, Shard failures>> for more 
+To ensure fast responses, the multi search API will respond with partial results
+if one or more shards fail. See <<shard-failures, Shard failures>> for more
 information.
diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc
index 1e8e874df2a9b..255cb0048be3c 100644
--- a/docs/reference/search/search.asciidoc
+++ b/docs/reference/search/search.asciidoc
@@ -25,7 +25,7 @@ GET /twitter/_search?q=tag:wow
 [[search-search-api-desc]]
 ==== {api-description-title}
 
-Allows you to execute a search query and get back search hits that match the 
+Allows you to execute a search query and get back search hits that match the
 query. The query can either be provided using a simple
 <<search-uri-request,query string as a parameter>>, or using a
 <<search-request-body,request body>>.
@@ -33,8 +33,8 @@ query. The query can either be provided using a simple
 [[search-partial-responses]]
 ===== Partial responses
 
-To ensure fast responses, the search API will respond with partial results if 
-one or more shards fail. See <<shard-failures, Shard failures>> for more 
+To ensure fast responses, the search API will respond with partial results if
+one or more shards fail. See <<shard-failures, Shard failures>> for more
 information.
 
 [[search-search-api-path-params]]
@@ -51,163 +51,167 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=allow-no-indices]
 Defaults to `true`.
 
 `allow_partial_search_results`::
-  (Optional, boolean) Indicates if an error should be returned if there is a 
+  (Optional, boolean) Indicates if an error should be returned if there is a
   partial search failure or timeout. Defaults to `true`.
 
 `analyzer`::
   (Optional, string) Defines the analyzer to use for the query string.
-  
+
 `analyze_wildcard`::
-  (Optional, boolean) If `true`, wildcard and prefix queries will also be 
+  (Optional, boolean) If `true`, wildcard and prefix queries will also be
   analyzed. Defaults to `false`.
-  
+
 `batched_reduce_size`::
-  (Optional, integer) The number of shard results that should be reduced at once 
-  on the coordinating node. This value should be used as a protection mechanism 
-  to reduce the memory overhead per search request if the potential number of 
+  (Optional, integer) The number of shard results that should be reduced at once
+  on the coordinating node. This value should be used as a protection mechanism
+  to reduce the memory overhead per search request if the potential number of
   shards in the request can be large. Defaults to `512`.
-  
+
 `ccs_minimize_roundtrips`::
-  (Optional, boolean) Indicates whether network round-trips should be minimized 
+  (Optional, boolean) Indicates whether network round-trips should be minimized
   as part of cross-cluster search requests execution. Defaults to `true`.
-  
+
 `default_operator`::
-  (Optional, string) The default operator for query string query (AND or OR). 
+  (Optional, string) The default operator for query string query (AND or OR).
   Defaults to `OR`.
-  
+
 `df`::
-  (Optional, string) Defines the field to use as default where no field prefix 
+  (Optional, string) Defines the field to use as default where no field prefix
   is given in the query string.
-  
+
 `docvalue_fields`::
-  (Optional, string) A comma-separated list of fields to return as the docvalue 
+  (Optional, string) A comma-separated list of fields to return as the docvalue
   representation of a field for each hit.
-  
+
 include::{docdir}/rest-api/common-parms.asciidoc[tag=expand-wildcards]
 +
 Defaults to `open`.
-  
+
 `explain`::
-  (Optional, boolean) If `true`, returns detailed information about score 
+  (Optional, boolean) If `true`, returns detailed information about score
   computation as part of a hit. Defaults to `false`.
-  
+
 `from`::
   (Optional, integer) Defines the starting offset. Defaults to `0`.
 
 `ignore_throttled`::
-  (Optional, boolean) If `true`, concrete, expanded or aliased indices will be 
+  (Optional, boolean) If `true`, concrete, expanded or aliased indices will be
   ignored when throttled. Defaults to `false`.
 
 include::{docdir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable]
-  
+
 `lenient`::
-  (Optional, boolean) If `true`, format-based query failures (such as 
+  (Optional, boolean) If `true`, format-based query failures (such as
   providing text to a numeric field) will be ignored. Defaults to `false`.
-  
+
 `max_concurrent_shard_requests`::
-  (Optional, integer) Defines the number of concurrent shard requests per node 
-  this search executes concurrently. This value should be used to limit the 
-  impact of the search on the cluster in order to limit the number of concurrent 
+  (Optional, integer) Defines the number of concurrent shard requests per node
+  this search executes concurrently. This value should be used to limit the
+  impact of the search on the cluster in order to limit the number of concurrent
   shard requests. Defaults to `5`.
-  
+
 `pre_filter_shard_size`::
-  (Optional, integer) Defines a threshold that enforces a pre-filter roundtrip 
-  to prefilter search shards based on query rewriting if the number of shards 
-  the search request expands to exceeds the threshold. This filter roundtrip can 
-  limit the number of shards significantly if for instance a shard can not match 
-  any documents based on it's rewrite method ie. if date filters are mandatory 
-  to match but the shard bounds and the query are disjoint. Defaults to `128`.
+  (Optional, integer) Defines a threshold that enforces a pre-filter roundtrip
+  to prefilter search shards based on query rewriting if the number of shards
+  the search request expands to exceeds the threshold. This filter roundtrip can
+  limit the number of shards significantly if for instance a shard can not match
+  any documents based on its rewrite method ie. if date filters are mandatory
+  to match but the shard bounds and the query are disjoint.
+  When unspecified, the pre-filter phase is executed if any of these conditions is met:
+    - The request targets more than `128` shards.
+    - The request targets one or more read-only index.
+    - The primary sort of the query targets an indexed field.
 
 `preference`::
-  (Optional, string) Specifies the node or shard the operation should be 
+  (Optional, string) Specifies the node or shard the operation should be
   performed on. Random by default.
-  
+
 `q`::
   (Optional, string) Query in the Lucene query string syntax.
 
 `request_cache`::
-  (Optional, boolean) If `true`, request cache will be used for this request. 
+  (Optional, boolean) If `true`, request cache will be used for this request.
   Defaults to index level settings.
-  
+
 `rest_total_hits_as_int`::
-  (Optional, boolean) Indicates whether hits.total should be rendered as an 
+  (Optional, boolean) Indicates whether hits.total should be rendered as an
   integer or an object in the rest search response. Defaults to `false`.
 
 `routing`::
-  (Optional, <<time-units, time units>>) Specifies how long a consistent view of 
+  (Optional, <<time-units, time units>>) Specifies how long a consistent view of
   the index should be maintained for scrolled search.
-  
+
 `search_type`::
-  (Optional, string) Defines the type of the search operation. Available 
+  (Optional, string) Defines the type of the search operation. Available
   options:
   * `query_then_fetch`
   * `dfs_query_then_fetch`
 
 `seq_no_primary_term`::
-  (Optional, boolean) If `true`, returns sequence number and primary term of the 
+  (Optional, boolean) If `true`, returns sequence number and primary term of the
   last modification of each hit.
 
 `size`::
   (Optional, integer) Defines the number of hits to return. Defaults to `10`.
-  
+
 `sort`::
   (Optional, string) A comma-separated list of <field>:<direction> pairs.
-  
+
 `_source`::
-  (Optional, string) True or false to return the `_source` field or not, or a 
+  (Optional, string) True or false to return the `_source` field or not, or a
   list of fields to return.
-  
+
 `_source_excludes`::
-  (Optional, string) A list of fields to exclude from the returned `_source` 
+  (Optional, string) A list of fields to exclude from the returned `_source`
   field.
-  
+
 `_source_includes`::
-  (Optional, string) A list of fields to extract and return from the `_source` 
+  (Optional, string) A list of fields to extract and return from the `_source`
   field.
-  
+
 `stats`::
-  (Optional, string) Specific `tag` of the request for logging and statistical 
+  (Optional, string) Specific `tag` of the request for logging and statistical
   purposes.
 
 `stored_fields`::
-  (Optional, string) A comma-separated list of stored fields to return as part 
+  (Optional, string) A comma-separated list of stored fields to return as part
   of a hit.
-  
+
 `suggest_field`::
   (Optional, string) Specifies which field to use for suggestions.
-  
+
 `suggest_mode`::
-  (Optional, string) Specifies suggest mode. Defaults to `missing`. Available 
+  (Optional, string) Specifies suggest mode. Defaults to `missing`. Available
   options:
   * `always`
   * `missing`
   * `popular`
-  
+
 `suggest_size`::
   (Optional, integer) Defines how many suggestions to return in response.
-  
+
 `suggest_text`::
-  (Optional, string) The source text for which the suggestions should be 
+  (Optional, string) The source text for which the suggestions should be
   returned.
-  
+
 `terminate_after`::
-  (Optional, integer) The maximum number of documents to collect for each shard, 
+  (Optional, integer) The maximum number of documents to collect for each shard,
   upon reaching which the query execution will terminate early.
-  
+
 include::{docdir}/rest-api/common-parms.asciidoc[tag=timeout]
 
 `track_scores`::
-  (Optional, boolean) If `true`, then calculates and returns scores even if they 
+  (Optional, boolean) If `true`, then calculates and returns scores even if they
   are not used for sorting.
-  
+
 `track_total_hits`::
-  (Optional, boolean) Indicates if the number of documents that match the query 
+  (Optional, boolean) Indicates if the number of documents that match the query
   should be tracked.
-  
+
 `typed_keys`::
-  (Optional, boolean) Specifies whether aggregation and suggester names should 
+  (Optional, boolean) Specifies whether aggregation and suggester names should
   be prefixed by their respective types in the response.
-  
+
 `version`::
   (Optional, boolean) If `true`, returns document version as part of a hit.
 
@@ -216,7 +220,7 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=timeout]
 ==== {api-request-body-title}
 
 `query`::
-  (Optional, <<query-dsl,query object>>) Defines the search definition using the 
+  (Optional, <<query-dsl,query object>>) Defines the search definition using the
   <<query-dsl,Query DSL>>.
 
 
diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc
index 1175fba036bb4..57bb61ebac31d 100644
--- a/docs/reference/transform/apis/get-transform-stats.asciidoc
+++ b/docs/reference/transform/apis/get-transform-stats.asciidoc
@@ -77,7 +77,8 @@ informational; you cannot update their values.
 `checkpointing`::
 (object) Contains statistics about <<transform-checkpoints,checkpoints>>.
 `checkpointing`.`changes_last_detected_at`:::
-(date) The timestamp when changes were last detected in the source indices.
+(date)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=checkpointing-changes-last-detected-at]
 `checkpointing`.`last`:::
 (object) Contains statistics about the last completed checkpoint. 
 `checkpointing`.`last`.`checkpoint`::::
@@ -127,56 +128,53 @@ started.
 `node`.`transport_address`:::
 (string) The host and port where transport HTTP connections are accepted. For
 example, `127.0.0.1:9300`.
+`reason`::
+(string)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=state-transform-reason]
 `state`::
-(string) The status of the {transform}, which can be one of the following values:
-+
---
-* `aborting`: The {transform} is aborting.
-* `failed`: The {transform} failed. For more information about the failure,
-check the reason field.
-* `indexing`: The {transform} is actively processing data and creating new
-documents.
-* `started`: The {transform} is running but not actively indexing data.
-* `stopped`: The {transform} is stopped.
-* `stopping`: The {transform} is stopping.
---
-
+(string)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=state-transform]
 `stats`::
 (object) An object that provides statistical information about the {transform}.
 `stats`.`documents_indexed`:::
 (long) The number of documents that have been indexed into the destination index
 for the {transform}.
 `stats`.`documents_processed`:::
-(long) The number of documents that have been processed from the source index of
-the {transform}.
+(long)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=docs-processed]
 `stats`.`exponential_avg_checkpoint_duration_ms`:::
-(double) Exponential moving average of the duration of the checkpoint, in milliseconds.
+(double)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=exponential-avg-checkpoint-duration-ms]
 `stats`.`exponential_avg_documents_indexed`:::
-(double) Exponential moving average of the number of new documents that have been
-indexed.
+(double)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-indexed]
 `stats`.`exponential_avg_documents_processed`:::
-(double) Exponential moving average of the number of documents that have been
-processed.
+(double)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-processed]
 `stats`.`index_failures`:::
-(long) The number of indexing failures.
+(long)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=index-failures]
 `stats`.`index_time_in_ms`:::
-(long) The amount of time spent indexing, in milliseconds.
+(long)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=index-time-ms]
 `stats`.`index_total`:::
-(long) The number of indices created.
+(long)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=index-total]
 `stats`.`pages_processed`:::
-(long) The number of search or bulk index operations processed. Documents are
-processed in batches instead of individually.
+(long)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=pages-processed]
 `stats`.`search_failures`:::
-(long) The number of search failures. 
+(long)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=search-failures]
 `stats`.`search_time_in_ms`:::
-(long) The amount of time spent searching, in milliseconds.
+(long)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=search-time-ms]
 `stats`.`search_total`:::
-(long) The number of search operations on the source index for the {transform}. 
+(long)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=search-total]
 `stats`.`trigger_count`:::
-(long) The number of times the {transform} has been triggered by the scheduler.
-For example, the scheduler triggers the {transform} indexer to check for updates
-or ingest new data at an interval specified in the
-<<put-transform-request-body,`frequency` property>>.
+(long)
+include::{docdir}/rest-api/common-parms.asciidoc[tag=trigger-count]
   
 [[get-transform-stats-response-codes]]
 ==== {api-response-codes-title}
diff --git a/docs/reference/transform/apis/preview-transform.asciidoc b/docs/reference/transform/apis/preview-transform.asciidoc
index 3ba1b945376cc..77effafa4aeea 100644
--- a/docs/reference/transform/apis/preview-transform.asciidoc
+++ b/docs/reference/transform/apis/preview-transform.asciidoc
@@ -30,7 +30,17 @@ on the source index for the {transform}. For more information, see
 This API generates a preview of the results that you will get when you run the
 <<put-transform,create {transforms} API>> with the same
 configuration. It returns a maximum of 100 results. The calculations are based
-on all the current data in the source index. 
+on all the current data in the source index.
+
+It also generates a list of mappings and settings for the destination index.
+If the destination index does not exist when you start a {transform}, these are
+the mappings and settings that are used. These values are determined based on
+the field types of the source index and the {transform} aggregations.
+
+TIP: There are some <<transform-aggresponse-limitations,limitations>> that
+might result in poor mappings. As a work-around, create the destination index
+or an index template with your preferred mappings before you start the
+{transform}.
 
 [[preview-transform-request-body]]
 ==== {api-request-body-title}
@@ -106,7 +116,16 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=sync-time-field]
 `preview`::
   (array) An array of documents. In particular, they are the JSON
   representation of the documents that would be created in the destination index
-  by the {transform}. 
+  by the {transform}.
+  
+`generated_dest_index`::
+  (object) Contains details about the destination index.
+    `mappings`:::
+    (object) The <<mapping,mappings>> for each document in the destination index. 
+    `settings`:::
+    (object) The <<index-modules-settings,index settings>> for the destination
+    index.
+    `aliases`::: The aliases for the destination index.
 
 ==== {api-examples-title}
 
@@ -156,15 +175,34 @@ The data that is returned for this example is as follows:
     }
     ...
   ],
-  "mappings": {
-    "properties": {
-      "max_price": {
-        "type": "double"
+  "generated_dest_index" : {
+    "mappings" : {
+      "_meta" : {
+        "_transform" : {
+          "transform" : "transform-preview",
+          "version" : {
+            "created" : "7.7.0"
+          },
+          "creation_date_in_millis" : 1584738236757
+        },
+        "created_by" : "transform"
       },
-      "customer_id": {
-        "type": "keyword"
+      "properties" : {
+        "max_price" : {
+          "type" : "half_float"
+        },
+        "customer_id" : {
+          "type" : "keyword"
+        }
       }
-    }
+    },
+    "settings" : {
+      "index" : {
+        "number_of_shards" : "1",
+        "auto_expand_replicas" : "0-1"
+      }
+    },
+    "aliases" : { }
   }
 }
 ----
diff --git a/docs/reference/transform/limitations.asciidoc b/docs/reference/transform/limitations.asciidoc
index 5a66dd8190768..88430cf37db94 100644
--- a/docs/reference/transform/limitations.asciidoc
+++ b/docs/reference/transform/limitations.asciidoc
@@ -206,3 +206,10 @@ this entity will not be updated.
 If using a `sync.time.field` that represents the data ingest time and using a 
 zero second or very small `sync.time.delay`, then it is more likely that this 
 issue will occur.
+
+[[transform-date-nanos]]
+==== Support for date nanoseconds data type
+
+If your data uses the <<date_nanos,date nanosecond data type>>, aggregations
+are nonetheless on millisecond resolution. This limitation also affects the
+aggregations in your {transforms}.
\ No newline at end of file
diff --git a/docs/reference/transform/overview.asciidoc b/docs/reference/transform/overview.asciidoc
index 18a1db0f106ff..409928ff438c1 100644
--- a/docs/reference/transform/overview.asciidoc
+++ b/docs/reference/transform/overview.asciidoc
@@ -28,20 +28,18 @@ The second step is deciding how you want to aggregate the grouped data. When
 using aggregations, you practically ask questions about the index. There are 
 different types of aggregations, each with its own purpose and output. To learn 
 more about the supported aggregations and group-by fields, see 
-{ref}/transform-resource.html[{transform-cap} resources].
+<<put-transform>>.
 
 As an optional step, you can also add a query to further limit the scope of the
 aggregation.
 
 The {transform} performs a composite aggregation that paginates through all the 
 data defined by the source index query. The output of the aggregation is stored 
-in a destination index. Each time the {transform} queries the source index, it 
+in a _destination index_. Each time the {transform} queries the source index, it 
 creates a _checkpoint_. You can decide whether you want the {transform} to run 
-once (batch {transform}) or continuously ({ctransform}). A batch {transform} is a 
-single operation that has a single checkpoint. {ctransforms-cap} continually 
-increment and process checkpoints as new source data is ingested.
-
-.Example
+once or continuously. A _batch {transform}_ is a single operation that has a
+single checkpoint. _{ctransforms-cap}_ continually increment and process
+checkpoints as new source data is ingested.
 
 Imagine that you run a webshop that sells clothes. Every order creates a 
 document that contains a unique order ID, the name and the category of the 
@@ -72,7 +70,7 @@ indices then index the results into the destination index. Therefore, a
 aggregation that it performs and the indexing process.
 
 For better performance, make sure that your search aggregations and queries are 
-optimized, so they don't process unnecessary data.
+optimized and that your {transform} is processing only necessary data.
 
 NOTE: When you use <<search-aggregations-bucket-datehistogram-aggregation>>, the 
 queries are not considered optimal as they run through a significant amount of 
diff --git a/docs/reference/transform/setup.asciidoc b/docs/reference/transform/setup.asciidoc
index 862b86eeb74c8..dae8297e7d18e 100644
--- a/docs/reference/transform/setup.asciidoc
+++ b/docs/reference/transform/setup.asciidoc
@@ -31,10 +31,18 @@ For more information, see <<transform-settings>> and <<modules-node>>.
 
 The {es} {security-features} provide <<built-in-roles,built-in roles>>
 and <<security-privileges,privileges>> that make it easier to control
-which users can manage or view {transforms}. 
+which users can manage or view {transforms}.
 
-For example, you need `manage_transform` cluster privileges to preview and
-create {transforms}. Members of the built-in `transform_admin` role have these 
-privileges. You also need `read` and `view_index_metadata` index privileges on
-the source index and `read`, `create_index`, and `index` privileges on the
-destination index.
+To _view_ the configuration and status of {transforms}, you must have:
+
+* `transform_user` built-in role or `monitor_transform` 
+cluster privileges
+
+To _manage_ {transforms}, you must have:
+
+* `transform_admin` built-in role or `manage_transform` 
+cluster privileges
+* `read` and `view_index_metadata` index privileges on source indices
+* `read`, `create_index`, and `index` index privileges on destination indices
+  
+For more information, see <<security-privileges>> and <<built-in-roles>>.
\ No newline at end of file
diff --git a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
index 249101cfc54a3..069945f1b7474 100644
--- a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
+++ b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
@@ -22,7 +22,6 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-
 import org.apache.http.HttpHost;
 import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.BytesRef;
@@ -46,6 +45,7 @@
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 import org.junit.After;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -102,6 +102,13 @@ protected ClientYamlTestClient initClientYamlTestClient(
         return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion, this::getClientBuilderWithSniffedHosts);
     }
 
+    @Before
+    public void waitForRequirements() throws Exception {
+        if (isCcrTest()) {
+            ESRestTestCase.waitForActiveLicense(adminClient());
+        }
+    }
+
     @After
     public void cleanup() throws Exception {
         if (isMachineLearningTest() || isTransformTest()) {
@@ -163,6 +170,11 @@ protected boolean isTransformTest() {
         return testName != null && (testName.contains("/transform/") || testName.contains("\\transform\\"));
     }
 
+    protected boolean isCcrTest() {
+        String testName = getTestName();
+        return testName != null && testName.contains("/ccr/");
+    }
+
     /**
      * Compares the results of running two analyzers against many random
      * strings. The goal is to figure out if two anlayzers are "the same" by
diff --git a/gradle/ide.gradle b/gradle/ide.gradle
index b3d866d8342b8..c73d6e5a1bfdd 100644
--- a/gradle/ide.gradle
+++ b/gradle/ide.gradle
@@ -35,27 +35,6 @@ tasks.register('configureIdeaGradleJvm') {
   }
 }
 
-tasks.register('configureIdeaRunConfigs') {
-  group = 'ide'
-  description = 'Configures default run configuration settings'
-
-  doLast {
-    modifyXml('.idea/workspace.xml') { xml ->
-      def runManager = xml.component.find { it.'@name' == 'RunManager' }
-      if (runManager == null) {
-        throw new GradleException("IntelliJ 'RunManager' configuration is missing from workspace.xml. You may need to refresh your Gradle project.")
-      }
-
-      def debugConfig = runManager.configuration.find { it.'@name' == 'Debug Elasticsearch' }
-      // Enable "auto restart" on remote debug run configuration
-      if (debugConfig.option.any { it.'@name' == 'AUTO_RESTART' && it.'@value' == 'true'} == false) {
-        def restart = new NodeBuilder().option(name: 'AUTO_RESTART', value: 'true')
-        debugConfig.append(restart)
-      }
-    }
-  }
-}
-
 idea {
   project {
     vcs = 'Git'
@@ -67,7 +46,7 @@ idea {
         testRunner = 'choose_per_test'
       }
       taskTriggers {
-        afterSync tasks.named('configureIdeaGradleJvm'), tasks.named('configureIdeaRunConfigs')
+        afterSync tasks.named('configureIdeaGradleJvm')
       }
       codeStyle {
         java {
@@ -108,7 +87,7 @@ idea {
                not use this file except in compliance with the License.
                You may obtain a copy of the License at
 
-                       http://www.apache.org/licenses/LICENSE-2.0
+                   http://www.apache.org/licenses/LICENSE-2.0
 
                Unless required by applicable law or agreed to in writing,
                software distributed under the License is distributed on an
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index f3d88b1c2faf2fc91d853cd5d4242b5547257070..490fda8577df6c95960ba7077c43220e5bb2c0d9 100644
GIT binary patch
delta 6577
zcmYkAbyQT*w}4>)kxq%Bq>+@Al<w~C97Gyq02vSlq+@6pI;Fdj?(RlHhY^qlL4?Qi
z_ugCYo<HvQ?Y+-AYwf%4KYMr#ZD<Uw_PYf73T_e;3@IT7G1nxKCJo;PwV1bW%5Vj}
zEC^IJWjDpnBpjqrj$x7Kd98lxY7|~y`**~(;(^hvbw;jWc0iTqUVDhYLDf_RPe6To
zVikTPa<ZDSDG7nx1GM@W(LDC;jWo2_Moy(lI7Fj7%>)yql!t+#+E>53IV^;nKUp^h
z4s3gkgN%3})P~|EIG7tA>p3fA-P09~3?!BA;4bImM)6XMVtxPCsNO*R8`BM+7JTT(
z%DMK_X0u;^`W#m#Ec6g#cs0%#ER_VbZbDE;Xfo6SxH#Jk{G(@Ad9*Ni==)yN&+Rs+
z!c5TRmq9CHM7*<Xc}ndtXubKB^~y-jiF^9bGgb}ACP>0Q{Uj9E>5GhmX#~DLb;+ll
z-!FDVFymGnKRbAxQ0Rzpxzf2^IIJZ1>a*fh3^K^l2iUjT$-gD*2u?zG!9_ig1Ulvk
zVy#gFy&uq-r`L2o`taG$t$-ROOh@WB(V7|PSzLEhBel)=tr_h5q~-=lfBiIaG-@wk
zBq3>qaP`ZEdoQnNbun7EP_R74YiH^8;&y3c`JXY2C}9eWD~SoPu(5u~BT-ou705&#
z(j53;{6KX%ts<zw%Y9OdWmeioS=om-_}UH@`W!42{Msjuur+eznQR}oy*6;T>|QD8
zmei!%J?bD0pGB6rrzF3Ql4*rgVKrN33Y||4vWuVRKs>deCPbA_CvjUl;RXEOrT4(m
zxINRPIa9#uO~1D1Q#bsM9eukHf}6O{pGw;+ppWNgFcO`3yrOJ5y(f`P;lLa*;FbRM
zB@6#w0+(7p)M&JU*^0=M55Aoo4{;;*yUD~nK0+Oa6Wk=2f3o#?BO2E}-q{g_3H_wg
z0;-~+W22xve~yBJB8{@|3ve$aMM2@_LP2>6s|At4rllw#)_$CkVXs~Am0ogKD*|j_
zgiI6wW=_0?pQ`6cF%!hwoxE7)ja4t2s;W$!XAP>%4?b0uF*&iXt(lmnIlq5b)v-z5
z@o_CEs960G(Va2M1b+Ve&u{3Tt&W=wujzA1L{0!A;<4?7f{1J9D<+5sp{o0Gl5$Qh
zvBaG^vwV&eGKy$?nc}Imhos%j6{jeAIh|0KF*kvI?($YQ(>(!ky77|cTSHMssfR~G
z$!TD|WuAv}uxh9`c^b%!wg_oPRMgR?<4-nbn$pQN=jV~oM~!_>Yym71wP48|FE*y1
z96R%lnZ`e5kFBux^GVnme^+#ojZ%|>Xp;`YTt;t&7%2RdyYrDTqAOysp!;^Q-zL2m
z{<3O67MM#{q;G@|kDYT#DpCIJl3H#GxYt0ge(`7+S_gDW^oSMNIwm;Zn$I<&Bf(q6
zXRfi^Ts7qA$iN`Y1fg>%(2}%hvhO1!6{>4Wyb#F1d4sm-*u{B+XkX)35({w=e9p@w
z!Pg7I))TN#nc`rdU`tKl&M>kWI4ayM{EB@QRb%u*hp0?(Z|kK`q<%-Mn|Rk$Kry&x
z=mbY6CaVbil`u$ZZ(N{TTq$+NqK_^ai;mb{lDg>40G|0=XRo2tJyC3p-5k}f^7?0m
z!}f`0iJ$zgCO+DX83Hi1e4nescg=5HJKW77vKP%&cungqf-bJ@?y8f`cxo82<Wy-u
zeQm}q<3$(KL_dUc36>Am4tdK5irHk!Zy(hjoC+G|8`B*GSSqK!XpB3>XX;C&&ThUp
z(T{Z|<VQP>%<&VjZseczWppu0qfOIq$Lpwg#xP`3*axm&594YRNEg^VdLLbql&Crh
zxk@ZEo?micfn~+C=G#?x?rA~#u&fZ4B$0|oO=>5vz&Kr7CNNmEd3)%nX`0iU3>HC!
zT?bwEC1;a$T-+#3;`a*P5!UkiVw=dO4u;bWwdE8VOW8ZCEPG&c8+TG;hC!Qi?L4?I
zpC)lC*?uKaF3_iZ?^3Bi#f72TX`BY)$Sz@TFjGb|Zko819O%|kphiM-?J-}y*4>24
z1Z`uQG#^U(&XK9hTXJ7k*3IpxwO28-Dcqg~T2-zRcbnj>tQ;LXWH2x&vxfUL{jOGO
z3G7epiCpEHPXb!vwOG}1y?}zf&~r@rl2pr0FJBLQe`Zx7xHwB+JF#v)zK?|P1<cZg
zxQQ)%0`sDeaz;qcxu_Zwb<{YM%wRcHM|)=~Q+0~d07CnYzTlnxN{&rYV`Q0KTEje{
zYVH-lKcc}(FNsb3VxY_r=h__GERhxC3gwTjnf+Y|{Aj!v&#T_G(8xgZv5jmaQwhZw
z5kKc&IalqcyytD<>iX%qe47=-$dP5eQmJLn)-7P*Q!|X_fg;{OP$8M}6aFDyBn9pp
zAG@AQAIDED;?BF7i8eLnRcFHyi)s-y#2l}t%q{o~>R{|~BTF`M^WV@5Cp9RwF;YB6
z<;I-(^`&Co1awRat-Ba9hLnXWmjQi;b*q2AmBvwGJ*HLuGRtUGBr-<{d2^Hu9VCZ`
zEmOQhVN;&3KEb$l;r&K7A0?lp9EmdU&B;|uK(khuYyBj6%w^jdc&x#vzIGg$3?Hm8
z@&DKtMcG{Syi=P=@)YSR&oIsVgN%b7)F$*IQZ&0Za*om#%Wi<02tTVqyF>I4B3MWt
z$6TfNCMHLfuNPIvoPmrVvin(*Mh=UE#s_GL15-#6WAt#bomte?X~%J9PErp?aWm_n
z6lC5s;l4)APgN^F#?aa2m|4Q`;UwvKYujR)bBgi{_!r2nF?gepca~A@k$Q-lOW9J@
zT}hH0!rO#xTxp@eRMm^NN=@IJWL+;(YROkv8}+<C$}@d&SeKU{lC>tG!s*uW>Q8j@
z8yI`^Q1vgVB+2|UR@B92xet~aB{n8TyP3Tk_Fj3<8o;FK;@Z5{Gg>9^7N=Q;5{>05
z?gpL*2unrhmi!!Ns>5h4>9`#B4c;3@=pp;6=&OFGw$~<JcQKxaWw<pjSP?#RF0h>@
z9Y6gX{2KFq*mUYB<U|}gTABsA@K3|9saG*D{<x+zf>(M5GKeOJH@BzLxEN4wMMkP&
zbZd=x`^V5OBR^aQz-jX^ef%>lW|0AxwHk&qir#mGAB{?bfHO#7H$G0T!6G}XdKt;y
zZc@qt${l)haQ|wn=A!ggAy$%+4%53k(rxLsA&}pBq(uty$Hw|v1n#zDnlDow{`uwy
zo?r@Fpm%qyWPIK<%_NqMdvJB27(^PubDrk?z-L){A^m{u86QAdaAxT90ECz$WCJ6n
zw!gWlc$H2?+$z9N3dl3KMKwpMrnp}8;Y7i3`i`;qDdSj=Ub7ple;(*p=p?WsYhDg3
zYJl$CU0Oh>nn`x>?apggqu-0Hky~UJADVt<w>4^=tRgQoMReTK!sFe)PN4;2&SS8W
zGIaS8t1|V~wXlXvDc)Mdp3H+2z795??E|9^aaGeDd<Z%w0Ej=g=^DAE>pnrjbPKoZ
zuU~yQPN-*{EAb2vp4|}=+_3IxJNAm&8$2TmUQdCrI9x(IVpJ#HD?mg2%|wT(3@<lG
z>N?2Ch8K}NQP5-Veg)fb^46sXoW4y10LgLp>&pXJ6ZL0<68iSn68NFv#Q<!@B&~Jg
z!`|=5e9SIJgj{gU5bh>3fB)8gl>sZdbrt485)IyFEm9l=S*!Je<KoV)orgF~t6#@_
zECDhPzp^)N*QhkGK+STGWxpQ6J-DaoCc2t@-Q8TW>&xWea7c*N9-;LD*Kr#-&UeRz
zad>a;uZ=i4>lcMsZqbIIAu%E&t==)^#MxS(qUoWse#ukF6Z2v}ZSol;W&?|Jr131@
zMtl}@2kRk*DR%yZp#*&iupcJ%T<wJO!Xc0WP3*$bqT93Q#q}w<Q<~B8oi>`0^|^K<
z3I^_?k9s2xUww#5&!)YD!Xecc4M}3rLqF0RvBrK9mpgStQ75;3?p1?R{i5ae?x(@3
z5aql@kOL)4FD`Z|xDw4M6bDPsa74e3@PO{?r)o|sL?4qN&>h;+w+pw+_f&AmIOMCW
z@=p^Y<fY?7*A#?M@geasx$ZQ*ULCQ_boJMGL)@#8L@2%tQS!ZHXM?`((Q<$WvWTR+
z)Pz3{Spu*%v+py=vTbO>>P7fDdt;J3Mv-(w{BI4b$NXWSAyevLFOMWsjUVo7OZLqE
z*?ZdqiHo?-m%L}ZecB>T-1DR@5FI@@O3@KF$SI*Tt9QdyUJLLc^IGYcH7z-=n=C^p
ziVaa<trB^-DD&IS*RlQ7yE<fR39i!1R<{UdtJpflP$l$UJxt-)^34~Tb1tpS4!}UZ
znPY7PE;pq1VfY#b6JfsT_=4Yu)v922Df}HkN=<-~^6X5=sdeT1UEU=VWLeB2uhRUq
zsiRrXHAZLiIGn4AT$Fm`cC8?i$^ZTs`S~<8dCW)WVT7~1T5@<j60g5S3d)REO>w>_
zz6kp8%4Iy$Moa{Inys8lHMdLni*TK<>prSjVxnv`)1mFAkVe%5eiLIEY@WiQW7uRx
z|K4S?+sOIa%WP2e>H_`-Lb-}_=>Kh$mu&oQmFwso2^JN-mA9J={gMk+Di>`!(|3!)
z#Hd2HS|Q*;#&Hk_KQ*)Q$JC<f<iZh6bMmiXte-@q9?aa$$<erohrUq_!KzU?0sAcC
zlfshyq3G2^yL%5$P}5dh)EFlEa2b(u_4q<_ZRsq&9NgzTK$epiLfN-Kc_v5Q1X@UR
zaE=^@Mn;7f8JP@B`YfCtCoI{@xh&AE;Oe>jusbivMi)FM^U3`4J*@J>(5cp4s;WO4
zaZ~J1_IHyYdhi4^y=X)|W4%8+6R#sv1(#$llI=pm)70JHa2&2*qNP*1qKmySp>KK+
zwoK}Im2^ODta_af$&3@pa8qp$cFcsRs8&z8d-^)98trqt2Y6j8mSu-5vS$gh_$Msk
zjY2X6Jway6GlU@yCqLpytlFhFWmsr%+bqVRDxO_}=Q1ujX^9)jwG($`l%b}CID2~z
zHSh=O<6IZOtQ9u`dzNl}&&)F-JW=q+c?G-SGSPAX>!(^s4d!~ZvX>K23UOk*%q41j
zOgi_lA??Qm?ENX!6AVw({2ar%w^yA})k7D!GZwOR@_%>(&GGRq#1ScYGp+T~*v+Id
z)1`{flq6+H#>V0k3=BNN?(I_)op!C8`i5sUSS8om(kV+`d6U_tD>jr<xLqJkVv<F~
zktCPO3|A}=zB+%B^7DnPk?+zcKHd`ka+!ssOnB2x-~S4LNHRy&?Qdq}I`<6&WVeFY
z%dSWFVw@%KSi|l^1&WI~_X%#<$2c&{YI958D#?F{OKKn$O%3tFu&XMfY$QzL(eZS3
zHy9slOLfiDlxCo-D)@T|a>ttEYbUzCvT~*T815Plap2EGI3m6BGFADJWSzH2gNbXK
zAMevc_gV`Hwqv_d6t2nD#8mRtLj}5u1A`p|zy^L7tn)2^#cmn5ttx>AzWu|}4319d
zmTCBd3DG$iJAc12RQBtaqtaDO<(lhp)saUjc}ckOF-?*CILc)CHQ3-c&R_bIx^<BD
zTRbSF6Z%k0C#>RC(Uh>H=?Hc!Jfq*<m;Wo^<~YJ?J}6dmdeNKDYS(uAL1skRTL_MW
zI$aFBx@2oO&XZ}R(W)RJC6YqewE7Xim9<Tzcq#bez+_eVOb*6+@rY+C+9X#q8b6B-
zeipi-G{$e?NOjZI9zr8|3|UTsm2psDJTnk->uf^5pvZ1qUEjUGFLA48xlJ@Id&^o~
zAxnaPkQJ{5`miM|3u`!5Yl>vOG3{InE)J-^?GFBYhs^S3{f%XmmMDbY929%)tXDK^
z4&0msZpvP=Oj^{;CiXzs=(d5-Tj9y&vR~?%ulrK|3M7R8AoRPFd*Jh%S=Iyda9Ke_
zrF5}XI&XAA(WM2qY$-Iw=VH7%AroF4;p~b8;9td1F#2cg%y^x}8|g+T(nMU&Zr#zB
z-RYWpGePM7mRPYj^xvwV5!U1{Qb-VxZQ=%)g%P$JAS;+A)+%LtlNZ;uSA+=6xC;W1
zZ&!}Qje-aZE$+yMeC&-WJLqg}I+P*%A{y4Qaq5y97gk+F4qy~fVTW7#R8qx7{kLj@
z_Ak&Hi`GnE(YIf+nBX>YuN&8z>0+n8Y4Mw_D`*=uT-^XHMD;CpOPj0`pX1G}5>QX=
zPS1iRQ#%re7!OK%X6W0M^BrF0IHK`4^^7#J+x`8GKi86<OR&I!6!U{!>ZU=OWN9Rd
zbc#BaTYr?doP4Q$Tbac6h=c1Tcuy;l?Gu<2wG$iKh^=kN1p-~6nuHE#vN&}$>STjm
zpd>NS?sZTc`Yti+^Jx(&e|e>jw51=3B!N5zF}}Z+dmjmLgD^?|K2t{vCP(Y5cxl45
z^#&!362V;(_~IFmEp7G&NyG+08Lf|URTC2r&e;9YS?LAO`7_Iiod$D!uB<kWKExXv
z6f7AY>3}mMv5NZLM!7V8_tEyUwc&kFa1isI?26Eogw$4lsNRB(#c3Ssm(>CFP`&lt
zuem=>#4!%PU48QZO<irdEiQ!+0SBU;3wvtyqnb!6QJFqN-jm(BQs=2d&=&M!JG&{H
zw*XXke~@sl#gK?{wxSm<J~qsxPQOZjz(?Y3cHgUGf=c>*F)iwJsf#~c=|+1W5feb`
z44pz7si?Qj-K8bF6s<lhN}8`|lGB6>L7&%FICc1M1vBmTxRa~P2hdeYJpZ#955J&b
zqeVyms=gR(%w^R?^1A&w#Ap@G%}hbE=bp6}sf~VMdpZjHb}bxykA59XXKm?+-Sd~%
z;Xw}ENaem6xp{yUqkQ@z^x;+Il6-@d59N}XiYXGL6;QWzd#QUz8R&)Ql$)Ph=q4%t
z2Unt^=Ru1Mji9_%K^h15uS`f6VVOTS&b2=_dU&nt%RSrsMUY+vWcC91ej!2YKzLFi
z7o|5#RqpAxW)fo!>%GSC=QWq}-chx2_7Cw$HaRJ14sv$m<azz4obe=j|AO|MdThuy
zg!qQ?^MhCVjd9Ok5&M{O{D?-NsCrSF-bB<0s*HaQ3-SKw2?tU?zK<~J#2~fewdrR?
z1k3cA@rxCFMDi;sh{1#jKn_2_uz4wK1R>%L#iajDtdxcqEnql!qgs1EZuI-bz*5EO
zAWxzL1X}g$g^3JgM8S%;%wjN|95AK3o{Z`BBlLV(B_zdIva)EKP4Y8<WaPpvvuxH4
zyp&~?#QK%IST(G)wpDKKDs+K)P<lZ6e6YU%D3ONcQyLo@#Fv-VVHdMZDY+sc9)01k
z)HSScKGYj)cf^^JS4e6UwO*SG+BuhkeT#XZ6Iq@UD$pBQos)3g)0?k3TF^B|spXcB
za$c9l-81DTEgJ1|h1<ufqxP$xyOHNz4@+?D0T0DrD)o{_>FOYwp<B$D8%@!OhqNu}
zM?fhgbeWJDV$_O9gimt$mMMFDMb@x3?Rp2ym9t!@Ng2R##23GXG5<(J)`8Non0p}T
z7s%%vBWcT}9JuTl`%&VrVAs4k0VVq7^3HfZ4#Uew{PNDR7tcVCC8NNuIAJjjec)ht
zf<Y5?_W1L5LH|%KPHHf0=sRBTVK-+awl*d36C$t>;$Raw@wT4E<{pj3{hDai8KZje
zcEuA-{d?JgLv!WnmKq5MyMEX52loR(6fdEA-RV<{G8H5Igxq1>w}%2S)_ju;wF_ZM
z$7!A^lLCtCZdv033jL{f&eI>9ISF2x$~~6;tnOzYI*(I*?>+6ozHgn+iutW-50rn%
ztIAoG0!guTBfvFW3Thg_WtLf?4+*6q61dY`qXbfO*(>@w!l|u3&BIZu84UE^j!yro
z^oi)PjvWObd1M?(HjP?Hjc1s_HH?DvC)%cciIXHNQnqKY1Mg3}aOh6*=l4mzd4Txc
zLVTFGo>@6$+loh+i-?qdkxJD?$<D54K@k(PmsSrBWuHfw1d3bj{g@KJvEN#Q=Z<@@
z*7lM36=d<Qz;Cc(q`^{x2O)<y91{WVo-3*+vPT;9h;7uI&@s0S2x>#HzVN62jNChy
z4YB@j$_b-hu>?T$VRfJvu%s0s0Ef{(lrq7C9j(X!@J;?lNnl2+?0`t?f7)S9^Q45Z
zG6zDOr=jV;rzj)?wzFyiNCrKXu>VVcSOWr1JYl$A%&@I}YQk6lTl(}a3eog}xp;BF
z2-ewA(_y0P;(%c<n42#p?5`IPR<X_w(D8jzi6<n4F>L?=XaO+#VrrP#hBP1}@E>Nc
z)4|rBGPfW9Y4aX6jC&IZkPLfLMi?Xv6E-?e2or%4;{NZwMIr3ae@SO35VpC=4w(A<
zPw^v(VQ;tC0lm@xG<HiKK%VdqZ?-f6BT4`8-L?YYBI6$}ZOa4Bv!C!4Oc+iE>)9oQ
zxqJfxZgT&HB=QJh)Z2tGvcms=GiKqxq<SIIzxp>jKmdC2Q%Df@d50Z<Quc&AFtxJ(
zY(?9BYTTEn5F?DI;y)W~pF*-#|3aZvv`=TS13uP1DFY0D?=@h#4~9C<0JC|*{jn$H
zhS6>k!pNuo|L1uQJKl2yY)r#$r^WuYHGdz7S_A9cR|BBV!D#1L$+T24p8a>Pgr3$<
MViXjGx&OBR0?kH%b^rhX

delta 6547
zcmYkAbx_pNyT)OqL%K`4yFnJ0Ub<sx6mTU4=~$$Z6230d-Q6J4NOyNiN(cy&!u999
zbMHNWJ<sPkXXc%E&YYR^_6<GhA9_-~BZGrpg?r);LP4Rz0BY>0fbkcY00Ec`v8pw#
zP1%=K=fTZQx1pfej+Ro3pZ{H+B$tvoY7*_j#twUpZpfOnC9Xc>mcgedjEy*!&BAw+
z!Pb8qzSx)i-geP%Y&mo93hXitf4u*5hTDllPo<K_(jyn%or_2CswF><r{DGoRr$WL
zt-u@bx_j@ilDPn$6W#gMiW~JD0=J(m{-7J|wNl96s7QE9T`{1@ZFVwSuDUO65o>sG
z#)a_<NfEl@Ax;n)kguzCoyJE1A`);U<tvRg#qY3FWGg2C%Yw<<$@I=ML`w6V#C{hK
zoF;vbnjgKbJY7FGH~R8sVWq-tJ;BFcrz$IDE~L;&ue;hpV%=JHc3O)TF^-)jG~xmA
z6)|9~+k7uic2{C00Wr89YiIsVnHaVu9q!1w{d-bOHZD!5`3Fmf8J{o@c_rTG@^Afa
zG}TO~hXE)p@qJXU4T8mPF*0N!A@)us?XVOfxIVSUI(e<*T@WucGAgav)&^@{J$WEi
zK;X3vv7Xc%<!;sfz&A=~#PI1y>-^*6(UY8N`S7#Hmosbzg7Pl<;TElEZd0hEZc|TV
zsfGsW_Cs|WF=Fk4&PWdE3~w?1)ajZRB`0|;a45l@mC9V@1@RVN@ykVBK8wj<R&XY+
z?MkkeH%aWX=91NhNrV}~o-3ZK;R1QYRwNS(EJBVr@{TY@dnT$rgdl2hN~o|XFv_;C
ztOe%#m9FH&Z+eS!EdK78q<$OQT*cEn<M=c+wrl!Ai&5}8CKhkwsX?r_;fPwo@eI^x
zyoWBG)Q&aco0p&$MBT21!Pw<3{grPdU_`R+a_nt-PZu@(&4e}6#mTcM*_<uz_m^xN
zBIY+&2Y@2sQZYiCM(qh24hjku777Xq1m*p^;r4t|6qFadC@9ZRfVZ<`fA1+JV(@eU
z#E5qogeFJc9fy`LheeEsicR>$z=wr@aDeA*lqRvbqEYcJ++2G(*rVbDu7M7;lVb@s
zUpiabP+>}OT-jh)W+<}$*eWiZ!a{(GunZh*`?>0O^2Pop%YFQ-&u%m(0r8~z!-&?N
zYn(_=J{6xvr3iEF<Tg}T*WyVaf@?zuC*G5&<GwiS{<+d_pQ`ZK_;*Q;Q8Zbt?eF*X
z(d<y5>hzT?{vM~CW%j8)1I6t@AfImYf>vJhH!Xrw5h_lkT}!v{y-23=jSt)Sxt`>B
z(!Au<2-0p1MQWh`&bz(aR;aC0Ywui+>UmdxbpB&%mezJJ*n&xThv`}u!B~E(N6-K3
z3_8U>zN>1nxd(h1iZ4Rq7~R3ap1mtva6>is57nm3v~T=d4VC6NTP-$W3|T+EOHnOs
z6tTAIq*mP>cz`uFr^&$b^x`)MujcOSgT=Yceij*Y2cU~z8-M<+1mERc*)H-}DR&(h
zw?8L`cL$at6C$(3&N&zm$_4RI;qh@^|D<^Q1j)=%Hg<)&3a~S>T?6fn(Y2$jXta6S
zO*-lYV;1+QIO#)S7L)%6kv;6q8ytk%rpw(R;ZohTbgfkyhu`}w@D}dQrJT<aAAO7V
zW;AJgGPEM5PlK11&t>kg$+${qm4m?HteM^(ho{20(c64>NjM2%I9G12_vO{<(vZQd
zeYr)er=*_dY|4^hg-E$#nyQ03GpQ4-Q>6Mi+kNh?FK_xpfIl`MPV4Yy3cqmDKrpYQ
zesF@i+ZSGz(@?*!1V@TSA=|@^9YkoSsgwI8i46HP#)kQLQx{t)nUusL!hR_fp_d86
zt6zUwGi1>GCU1(kw9Tn*Z*I4U?>Bm*Gn!a26D8kkO%asgWz9h?L?M`Aamwl&@P$p8
z-0z1ko0m^H#GcxW?8A@Qr~$iG<1%aA=Y(bR-G`#gEI$V!O^dX_dwmioj(5~kcZc}q
z!j}a(&4VKAIw7#H5%M(h8rbr}@-_RxC5_YaHM%uX&ADKNdnWvcPF=7P{=yoTljgvk
z6!VD4fE~l^=#+;87bGzasykginl9YLMr2J*O+NeCPMyo2Gra8fsqiQ`7s-BU8kRw}
z=mQ^6!JW;kd*js3IK%X_n$<P8X`r^bdp*olz$6Q+r<Dq%#j`(ZTdH@_oO&t7_H?s~
zIMcCy!CnzTNc__2yxAg4K+ewZK-G%qg&Fmmsqcm5j-*9<g;&KfjV$V?vkh2wRCcG%
znj&I2Gj(Eoka{QQh$~UOmqUGpjcDr3chH7!FxI$A!G>F2?gnyPdmMz;<}hhX8vL8#
zDwb%YeX5HF4~B8Zit^3_wRA8m_7pTF3j1!)mdP4XLSH2=$J-dPiqH6Dh@j@?CD;r`
zR$IQ+WWpb>Xw^^DmRHcmN+#F^#-;d8?l%bv<Nv86O!YO%`0d$Idho*sI-R$n4a7A8
zSOl*x?c{JxB?jaAiIv)%6?;(HDio{4FEJeYC>l|*4MN7OhV)mNH&72YV%wl(zBp+!
zp{cou)D(g0n+xXCANKg!ER|_wPC>bx7-khT3EI#3PL)x9?_em_p`|iUe;3QW2p4Uc
zv$CIRUL;gYhF`->`J<_bMn!l*UX&>W{xC7-XnRWc1|lH6m4ygrIo&mVs`>#<DDUb+
zoWa3I^N+tf8ghj`@Wv|O#$-Z;CNOk;c!(zvxme0%3WCg3p*WVPK=@We4#$|l{tvF?
z7SuPrTeGPs{9hpihlvjOn0G<iQS9Y)=(<~@M(cPUcyyigjgx{<loOn`Clu_t_7T}<
z-u)887eUJLsY-rJ*h%ke>Pb1v<?L9Bx}F?-V(dz^spWcUHkDhAgI#+rlx(0tLL`%c
zm^V){KEW`5O0jdKpE+{a@f9E1uDeNxV~Jir*AH6~cB7cC2lc1#@?^$}8IRd+4w8{`
z---^m0EJpvV>8>{GX-P4kK_KxSuyies;QBq1e->cP5+I;eAg9LbM^wtQ6eSW_zWF8
zI^>q<)j(@pva4?EE_PMo%gu%y`?E7d?e(WTWB>9&u`(yaalT)+pV9kcLPsL0KfV%u
zc`H~JJ^Mh-J-BS0P}*69ouWEE<<9j7`A|5;d{M00Q6yV@At949h5jx_bv?(4%R{?J
z_4E1c!gX?~p~<^gRf=g=E+_Vx$91C{%zJsH*EwHU74kDfi9elX)j7Vu%$osz1mq6S
z+B0uR{A^U4QBOY9fAqYUmBU~EL2x~|c|3g-%f>aR(w}?1@Z7oGd`J3P^A-Ibj>6_w
z{k0xhog3$N<sCntDv_%af7ze9H>kb<d>Wcm+%+P{D8VWVW?dkh{@(R^1TWWEv_V^>
zSaBI*x8WKK6-py7SIMl02$MS^6zB<lI7xi!Iqsr@xBP||3;KqdHr%M=m9@r8>z{1@
z;bPeEOV*SwCmd}1zQ9Bt<1dP>ANcVrX`sqZ#Lctm56lic7SnjvsdF;>)i~)4)}6<8
zw>3kuJ6R?7l<GsHhXwoCv!*FBln(D;)Q&S7^WoSxllQObT#r>qCYM4+5le<L%_j4C
z5WVPn6>LIB{FKq@^Srr;_e9vKqp49!1e$Mo?uyV%V<^c}k0JY$e141jJkVTsm>WF?
zzUm(myxyEf#<`GTnpaS5;b$-*bddR+=ipA45;OVx0Ci>}3ay2L1rZ&dWRo=voeU)U
zukSaL`h57RPMmtbU6(#zA_lo?M$T~-&?rm`EIP1}2tL8<<{_<907tgqeEL3SsAI!k
z2jgOUsW&{QL9N^1M$%VrXYb}SSI09g{%-q=@X+@NcaGE;Sk$ED=7Ox*;0*3Wi3^HW
zfICY#b-$>~7%kFL&inoFFjq%+hvAJu*EQCjZXD-^tNyY(*JC&W!5tIGKI+i+N%gZY
zSI5{_ZHY*1*6KBtgiF3f{Xo5ez5t)u!c$YO$IQp<a|Twu-AgTQC<$p&Kk+68x3Ts-
z(~)kNQ*}X6ExuS^Ol@|4x-92?<IvAj;Q{xW>v|5==g7wqgwAyp*JJEs<+<#2R<c7w
z83eh$bEW@4(Gd?q%+x&S9LPL04KYMgJoYdYjx1NH=-n_wJs!yz;tvkr8?==+C8SV^
zyQ{`L)?9CqAi;%N`VXI~3~vzr(m01b?0$uu$|hIYx|$klXnCsXK1U4eoK>b{s&@eV
z;<G=<o{6>2pLXV}CIoejpWOF`HSeP>^@;wg--*snbwmz`h7Km33$+4sZ4=Hmpex-O
zqJ1uQVCQliL8^Z2hc8r1pwrjeeG2L?3*AUK8hh7QV|M3XApI#FY-5`B0)FYsr+=TV
zW?AHTHxy>#QbyO{Hb$0bq!##z*Ym!$b|RRW%<5ZHstN4rCK^^7pXU)ZD$diO;3SMm
z-`5g7n|)S@A4GiKE1ec08xG$SOOPM=Ca1DfbRDca!_%7>sjyFiOWb;e>%9W&D<v_L
zYj$LsWWJyy;}*`gzLpF+HVW|iMU8HkdA)=9AZ{<vb<Q>$+?cLXYCh4ba##?-1<&69
zaH<~z9paWS)W!bcJ>&>%5zAt1xWSIIq5I>NE=@0mFzu$HKeDf>M`UydKzZyyx3FPV
zeRI)5yX39+UAoH#@F)&0l$T-Q32(<J1Y{-?=Irgxmum=aCJ(n}$K}o#sj51=lHBaA
zEe#e-;AgwuJm~@+QNPxZa?(T0`ScV|S4SL>vjWcJ8eIYr*4HhHYu%Gzp;u^`rY^W9
z9F01NSn<PWgGq-~w!aJJw4b$RA?3~rN~EZA;rJ*=Z<X8Me<PRAcKAXbtEjUV?qsV>
zDq+@Ud?UjbN4hEecEWu;zy1v)2|B(eJ@>Y7Tx@Gh>-?RsXZ|m`h$HcGdoCYKwmdKt
z!(gspq5CDyr$8fzL?5HV6GmaPn2^yS@h89yg7<RmuD(W@V&K@Q6jTQekLM!D=Q{>P
zv>kt>NjC;EWQ^Fk5ru=wy$FaZ-QCgW9%v=u{A~W?Tclu3=TMA6jUg>Q%<ShPGtl>z
z0DZE&sp8FZymao0;o)X{%Kqin7mz{+-}O9v=eaHJm*EyfbIhlxL9)+En^Fen+s9N8
z?9Ax9wJ!8+3B12oy|Xcu{_u^c3VR%TaC=L%`u^w<tVt{)Q$}i%?9*~RV<JG`e=e1k
z+#c4%b1Swp5P+WPzJ1>PqiI^v5FuzD97y?^zu;%?ANsX1Oib}xXjsN4^999+mULA4
zgAz^MtI5vp+<<&i@}JBu)`MW``uU|zgiw9nK(r^5AqHH64wH&)Qevoo`c(_9aG01@
zOK>GiZKeWSW2QnW&mnZ%&H5dtc^FZGo$L)1(otL-f>EU)oZoVaN*x-JV|x<xHN<V`
z`4m3)FB^ZW&Sk}(L!2jTK}F5Q`GA}B>u-6Vyj&P0i{$#{T=~MwSw&I{A?F84i1gv(
z)hRc=+_D2|mF=9Hi-23y=4-gvA3{SnYbVCzd<IC}<iszj7J7XWINoXyJO66rtaEuQ
zw!^86%VE!+%}9I-uA9<HQRfgJwSmzLG_TCueN{eULcU!{t5W(9gIbUH^xZ`AmlLzA
z%hjJ0n9nQCI3zv2u9kc0L@M!Mh`6f5e5;hL)WhLR^j1ul!UOpd>5b9L(c9g?RP7|X
zfs^d06B_u77gR!RA#r8+96}-`o@w!3Ua}0@QXG~eTeTy#G2yvRp$i%!$*HKZgl67s
zu|>QhVci1yp>ajz$vxQsho-|ozQ!k%SwpGlrDD35d#FL5P0j9;aVK~M5V~R&*^=+L
zSCzmzQciQYuf=0RCpt@)51vxm3rMU&y&##ir%NGZ&Zk(@TKmq)9z>pPm|7MW(fbxl
zxZ<mrVAkyej1;3BchVuf{lGAKtFYfhv3J4YTacVH|424&{Rd9`qaEPs4|SVz8X9kf
zQ@Q4DJ#)7pHep)n=k_1&5Y}A9?qu^vMJ0D7rxqimjL4OslC5^V$|w4Xik+#QZn7GF
z#0mugk*{oS+mi5XE(QhSzok(mNKw9}R!|`A7QrP%XJ-^MVw4hR<m)Z2bHbj+aI8P=
zo0|?i9RPjSco#$;imDa7+#+yT7$HyvvrGCCA;_*oqh2Xbs=vQcjF1<>wmY;<QqB%>
zN}{MPKvPp3B+<7pUV#b^t*{b12zyQPbh;WkjXCz}Ru>nJ#lDvm^~g+2m2&Ci#rf=W
zlJ_Ne%V*;Dx(!}T2D|P6(VS$XM*iB2tVXeM6k^E?d+?5QXHqc1K{0n$%%*tB^=D>C
z{Rv@&Y!C1X_)ss(h1eJ5{yqpOSSDRwxO1!itaD>RV1%dmf;F}BSF>z$+!ZNCm9>%3
zB$H}@JlE71f7KotsYWn%*}UuP-u5Lk4KCN2ahPFJs6v=g4a{r>xdoBi>Ku#l+Z>K=
zwezjvKQ#3mdA(SahO=mcpI~JXIP!P>a*IrMJHz{yqYw^43@u);$e^P?Gl5N#L7VQX
zb<;DDo;5P(0!j*-Ol}^`?3^Xd62%kK*S5*8(>qs@nJ8z%hMxE6519pfM|vn27qDE}
zaJ>x&>A|+9=<^>R+%%8!d%3@~L?_MoFch9k8I9>)gNs0!m?%lJ@1~%hFpIc)ymh0K
zd|UJS+{$Q#W+iY{stH?!&L(ymcFmPp%e!D^=o;<%1)qad$Ec-kK<%kdOG^}6NJy$G
z)-+x^HXfcue(T86JkI|61%F15!*t1QUQa~Zk?9V@%;2+9n1|TEn<#9XV56}1AgZXl
zEh`qo?!^}YIboKsV&BnqLav{2(1Y+83WbvGuyYYP<Y(2`AD2Oq8JII>D9q+)<7S|B
zv-f*t`|zOOR4wEft=PL?k(rp6xJk;UDDyB{zVT`P3c`{8>*$4wl)kAd6io(Cm^}aF
z@C!An4E3sss?9XD7k6BLFka4g)>Tcp@K(zv^>w~9bj{;Xq`%KV|84fFZ+^<E>RDD5
z&D||R7u@IaMNW;>*F1*|X9|Zd_bnyKvu5EamB_jG`JPsUj_cXtfG9+Gjipd&=k*=@
zSAhOH1m8eW(ic<IZdH!{ihe%xo&TQkRy*3HWw?(e>WXDUj9~ZfM}7GM$VC!a9aC-m
z$9&}vXeQ@XN!yio)>wnSzdn=;q=i?)3mhg93pVMVBsjb;$m27x6+9D7HHXZ%-ySdS
z%3-ymPnpOtY1D7si5fq6BpxnqYV$BGQ`pqmw2tS?7BLGj=p*uFAyE(xmF>T8^XMzz
zw6z-2<j=o*o|m_POFCb;1!#tl-r~mpAyu#4wp1P@c~o@dOQ_F_PTeUdSX2S)jivA+
z+{Qvr3GF(0Z2HyTvF}8=^_~-l_|nY;YkQYOkIB(ed9>|HajrqxK4b-%h7+T@usb1>
z->hmpIo^MR&k=ug(hd`I0w7tJq^B~q6snow@@qlwFrL0U_=9red9nQV!BLB*n%au_
z7SnFMfboKV`|!#-oxrN~aRU2-@%*wMv2nra9iSwbJ^W%l?!oMq_Pzy9gWK=ig7*ih
zB4=|XT0P7ng?xD0PG3&1^@!%hf88|Yw;)fv9#>!EWu<)Ax(s=2e1TwHbCi+=oj+08
zYBbA9IG4oN*_Z#e$jD{DF%?^1`f9_>PM~~3ITW_pk)`WtDBgMk1&kTF^j1$1=|$tJ
zjtNrAbC8($17KUyjjj)^@<#sc>1}DWs&?n>sE4Im$OpCZ^NIkktFI`#ivyY!GJ81&
z3AJgh3$7e@uki@7pOuM3VcMnN-@w(jd&ay>k_L(%yKLOf<C}(9D4wNZ4zDNGhpL)H
z<clnMMY?@a-e&}R@BYf4ATwq~BpKpFyLR9C;@DO-^uAUL&DM~!WKq=KP+g|N0Sx@}
zwdNSXc4Mj-uAII(=PH4ob`?u#Vh%i>HOtmDSNr6C3u$I%N$SQHW%=$FPV6i$Fz%`f
zvTF|4kS7dRnJ>42(TDsLqaLY5@&Ey0u$q}4o#Y||v|WUqL1NK1mLOKneC`^BVDKV^
z+z6G7-OEnW<=4(hE4U}46Ng}{OS8|)el0=}!}g3YXD{bM1NRr-cDVaKP2}q4tH-0Q
zC<%qSM}j(pfkZIce@5`Y*LfrC|DAIJGz*rXAcKFC&T0cZAY*|G#AE!=%EIu0!v#4I
z0qlP)2{5=q2-q)DgFaaQLoL>H|4@+~A@1Mt>A#i#J{8zlgn^K7U~`cc7=b?pFy{#Y
z&n0TqQy^hU8>H<HIJk2IPnWq+Qvk{3=b!<1z}b@u__(Y9()0W)Wq^;{gaGpjH3-cQ
zcve6Qn65xTss8_Dx+lTEbn$O`8}JV`e5VC?SE)fSLjPK3ATSgY7?>smB*F;s{;wwP
zuzw*uj2c*3<pPaI{<HW|#DL5i7pOb>KQ=Lj=5I&{G_6sCC_nz&@Ow=QG?@5LzFAj7
zy#Q*~;h<mXe*q^To<R<DY!Lzu8`PldtbeF_LkfhI_fM+l5&cWWHmOn30i8`N5Nh$?
zfFDpRCImV+2m#O*HR!bTFJcDx%Km$=&0nNY@ec)8(EmNo1q!VGYncH2?*gEO?!;jx
z!1}M+8A=@H1L*#iN<Q}g8;=bfeYXd#jsw@@(to?eg<1`K-Bt&IrU1Grc0gc<5JP$D
HKP~w$%`S9K

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 92b434997d92a..900fd2c0ce42e 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.2.2-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=838fa0e3373a9c8d953eff972449edc6be5fa12b26509ac3387727de85138cc0
+distributionSha256Sum=0f316a67b971b7b571dac7215dcf2591a30994b3450e0629925ffcfe2c68cc5c
diff --git a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java
deleted file mode 100644
index f917c477493cc..0000000000000
--- a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.kibana;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.IndexScopedSettings;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsFilter;
-import org.elasticsearch.index.reindex.RestDeleteByQueryAction;
-import org.elasticsearch.indices.SystemIndexDescriptor;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SystemIndexPlugin;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction;
-import org.elasticsearch.rest.action.admin.indices.RestGetAliasesAction;
-import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction;
-import org.elasticsearch.rest.action.admin.indices.RestIndexPutAliasAction;
-import org.elasticsearch.rest.action.admin.indices.RestRefreshAction;
-import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction;
-import org.elasticsearch.rest.action.document.RestBulkAction;
-import org.elasticsearch.rest.action.document.RestDeleteAction;
-import org.elasticsearch.rest.action.document.RestGetAction;
-import org.elasticsearch.rest.action.document.RestIndexAction;
-import org.elasticsearch.rest.action.document.RestIndexAction.AutoIdHandler;
-import org.elasticsearch.rest.action.document.RestIndexAction.CreateHandler;
-import org.elasticsearch.rest.action.document.RestMultiGetAction;
-import org.elasticsearch.rest.action.document.RestUpdateAction;
-import org.elasticsearch.rest.action.search.RestClearScrollAction;
-import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.rest.action.search.RestSearchScrollAction;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
-public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
-
-    public static final Setting<List<String>> KIBANA_INDEX_NAMES_SETTING = Setting.listSetting("kibana.system_indices",
-        List.of(".kibana*", ".reporting"), Function.identity(), Property.NodeScope);
-
-    @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-        return KIBANA_INDEX_NAMES_SETTING.get(settings).stream()
-            .map(pattern -> new SystemIndexDescriptor(pattern, "System index used by kibana"))
-            .collect(Collectors.toUnmodifiableList());
-    }
-
-    @Override
-    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
-                                             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
-                                             IndexNameExpressionResolver indexNameExpressionResolver,
-                                             Supplier<DiscoveryNodes> nodesInCluster) {
-        // TODO need to figure out what subset of system indices Kibana should have access to via these APIs
-        final List<String> allowedIndexPatterns = List.of();
-        return List.of(
-            // Based on https://github.com/elastic/kibana/issues/49764
-            // apis needed to perform migrations... ideally these will go away
-            new KibanaWrappedRestHandler(new RestCreateIndexAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestGetAliasesAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestIndexPutAliasAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestRefreshAction(), allowedIndexPatterns),
-
-            // apis needed to access saved objects
-            new KibanaWrappedRestHandler(new RestGetAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestMultiGetAction(settings), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestSearchAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestBulkAction(settings), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestDeleteAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestDeleteByQueryAction(), allowedIndexPatterns),
-
-            // api used for testing
-            new KibanaWrappedRestHandler(new RestUpdateSettingsAction(), allowedIndexPatterns),
-
-            // apis used specifically by reporting
-            new KibanaWrappedRestHandler(new RestGetIndicesAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestIndexAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new CreateHandler(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new AutoIdHandler(nodesInCluster), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestUpdateAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestSearchScrollAction(), allowedIndexPatterns),
-            new KibanaWrappedRestHandler(new RestClearScrollAction(), allowedIndexPatterns)
-        );
-
-    }
-
-    @Override
-    public List<Setting<?>> getSettings() {
-        return List.of(KIBANA_INDEX_NAMES_SETTING);
-    }
-
-    static class KibanaWrappedRestHandler extends BaseRestHandler.Wrapper {
-
-        private final List<String> allowedIndexPatterns;
-
-        KibanaWrappedRestHandler(BaseRestHandler delegate, List<String> allowedIndexPatterns) {
-            super(delegate);
-            this.allowedIndexPatterns = allowedIndexPatterns;
-        }
-
-        @Override
-        public String getName() {
-            return "kibana_" + super.getName();
-        }
-
-        @Override
-        public List<Route> routes() {
-            return super.routes().stream().map(route -> new Route(route.getMethod(), "/_kibana" + route.getPath()))
-                .collect(Collectors.toUnmodifiableList());
-        }
-
-        @Override
-        protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-            client.threadPool().getThreadContext().allowSystemIndexAccess(allowedIndexPatterns);
-            return super.prepareRequest(request, client);
-        }
-    }
-}
diff --git a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java
deleted file mode 100644
index 5094dd7178bcb..0000000000000
--- a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java
+++ /dev/null
@@ -1,46 +0,0 @@
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.kibana;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.SystemIndexDescriptor;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.is;
-
-public class KibanaPluginTests extends ESTestCase {
-
-    public void testKibanaIndexNames() {
-        assertThat(new KibanaPlugin().getSettings(), contains(KibanaPlugin.KIBANA_INDEX_NAMES_SETTING));
-        assertThat(new KibanaPlugin().getSystemIndexDescriptors(Settings.EMPTY).stream()
-            .map(SystemIndexDescriptor::getIndexPattern).collect(Collectors.toUnmodifiableList()),
-            contains(".kibana*", ".reporting"));
-        final List<String> names = List.of("." + randomAlphaOfLength(4), "." + randomAlphaOfLength(6));
-        final List<String> namesFromDescriptors = new KibanaPlugin().getSystemIndexDescriptors(
-                Settings.builder().putList(KibanaPlugin.KIBANA_INDEX_NAMES_SETTING.getKey(), names).build()
-            ).stream().map(SystemIndexDescriptor::getIndexPattern).collect(Collectors.toUnmodifiableList());
-        assertThat(namesFromDescriptors, is(names));
-    }
-}
diff --git a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java
deleted file mode 100644
index f3901112e839f..0000000000000
--- a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.kibana;
-
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.ESRestTestCase;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.is;
-
-public class KibanaSystemIndexIT extends ESRestTestCase {
-
-    public void testCreateIndex() throws IOException {
-        Request request = new Request("PUT", "/_kibana/.kibana-1");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testAliases() throws IOException {
-        Request request = new Request("PUT", "/_kibana/.kibana-1");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("PUT", "/_kibana/.kibana-1/_alias/.kibana");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("GET", "/_kibana/_aliases");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        assertThat(EntityUtils.toString(response.getEntity()), containsString(".kibana"));
-    }
-
-    public void testBulkToKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testRefresh() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("GET", "/_kibana/.kibana/_refresh");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request getRequest = new Request("GET", "/_kibana/.kibana/_doc/1");
-        Response getResponse = client().performRequest(getRequest);
-        assertThat(getResponse.getStatusLine().getStatusCode(), is(200));
-        String responseBody = EntityUtils.toString(getResponse.getEntity());
-        assertThat(responseBody, containsString("foo"));
-        assertThat(responseBody, containsString("bar"));
-    }
-
-    public void testGetFromKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n");
-        request.addParameter("refresh", "true");
-
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request getRequest = new Request("GET", "/_kibana/.kibana/_doc/1");
-        Response getResponse = client().performRequest(getRequest);
-        assertThat(getResponse.getStatusLine().getStatusCode(), is(200));
-        String responseBody = EntityUtils.toString(getResponse.getEntity());
-        assertThat(responseBody, containsString("foo"));
-        assertThat(responseBody, containsString("bar"));
-    }
-
-    public void testMultiGetFromKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n");
-        request.addParameter("refresh", "true");
-
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request getRequest = new Request("GET", "/_kibana/_mget");
-        getRequest.setJsonEntity("{ \"docs\" : [ { \"_index\" : \".kibana\", \"_id\" : \"1\" }, " +
-            "{ \"_index\" : \".kibana\", \"_id\" : \"2\" } ] }\n");
-        Response getResponse = client().performRequest(getRequest);
-        assertThat(getResponse.getStatusLine().getStatusCode(), is(200));
-        String responseBody = EntityUtils.toString(getResponse.getEntity());
-        assertThat(responseBody, containsString("foo"));
-        assertThat(responseBody, containsString("bar"));
-        assertThat(responseBody, containsString("baz"));
-        assertThat(responseBody, containsString("tag"));
-    }
-
-    public void testSearchFromKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n");
-        request.addParameter("refresh", "true");
-
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request searchRequest = new Request("GET", "/_kibana/.kibana/_search");
-        searchRequest.setJsonEntity("{ \"query\" : { \"match_all\" : {} } }\n");
-        Response getResponse = client().performRequest(searchRequest);
-        assertThat(getResponse.getStatusLine().getStatusCode(), is(200));
-        String responseBody = EntityUtils.toString(getResponse.getEntity());
-        assertThat(responseBody, containsString("foo"));
-        assertThat(responseBody, containsString("bar"));
-        assertThat(responseBody, containsString("baz"));
-        assertThat(responseBody, containsString("tag"));
-    }
-
-    public void testDeleteFromKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n");
-        request.addParameter("refresh", "true");
-
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request deleteRequest = new Request("DELETE", "/_kibana/.kibana/_doc/1");
-        Response deleteResponse = client().performRequest(deleteRequest);
-        assertThat(deleteResponse.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testDeleteByQueryFromKibanaIndex() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n");
-        request.addParameter("refresh", "true");
-
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request dbqRequest = new Request("POST", "/_kibana/.kibana/_delete_by_query");
-        dbqRequest.setJsonEntity("{ \"query\" : { \"match_all\" : {} } }\n");
-        Response dbqResponse = client().performRequest(dbqRequest);
-        assertThat(dbqResponse.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testUpdateIndexSettings() throws IOException {
-        Request request = new Request("PUT", "/_kibana/.kibana-1");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("PUT", "/_kibana/.kibana-1/_settings");
-        request.setJsonEntity("{ \"index.blocks.read_only\" : false }");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testGetIndex() throws IOException {
-        Request request = new Request("PUT", "/_kibana/.kibana-1");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("GET", "/_kibana/.kibana-1");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        assertThat(EntityUtils.toString(response.getEntity()), containsString(".kibana-1"));
-    }
-
-    public void testIndexingAndUpdatingDocs() throws IOException {
-        Request request = new Request("PUT", "/_kibana/.kibana-1/_doc/1");
-        request.setJsonEntity("{ \"foo\" : \"bar\" }");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(201));
-
-        request = new Request("PUT", "/_kibana/.kibana-1/_create/2");
-        request.setJsonEntity("{ \"foo\" : \"bar\" }");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(201));
-
-        request = new Request("POST", "/_kibana/.kibana-1/_doc");
-        request.setJsonEntity("{ \"foo\" : \"bar\" }");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(201));
-
-        request = new Request("GET", "/_kibana/.kibana-1/_refresh");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        request = new Request("POST", "/_kibana/.kibana-1/_update/1");
-        request.setJsonEntity("{ \"doc\" : { \"foo\" : \"baz\" } }");
-        response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-    }
-
-    public void testScrollingDocs() throws IOException {
-        Request request = new Request("POST", "/_kibana/_bulk");
-        request.setJsonEntity("{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"2\" } }\n{ \"baz\" : \"tag\" }\n" +
-            "{ \"index\" : { \"_index\" : \".kibana\", \"_id\" : \"3\" } }\n{ \"baz\" : \"tag\" }\n");
-        request.addParameter("refresh", "true");
-        Response response = client().performRequest(request);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-
-        Request searchRequest = new Request("GET", "/_kibana/.kibana/_search");
-        searchRequest.setJsonEntity("{ \"size\" : 1,\n\"query\" : { \"match_all\" : {} } }\n");
-        searchRequest.addParameter("scroll", "1m");
-        response = client().performRequest(searchRequest);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        Map<String, Object> map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false);
-        assertNotNull(map.get("_scroll_id"));
-        String scrollId = (String) map.get("_scroll_id");
-
-        Request scrollRequest = new Request("POST", "/_kibana/_search/scroll");
-        scrollRequest.addParameter("scroll_id", scrollId);
-        scrollRequest.addParameter("scroll", "1m");
-        response = client().performRequest(scrollRequest);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false);
-        assertNotNull(map.get("_scroll_id"));
-        scrollId = (String) map.get("_scroll_id");
-
-        Request clearScrollRequest = new Request("DELETE", "/_kibana/_search/scroll");
-        clearScrollRequest.addParameter("scroll_id", scrollId);
-        response = client().performRequest(clearScrollRequest);
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-    }
-}
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 19c383c96f0a0..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1219c9aca51a37ea3e22cf88ad2e8745d1a6e02f
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.5.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..ea1af2536868e
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-8.5.0.jar.sha1
@@ -0,0 +1 @@
+41fcbae8104c54487c83c002cf3da6a13065b7a4
\ No newline at end of file
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
index d70c6790317f9..7d15d9a9ba83f 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
@@ -19,44 +19,36 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 
+import java.io.IOException;
+
 /**
  * A ValueSource to create FunctionValues to get the count of the number of values in a field for a document.
  */
-final class CountMethodValueSource extends ValueSource {
-    IndexFieldData<?> fieldData;
+final class CountMethodValueSource extends FieldDataBasedDoubleValuesSource {
 
     CountMethodValueSource(IndexFieldData<?> fieldData) {
-        Objects.requireNonNull(fieldData);
-
-        this.fieldData = fieldData;
+        super(fieldData);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
+    public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) {
+        LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(ctx);
         final SortedNumericDoubleValues values = leafData.getDoubleValues();
+        return new DoubleValues() {
+            @Override
+            public double doubleValue() {
+                return values.docValueCount();
+            }
 
-        return new DoubleDocValues(this) {
             @Override
-            public double doubleVal(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    return values.docValueCount();
-                } else {
-                    return 0;
-                }
+            public boolean advanceExact(int doc) throws IOException {
+                return values.advanceExact(doc);
             }
         };
     }
@@ -65,19 +57,18 @@ public double doubleVal(int doc) throws IOException {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
-        FieldDataValueSource that = (FieldDataValueSource) o;
-
+        CountMethodValueSource that = (CountMethodValueSource) o;
         return fieldData.equals(that.fieldData);
     }
 
     @Override
-    public int hashCode() {
-        return 31 * getClass().hashCode() + fieldData.hashCode();
+    public String toString() {
+        return "count: field(" + fieldData.getFieldName() + ")";
     }
 
     @Override
-    public String description() {
-        return "count: field(" + fieldData.getFieldName() + ")";
+    public int hashCode() {
+        return 31 * getClass().hashCode() + fieldData.hashCode();
     }
+
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
index 2d50f160812dc..3ce572cc43710 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
@@ -19,12 +19,12 @@
  * under the License.
  */
 
-import java.util.Calendar;
-
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.search.MultiValueMode;
 
+import java.util.Calendar;
+
 /**
  * Expressions API for date fields.
  */
@@ -56,7 +56,7 @@ private DateField() {}
     static final String GET_MINUTES_METHOD      = "getMinutes";
     static final String GET_SECONDS_METHOD      = "getSeconds";
 
-    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+    static DoubleValuesSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
         switch (variable) {
             case VALUE_VARIABLE:
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
@@ -69,7 +69,7 @@ static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, St
         }
     }
 
-    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+    static DoubleValuesSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
         switch (method) {
             case GETVALUE_METHOD:
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
index ee8c604a39ba2..f1c918e2d2447 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
@@ -19,21 +19,19 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Calendar;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TimeZone;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
 
+import java.io.IOException;
+import java.util.Calendar;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.TimeZone;
+
 /** Extracts a portion of a date field with {@code Calendar.get()} */
 class DateMethodValueSource extends FieldDataValueSource {
 
@@ -50,27 +48,26 @@ class DateMethodValueSource extends FieldDataValueSource {
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
         NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues());
-        return new DoubleDocValues(this) {
+        return new DoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
+                calendar.setTimeInMillis((long)docValues.doubleValue());
+                return calendar.get(calendarType);
+            }
+
             @Override
-            public double doubleVal(int docId) throws IOException {
-                if (docValues.advanceExact(docId)) {
-                    long millis = (long)docValues.doubleValue();
-                    calendar.setTimeInMillis(millis);
-                    return calendar.get(calendarType);
-                } else {
-                    return 0;
-                }
+            public boolean advanceExact(int doc) throws IOException {
+                return docValues.advanceExact(doc);
             }
         };
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return methodName + ": field(" + fieldData.getFieldName() + ")";
     }
 
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java
index 66cc56370a32d..e5d12bcff1152 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java
@@ -19,7 +19,7 @@
  * under the License.
  */
 
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.search.MultiValueMode;
 import org.joda.time.ReadableDateTime;
@@ -30,7 +30,7 @@
 final class DateObject {
     // no instance
     private DateObject() {}
-    
+
     // supported variables
     static final String CENTURY_OF_ERA_VARIABLE       = "centuryOfEra";
     static final String DAY_OF_MONTH_VARIABLE         = "dayOfMonth";
@@ -50,7 +50,7 @@ private DateObject() {}
     static final String YEAR_VARIABLE                 = "year";
     static final String YEAR_OF_CENTURY_VARIABLE      = "yearOfCentury";
     static final String YEAR_OF_ERA_VARIABLE          = "yearOfEra";
-    
+
     // supported methods
     static final String GETCENTURY_OF_ERA_METHOD      = "getCenturyOfEra";
     static final String GETDAY_OF_MONTH_METHOD        = "getDayOfMonth";
@@ -70,8 +70,8 @@ private DateObject() {}
     static final String GETYEAR_METHOD                = "getYear";
     static final String GETYEAR_OF_CENTURY_METHOD     = "getYearOfCentury";
     static final String GETYEAR_OF_ERA_METHOD         = "getYearOfEra";
-    
-    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+
+    static DoubleValuesSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
         switch (variable) {
             case CENTURY_OF_ERA_VARIABLE:
                 return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getCenturyOfEra);
@@ -110,12 +110,12 @@ static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, St
             case YEAR_OF_ERA_VARIABLE:
                 return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getYearOfEra);
             default:
-                throw new IllegalArgumentException("Member variable [" + variable + 
+                throw new IllegalArgumentException("Member variable [" + variable +
                                                    "] does not exist for date object on field [" + fieldName + "].");
         }
     }
-    
-    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+
+    static DoubleValuesSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
         switch (method) {
             case GETCENTURY_OF_ERA_METHOD:
                 return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getCenturyOfEra);
@@ -154,7 +154,7 @@ static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, Stri
             case GETYEAR_OF_ERA_METHOD:
                 return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getYearOfEra);
             default:
-                throw new IllegalArgumentException("Member method [" + method + 
+                throw new IllegalArgumentException("Member method [" + method +
                                                    "] does not exist for date object on field [" + fieldName + "].");
         }
     }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
index 89f991c174d30..d49dd6c0a1943 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
@@ -19,14 +19,8 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-import java.util.function.ToIntFunction;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
@@ -35,6 +29,10 @@
 import org.joda.time.MutableDateTime;
 import org.joda.time.ReadableDateTime;
 
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.ToIntFunction;
+
 /** Extracts a portion of a date field with joda time */
 class DateObjectValueSource extends FieldDataValueSource {
 
@@ -52,27 +50,26 @@ class DateObjectValueSource extends FieldDataValueSource {
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         MutableDateTime joda = new MutableDateTime(0, DateTimeZone.UTC);
         NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues());
-        return new DoubleDocValues(this) {
+        return DoubleValues.withDefault(new DoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
+                joda.setMillis((long)docValues.doubleValue());
+                return function.applyAsInt(joda);
+            }
+
             @Override
-            public double doubleVal(int docId) throws IOException {
-                if (docValues.advanceExact(docId)) {
-                    long millis = (long)docValues.doubleValue();
-                    joda.setMillis(millis);
-                    return function.applyAsInt(joda);
-                } else {
-                    return 0;
-                }
+            public boolean advanceExact(int doc) throws IOException {
+                return docValues.advanceExact(doc);
             }
-        };
+        }, 0);
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return methodName + ": field(" + fieldData.getFieldName() + ")";
     }
 
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
index 750b03762a7f9..26d300547fbac 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
@@ -20,44 +20,39 @@
 package org.elasticsearch.script.expression;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 
 import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
 
 /**
  * ValueSource to return non-zero if a field is missing.
  * <p>
  * This is essentially sugar over !count()
  */
-final class EmptyMemberValueSource extends ValueSource {
-    final IndexFieldData<?> fieldData;
+final class EmptyMemberValueSource extends FieldDataBasedDoubleValuesSource {
 
     EmptyMemberValueSource(IndexFieldData<?> fieldData) {
-        this.fieldData = Objects.requireNonNull(fieldData);
+        super(fieldData);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         final SortedNumericDoubleValues values = leafData.getDoubleValues();
-        return new DoubleDocValues(this) {
+        return DoubleValues.withDefault(new DoubleValues() {
             @Override
-            public double doubleVal(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    return 0;
-                } else {
-                    return 1;
-                }
+            public double doubleValue() {
+                return 0;
             }
-        };
+
+            @Override
+            public boolean advanceExact(int doc) throws IOException {
+                return values.advanceExact(doc);
+            }
+        }, 1);
     }
 
     @Override
@@ -71,12 +66,12 @@ public boolean equals(Object obj) {
         if (obj == null) return false;
         if (getClass() != obj.getClass()) return false;
         EmptyMemberValueSource other = (EmptyMemberValueSource) obj;
-        if (!fieldData.equals(other.fieldData)) return false;
-        return true;
+        return fieldData.equals(other.fieldData);
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return "empty: field(" + fieldData.getFieldName() + ")";
     }
+
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
index b89a6231a67da..19865bd361e6c 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
@@ -23,8 +23,8 @@
 import org.apache.lucene.expressions.SimpleBindings;
 import org.apache.lucene.expressions.js.JavascriptCompiler;
 import org.apache.lucene.expressions.js.VariableContext;
-import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
@@ -251,9 +251,9 @@ private static NumberSortScript.LeafFactory newSortScript(Expression expr, Searc
                 } else {
                     // delegate valuesource creation based on field's type
                     // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
-                    final ValueSource valueSource = getDocValueSource(variable, lookup);
-                    needsScores |= valueSource.getSortField(false).needsScores();
-                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                    final DoubleValuesSource valueSource = getDocValueSource(variable, lookup);
+                    needsScores |= valueSource.needsScores();
+                    bindings.add(variable, valueSource);
                 }
             } catch (Exception e) {
                 // we defer "binding" of variables until here: give context for that variable
@@ -275,8 +275,7 @@ private static TermsSetQueryScript.LeafFactory newTermsSetQueryScript(Expression
                 } else {
                     // delegate valuesource creation based on field's type
                     // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
-                    final ValueSource valueSource = getDocValueSource(variable, lookup);
-                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                    bindings.add(variable, getDocValueSource(variable, lookup));
                 }
             } catch (Exception e) {
                 // we defer "binding" of variables until here: give context for that variable
@@ -310,9 +309,9 @@ private static AggregationScript.LeafFactory newAggregationScript(Expression exp
                 } else {
                     // delegate valuesource creation based on field's type
                     // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
-                    final ValueSource valueSource = getDocValueSource(variable, lookup);
-                    needsScores |= valueSource.getSortField(false).needsScores();
-                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                    final DoubleValuesSource valueSource = getDocValueSource(variable, lookup);
+                    needsScores |= valueSource.needsScores();
+                    bindings.add(variable, valueSource);
                 }
             } catch (Exception e) {
                 // we defer "binding" of variables until here: give context for that variable
@@ -329,8 +328,7 @@ private static FieldScript.LeafFactory newFieldScript(Expression expr, SearchLoo
                 if (vars != null && vars.containsKey(variable)) {
                     bindFromParams(vars, bindings, variable);
                 } else {
-                    final ValueSource valueSource = getDocValueSource(variable, lookup);
-                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                    bindings.add(variable, getDocValueSource(variable, lookup));
                 }
             } catch (Exception e) {
                 throw convertToScriptException("link error", expr.sourceText, variable, e);
@@ -382,9 +380,9 @@ private static ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLoo
                 } else {
                     // delegate valuesource creation based on field's type
                     // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
-                    final ValueSource valueSource = getDocValueSource(variable, lookup);
-                    needsScores |= valueSource.getSortField(false).needsScores();
-                    bindings.add(variable, valueSource.asDoubleValuesSource());
+                    final DoubleValuesSource valueSource = getDocValueSource(variable, lookup);
+                    needsScores |= valueSource.needsScores();
+                    bindings.add(variable, valueSource);
                 }
             } catch (Exception e) {
                 // we defer "binding" of variables until here: give context for that variable
@@ -412,7 +410,7 @@ private static ScriptException convertToScriptException(String message, String s
         throw new ScriptException(message, cause, stack, source, NAME);
     }
 
-    private static ValueSource getDocValueSource(String variable, SearchLookup lookup) throws ParseException {
+    private static DoubleValuesSource getDocValueSource(String variable, SearchLookup lookup) throws ParseException {
         VariableContext[] parts = VariableContext.parse(variable);
         if (parts[0].text.equals("doc") == false) {
             throw new ParseException("Unknown variable [" + parts[0].text + "]", 0);
@@ -463,7 +461,7 @@ private static ValueSource getDocValueSource(String variable, SearchLookup looku
         }
 
         IndexFieldData<?> fieldData = lookup.doc().getForField(fieldType);
-        final ValueSource valueSource;
+        final DoubleValuesSource valueSource;
         if (fieldType instanceof GeoPointFieldType) {
             // geo
             if (methodname == null) {
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java
new file mode 100644
index 0000000000000..ca36ba26bd74e
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+
+import java.util.Objects;
+
+abstract class FieldDataBasedDoubleValuesSource extends DoubleValuesSource {
+
+    FieldDataBasedDoubleValuesSource(IndexFieldData<?> fieldData) {
+        this.fieldData = Objects.requireNonNull(fieldData);
+    }
+
+    protected final IndexFieldData<?> fieldData;
+
+    @Override
+    public boolean needsScores() {
+        return false;
+    }
+
+    @Override
+    public DoubleValuesSource rewrite(IndexSearcher reader) {
+        return this;
+    }
+
+    @Override
+    public boolean isCacheable(LeafReaderContext ctx) {
+        return DocValues.isCacheable(ctx, fieldData.getFieldName());
+    }
+
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
index 63050a07da8bf..7ab8c3dd8f8bd 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
@@ -19,29 +19,26 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
 
+import java.io.IOException;
+import java.util.Objects;
+
 /**
  * A {@link ValueSource} wrapper for field data.
  */
-class FieldDataValueSource extends ValueSource {
+class FieldDataValueSource extends FieldDataBasedDoubleValuesSource {
 
-    final IndexFieldData<?> fieldData;
     final MultiValueMode multiValueMode;
 
     protected FieldDataValueSource(IndexFieldData<?> fieldData, MultiValueMode multiValueMode) {
-        this.fieldData = Objects.requireNonNull(fieldData);
+        super(fieldData);
         this.multiValueMode = Objects.requireNonNull(multiValueMode);
     }
 
@@ -65,24 +62,25 @@ public int hashCode() {
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafNumericFieldData leafData = (LeafNumericFieldData) fieldData.load(leaf);
         NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues());
-        return new DoubleDocValues(this) {
-          @Override
-          public double doubleVal(int doc) throws IOException {
-            if (docValues.advanceExact(doc)) {
+        return new DoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
                 return docValues.doubleValue();
-            } else {
-                return 0;
             }
-          }
+
+            @Override
+            public boolean advanceExact(int doc) throws IOException {
+                return docValues.advanceExact(doc);
+            }
         };
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return "field(" + fieldData.getFieldName() + ")";
     }
+
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
index 2949e1dd9287a..5e3a20f532e7f 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
@@ -19,43 +19,38 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 
+import java.io.IOException;
+
 /**
  * ValueSource to return non-zero if a field is missing.
  */
-final class GeoEmptyValueSource extends ValueSource {
-    IndexFieldData<?> fieldData;
+final class GeoEmptyValueSource extends FieldDataBasedDoubleValuesSource {
 
     GeoEmptyValueSource(IndexFieldData<?> fieldData) {
-        this.fieldData = Objects.requireNonNull(fieldData);
+        super(fieldData);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafGeoPointFieldData leafData = (LeafGeoPointFieldData) fieldData.load(leaf);
         final MultiGeoPointValues values = leafData.getGeoPointValues();
-        return new DoubleDocValues(this) {
+        return DoubleValues.withDefault(new DoubleValues() {
             @Override
-            public double doubleVal(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    return 1;
-                } else {
-                    return 0;
-                }
+            public double doubleValue() {
+                return 1;
             }
-        };
+
+            @Override
+            public boolean advanceExact(int doc) throws IOException {
+                return values.advanceExact(doc);
+            }
+        }, 0);
     }
 
     @Override
@@ -69,12 +64,12 @@ public boolean equals(Object obj) {
         if (obj == null) return false;
         if (getClass() != obj.getClass()) return false;
         GeoEmptyValueSource other = (GeoEmptyValueSource) obj;
-        if (!fieldData.equals(other.fieldData)) return false;
-        return true;
+        return fieldData.equals(other.fieldData);
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return "empty: field(" + fieldData.getFieldName() + ")";
     }
+
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
index 0ce74a187df77..4b15e30aba414 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
@@ -19,7 +19,7 @@
  * under the License.
  */
 
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 
 /**
@@ -28,18 +28,18 @@
 final class GeoField {
     // no instance
     private GeoField() {}
-    
+
     // supported variables
     static final String EMPTY_VARIABLE        = "empty";
     static final String LAT_VARIABLE          = "lat";
     static final String LON_VARIABLE          = "lon";
-    
+
     // supported methods
     static final String ISEMPTY_METHOD        = "isEmpty";
     static final String GETLAT_METHOD         = "getLat";
     static final String GETLON_METHOD         = "getLon";
-    
-    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+
+    static DoubleValuesSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
         switch (variable) {
             case EMPTY_VARIABLE:
                 return new GeoEmptyValueSource(fieldData);
@@ -51,8 +51,8 @@ static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, St
                 throw new IllegalArgumentException("Member variable [" + variable + "] does not exist for geo field [" + fieldName + "].");
         }
     }
-    
-    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+
+    static DoubleValuesSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
         switch (method) {
             case ISEMPTY_METHOD:
                 return new GeoEmptyValueSource(fieldData);
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
index 3c04de1658669..8b9a3282ec78b 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
@@ -19,43 +19,38 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 
+import java.io.IOException;
+
 /**
  * ValueSource to return latitudes as a double "stream" for geopoint fields
  */
-final class GeoLatitudeValueSource extends ValueSource {
-    final IndexFieldData<?> fieldData;
+final class GeoLatitudeValueSource extends FieldDataBasedDoubleValuesSource {
 
     GeoLatitudeValueSource(IndexFieldData<?> fieldData) {
-        this.fieldData = Objects.requireNonNull(fieldData);
+        super(fieldData);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafGeoPointFieldData leafData = (LeafGeoPointFieldData) fieldData.load(leaf);
         final MultiGeoPointValues values = leafData.getGeoPointValues();
-        return new DoubleDocValues(this) {
+        return DoubleValues.withDefault(new DoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
+                return values.nextValue().getLat();
+            }
+
             @Override
-            public double doubleVal(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    return values.nextValue().getLat();
-                } else {
-                    return 0.0;
-                }
+            public boolean advanceExact(int doc) throws IOException {
+                return values.advanceExact(doc);
             }
-        };
+        }, 0);
     }
 
     @Override
@@ -69,12 +64,11 @@ public boolean equals(Object obj) {
         if (obj == null) return false;
         if (getClass() != obj.getClass()) return false;
         GeoLatitudeValueSource other = (GeoLatitudeValueSource) obj;
-        if (!fieldData.equals(other.fieldData)) return false;
-        return true;
+        return fieldData.equals(other.fieldData);
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return "lat: field(" + fieldData.getFieldName() + ")";
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
index 8b2c5d83c0257..9bd285f8da871 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
@@ -19,43 +19,38 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
 import org.elasticsearch.index.fielddata.LeafGeoPointFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 
+import java.io.IOException;
+
 /**
  * ValueSource to return longitudes as a double "stream" for geopoint fields
  */
-final class GeoLongitudeValueSource extends ValueSource {
-    final IndexFieldData<?> fieldData;
+final class GeoLongitudeValueSource extends FieldDataBasedDoubleValuesSource {
 
     GeoLongitudeValueSource(IndexFieldData<?> fieldData) {
-        this.fieldData = Objects.requireNonNull(fieldData);
+        super(fieldData);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+    public DoubleValues getValues(LeafReaderContext leaf, DoubleValues scores) {
         LeafGeoPointFieldData leafData = (LeafGeoPointFieldData) fieldData.load(leaf);
         final MultiGeoPointValues values = leafData.getGeoPointValues();
-        return new DoubleDocValues(this) {
+        return DoubleValues.withDefault(new DoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
+                return values.nextValue().getLon();
+            }
+
             @Override
-            public double doubleVal(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    return values.nextValue().getLon();
-                } else {
-                    return 0.0;
-                }
+            public boolean advanceExact(int doc) throws IOException {
+                return values.advanceExact(doc);
             }
-        };
+        }, 0.0);
     }
 
     @Override
@@ -74,7 +69,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public String description() {
+    public String toString() {
         return "lon: field(" + fieldData.getFieldName() + ")";
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
index 06875632134d7..0eeff96ba08e4 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
@@ -19,7 +19,7 @@
  * under the License.
  */
 
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.search.MultiValueMode;
 
@@ -46,7 +46,7 @@ private NumericField() {}
     static final String SUM_METHOD              = "sum";
     static final String COUNT_METHOD            = "count";
 
-    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+    static DoubleValuesSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
         switch (variable) {
             case VALUE_VARIABLE:
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
@@ -60,7 +60,7 @@ static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, St
         }
     }
 
-    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+    static DoubleValuesSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
         switch (method) {
             case GETVALUE_METHOD:
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
index a0cfd721adb5a..c237ca3101673 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
@@ -80,7 +80,7 @@ public boolean isCacheable(LeafReaderContext ctx) {
     }
 
     @Override
-    public DoubleValuesSource rewrite(IndexSearcher reader) throws IOException {
+    public DoubleValuesSource rewrite(IndexSearcher reader) {
         return this;
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
index 557f9ad812e0c..4efc8f9791cd9 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
@@ -21,8 +21,6 @@
 
 import org.apache.lucene.search.DoubleValues;
 
-import java.io.IOException;
-
 /**
  * A support class for an executable expression script that allows the double returned
  * by a {@link DoubleValues} to be modified.
@@ -35,12 +33,12 @@ void setValue(double value) {
     }
 
     @Override
-    public double doubleValue() throws IOException {
+    public double doubleValue() {
         return value;
     }
 
     @Override
-    public boolean advanceExact(int doc) throws IOException {
+    public boolean advanceExact(int doc) {
         return true;
     }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java
index dadaf7cb05a09..bc2577c8b4b02 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java
@@ -53,7 +53,7 @@ protected MultiSearchTemplateResponse createTestInstance() {
             SearchResponse.Clusters clusters = randomClusters();
             SearchTemplateResponse searchTemplateResponse = new SearchTemplateResponse();
             SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, totalShards,
-                    successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters);
+                    successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters, null);
             searchTemplateResponse.setResponse(searchResponse);
             items[i] = new MultiSearchTemplateResponse.Item(searchTemplateResponse, null);
         }
@@ -82,7 +82,7 @@ private static  MultiSearchTemplateResponse createTestInstanceWithFailures() {
                 SearchResponse.Clusters clusters = randomClusters();
                 SearchTemplateResponse searchTemplateResponse = new SearchTemplateResponse();
                 SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, totalShards,
-                        successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters);
+                        successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters, null);
                 searchTemplateResponse.setResponse(searchResponse);
                 items[i] = new MultiSearchTemplateResponse.Item(searchTemplateResponse, null);
             } else {
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java
index 1d93e202d1a13..3f25f05af6ee9 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java
@@ -69,7 +69,7 @@ private static SearchResponse createSearchResponse() {
         InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
 
         return new SearchResponse(internalSearchResponse, null, totalShards, successfulShards,
-            skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+            skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
     }
 
     private static BytesReference createSource() {
@@ -171,7 +171,7 @@ public void testSearchResponseToXContent() throws IOException {
         InternalSearchResponse internalSearchResponse = new InternalSearchResponse(
             new SearchHits(hits, new TotalHits(100, TotalHits.Relation.EQUAL_TO), 1.5f), null, null, null, false, null, 1);
         SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null,
-            0, 0, 0, 0, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+            0, 0, 0, 0, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
 
         SearchTemplateResponse response = new SearchTemplateResponse();
         response.setResponse(searchResponse);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index ab09010d248d4..e9dcdb1ef409d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -210,8 +210,7 @@ ScriptRoot compile(Loader loader, String name, String source, CompilerSettings s
         ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SClass root = Walker.buildPainlessTree(scriptClassInfo, name, source, settings, painlessLookup, null);
         ScriptRoot scriptRoot = new ScriptRoot(painlessLookup, settings, scriptClassInfo, root);
-        root.analyze(scriptRoot);
-        ClassNode classNode = root.writeClass();
+        ClassNode classNode = root.writeClass(scriptRoot);
         DefBootstrapInjectionPhase.phase(classNode);
         ScriptInjectionPhase.phase(scriptRoot, classNode);
         byte[] bytes = classNode.write();
@@ -240,8 +239,7 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de
         ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SClass root = Walker.buildPainlessTree(scriptClassInfo, name, source, settings, painlessLookup, debugStream);
         ScriptRoot scriptRoot = new ScriptRoot(painlessLookup, settings, scriptClassInfo, root);
-        root.analyze(scriptRoot);
-        ClassNode classNode = root.writeClass();
+        ClassNode classNode = root.writeClass(scriptRoot);
         DefBootstrapInjectionPhase.phase(classNode);
         ScriptInjectionPhase.phase(scriptRoot, classNode);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 189cb94dcd70b..e7fb9e951c59e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -79,6 +79,18 @@ public static class Output {
          * called on this node to get the type of the node after the cast.</b>
          */
         Class<?> actual = null;
+
+        /**
+         * The {@link PainlessCast} to convert this expression's actual type
+         * to the parent expression's expected type. {@code null} if no cast
+         * is required.
+         */
+        PainlessCast painlessCast = null;
+
+        /**
+         * The {@link ExpressionNode}(s) generated from this expression.
+         */
+        ExpressionNode expressionNode = null;
     }
 
     /**
@@ -90,16 +102,6 @@ public static class Output {
      */
     AExpression prefix;
 
-    // TODO: remove placeholders once analysis and write are combined into build
-    // TODO: https://github.com/elastic/elasticsearch/issues/53561
-    // This are used to support the transition from a mutable to immutable state.
-    // Currently, the IR tree is built during the user tree "write" phase, so
-    // these are stored on the node to set during the "semantic" phase and then
-    // use during the "write" phase.
-    Input input = null;
-    Output output = null;
-    PainlessCast cast = null;
-
     /**
      * Standard constructor with location used for error tracking.
      */
@@ -121,29 +123,24 @@ public static class Output {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
         throw new UnsupportedOperationException();
     }
 
-    /**
-     * Writes ASM based on the data collected during the analysis phase.
-     */
-    abstract ExpressionNode write(ClassNode classNode);
-
-    void cast() {
-        cast = AnalyzerCaster.getLegalCast(location, output.actual, input.expected, input.explicit, input.internal);
+    void cast(Input input, Output output) {
+        output.painlessCast = AnalyzerCaster.getLegalCast(location, output.actual, input.expected, input.explicit, input.internal);
     }
 
-    ExpressionNode cast(ExpressionNode expressionNode) {
-        if (cast == null) {
-            return expressionNode;
+    ExpressionNode cast(Output output) {
+        if (output.painlessCast == null) {
+            return output.expressionNode;
         }
 
         CastNode castNode = new CastNode();
         castNode.setLocation(location);
-        castNode.setExpressionType(cast.targetType);
-        castNode.setCast(cast);
-        castNode.setChildNode(expressionNode);
+        castNode.setExpressionType(output.painlessCast.targetType);
+        castNode.setCast(output.painlessCast);
+        castNode.setChildNode(output.expressionNode);
 
         return castNode;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
index 0566399095565..4790e222e99d7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.ir.ClassNode;
-import org.elasticsearch.painless.ir.IRNode;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -51,12 +49,6 @@ public abstract class ANode {
         this.location = Objects.requireNonNull(location);
     }
 
-    /**
-     * Writes ASM based on the data collected during the analysis phase.
-     * @param classNode the root {@link ClassNode}
-     */
-    abstract IRNode write(ClassNode classNode);
-
     /**
      * Create an error with location information pointing to this node.
      */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index d8d12d8a14698..06c1affc4a4a7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -94,13 +94,13 @@ public static class Output {
          * Set to the approximate number of statements in a loop block to prevent
          * infinite loops during runtime.
          */
-        int statementCount = 0;
-    }
+        int statementCount = 1;
 
-    // TODO: remove placeholders once analysis and write are combined into build
-    // TODO: https://github.com/elastic/elasticsearch/issues/53561
-    Input input;
-    Output output;
+        /**
+         * The {@link StatementNode}(s) generated from this expression.
+         */
+        StatementNode statementNode = null;
+    }
 
     /**
      * Standard constructor with location used for error tracking.
@@ -112,12 +112,7 @@ public static class Output {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
         throw new UnsupportedOperationException();
     }
-
-    /**
-     * Writes ASM based on the data collected during the analysis phase.
-     */
-    abstract StatementNode write(ClassNode classNode);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
index cc7262f670a94..4bca9bb37cbc5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStoreable.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -57,7 +58,7 @@ public static class Input extends AExpression.Input {
         this.prefix = Objects.requireNonNull(prefix);
     }
 
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
         throw new UnsupportedOperationException();
     }
 
@@ -66,11 +67,4 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
      * rhs actual type to avoid an unnecessary cast.
      */
     abstract boolean isDefOptimized();
-
-    /**
-     * If this node or a sub-node of this node uses dynamic calls then
-     * actual will be set to this value. This is used for an optimization
-     * during assignment to def type targets.
-     */
-    abstract void updateActual(Class<?> actual);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
index e995d637520c7..970d31935831b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
@@ -25,7 +25,13 @@
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.AssignmentNode;
+import org.elasticsearch.painless.ir.BinaryMathNode;
+import org.elasticsearch.painless.ir.BraceNode;
+import org.elasticsearch.painless.ir.BraceSubDefNode;
 import org.elasticsearch.painless.ir.ClassNode;
+import org.elasticsearch.painless.ir.DotNode;
+import org.elasticsearch.painless.ir.DotSubDefNode;
+import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.lookup.PainlessCast;
 import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -37,19 +43,13 @@
 /**
  * Represents an assignment with the lhs and rhs as child nodes.
  */
-public final class EAssignment extends AExpression {
+public class EAssignment extends AExpression {
 
-    private AExpression lhs;
-    private AExpression rhs;
-    private final boolean pre;
-    private final boolean post;
-    private Operation operation;
-
-    private boolean cat = false;
-    private Class<?> promote = null;
-    private Class<?> shiftDistance; // for shifts, the RHS is promoted independently
-    private PainlessCast there = null;
-    private PainlessCast back = null;
+    protected final AExpression lhs;
+    protected final AExpression rhs;
+    protected final boolean pre;
+    protected final boolean post;
+    protected final Operation operation;
 
     public EAssignment(Location location, AExpression lhs, AExpression rhs, boolean pre, boolean post, Operation operation) {
         super(location);
@@ -62,11 +62,20 @@ public EAssignment(Location location, AExpression lhs, AExpression rhs, boolean
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        AExpression rhs = this.rhs;
+        Operation operation = this.operation;
+        boolean cat = false;
+        Class<?> promote = null;
+        Class<?> shiftDistance = null;
+        PainlessCast there = null;
+        PainlessCast back = null;
 
         Output leftOutput;
+
+        Input rightInput = new Input();
         Output rightOutput;
 
         if (lhs instanceof AStoreable) {
@@ -75,7 +84,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
             leftInput.read = input.read;
             leftInput.write = true;
-            leftOutput = lhs.analyze(scriptRoot, scope, leftInput);
+            leftOutput = lhs.analyze(classNode, scriptRoot, scope, leftInput);
         } else {
             throw new IllegalArgumentException("Left-hand side cannot be assigned a value.");
         }
@@ -117,7 +126,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
 
         if (operation != null) {
-            rightOutput = rhs.analyze(scriptRoot, scope, new Input());
+            rightOutput = rhs.analyze(classNode, scriptRoot, scope, rightInput);
             boolean shift = false;
 
             if (operation == Operation.MUL) {
@@ -161,25 +170,25 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
             if (cat) {
                 if (rhs instanceof EBinary && ((EBinary)rhs).operation == Operation.ADD && rightOutput.actual == String.class) {
-                    ((EBinary)rhs).cat = true;
+                    ((BinaryMathNode)rightOutput.expressionNode).setCat(true);
                 }
             }
 
             if (shift) {
                 if (promote == def.class) {
                     // shifts are promoted independently, but for the def type, we need object.
-                    rhs.input.expected = promote;
+                    rightInput.expected = promote;
                 } else if (shiftDistance == long.class) {
-                    rhs.input.expected = int.class;
-                    rhs.input.explicit = true;
+                    rightInput.expected = int.class;
+                    rightInput.explicit = true;
                 } else {
-                    rhs.input.expected = shiftDistance;
+                    rightInput.expected = shiftDistance;
                 }
             } else {
-                rhs.input.expected = promote;
+                rightInput.expected = promote;
             }
 
-            rhs.cast();
+            rhs.cast(rightInput, rightOutput);
 
             there = AnalyzerCaster.getLegalCast(location, leftOutput.actual, promote, false, false);
             back = AnalyzerCaster.getLegalCast(location, promote, leftOutput.actual, true, false);
@@ -188,24 +197,33 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         } else if (rhs != null) {
             AStoreable lhs = (AStoreable)this.lhs;
 
+            // TODO: move this optimization to a later phase
             // If the lhs node is a def optimized node we update the actual type to remove the need for a cast.
             if (lhs.isDefOptimized()) {
-                rightOutput = rhs.analyze(scriptRoot, scope, new Input());
+                rightOutput = rhs.analyze(classNode, scriptRoot, scope, rightInput);
 
                 if (rightOutput.actual == void.class) {
                     throw createError(new IllegalArgumentException("Right-hand side cannot be a [void] type for assignment."));
                 }
 
-                rhs.input.expected = rightOutput.actual;
-                lhs.updateActual(rightOutput.actual);
+                rightInput.expected = rightOutput.actual;
+                leftOutput.actual = rightOutput.actual;
+                leftOutput.expressionNode.setExpressionType(rightOutput.actual);
+
+                ExpressionNode expressionNode = leftOutput.expressionNode;
+
+                if (expressionNode instanceof DotNode && ((DotNode)expressionNode).getRightNode() instanceof DotSubDefNode) {
+                    ((DotNode)expressionNode).getRightNode().setExpressionType(leftOutput.actual);
+                } else if (expressionNode instanceof BraceNode && ((BraceNode)expressionNode).getRightNode() instanceof BraceSubDefNode) {
+                    ((BraceNode)expressionNode).getRightNode().setExpressionType(leftOutput.actual);
+                }
             // Otherwise, we must adapt the rhs type to the lhs type with a cast.
             } else {
-                Input rightInput = new Input();
                 rightInput.expected = leftOutput.actual;
-                rhs.analyze(scriptRoot, scope, rightInput);
+                rightOutput = rhs.analyze(classNode, scriptRoot, scope, rightInput);
             }
 
-            rhs.cast();
+            rhs.cast(rightInput, rightOutput);
         } else {
             throw new IllegalStateException("Illegal tree structure.");
         }
@@ -213,21 +231,10 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         output.statement = true;
         output.actual = input.read ? leftOutput.actual : void.class;
 
-        return output;
-    }
-
-    /**
-     * Handles writing byte code for variable/method chains for all given possibilities
-     * including String concatenation, compound assignment, regular assignment, and simple
-     * reads.  Includes proper duplication for chained assignments and assignments that are
-     * also read from.
-     */
-    @Override
-    AssignmentNode write(ClassNode classNode) {
         AssignmentNode assignmentNode = new AssignmentNode();
 
-        assignmentNode.setLeftNode(lhs.write(classNode));
-        assignmentNode.setRightNode(rhs.cast(rhs.write(classNode)));
+        assignmentNode.setLeftNode(leftOutput.expressionNode);
+        assignmentNode.setRightNode(rhs.cast(rightOutput));
 
         assignmentNode.setLocation(location);
         assignmentNode.setExpressionType(output.actual);
@@ -240,7 +247,9 @@ AssignmentNode write(ClassNode classNode) {
         assignmentNode.setThere(there);
         assignmentNode.setBack(back);
 
-        return assignmentNode;
+        output.expressionNode = assignmentNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 5a65d74d4158c..9f6fc56f97ff2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -35,16 +35,11 @@
 /**
  * Represents a binary math expression.
  */
-public final class EBinary extends AExpression {
+public class EBinary extends AExpression {
 
-    final Operation operation;
-    private AExpression left;
-    private AExpression right;
-
-    private Class<?> promote = null;            // promoted type
-    private Class<?> shiftDistance = null;      // for shifts, the rhs is promoted independently
-    boolean cat = false;
-    private boolean originallyExplicit = false; // record whether there was originally an explicit cast
+    protected final Operation operation;
+    protected final AExpression left;
+    protected final AExpression right;
 
     public EBinary(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
@@ -55,18 +50,22 @@ public EBinary(Location location, Operation operation, AExpression left, AExpres
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Class<?> promote = null;            // promoted type
+        Class<?> shiftDistance = null;      // for shifts, the rhs is promoted independently
+        boolean originallyExplicit = input.explicit; // record whether there was originally an explicit cast
+
+        Output output = new Output();
 
-        originallyExplicit = input.explicit;
+        Input leftInput = new Input();
+        Output leftOutput = left.analyze(classNode, scriptRoot, scope, leftInput);
 
-        Output leftOutput = left.analyze(scriptRoot, scope, new Input());
-        Output rightOutput = right.analyze(scriptRoot, scope, new Input());
+        Input rightInput = new Input();
+        Output rightOutput = right.analyze(classNode, scriptRoot, scope, rightInput);
 
         if (operation == Operation.FIND || operation == Operation.MATCH) {
-            left.input.expected = String.class;
-            right.input.expected = Pattern.class;
+            leftInput.expected = String.class;
+            rightInput.expected = Pattern.class;
             promote = boolean.class;
             output.actual = boolean.class;
         } else {
@@ -99,61 +98,58 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = promote;
 
             if (operation == Operation.ADD && promote == String.class) {
-                left.input.expected = leftOutput.actual;
-                right.input.expected = rightOutput.actual;
+                leftInput.expected = leftOutput.actual;
+                rightInput.expected = rightOutput.actual;
 
                 if (left instanceof EBinary && ((EBinary) left).operation == Operation.ADD && leftOutput.actual == String.class) {
-                    ((EBinary) left).cat = true;
+                    ((BinaryMathNode)leftOutput.expressionNode).setCat(true);
                 }
 
                 if (right instanceof EBinary && ((EBinary) right).operation == Operation.ADD && rightOutput.actual == String.class) {
-                    ((EBinary) right).cat = true;
+                    ((BinaryMathNode)rightOutput.expressionNode).setCat(true);
                 }
-            } else if (promote == def.class || shiftDistance != null && shiftDistance == def.class) {
-                left.input.expected = leftOutput.actual;
-                right.input.expected = rightOutput.actual;
+            } else if (promote == def.class || shiftDistance == def.class) {
+                leftInput.expected = leftOutput.actual;
+                rightInput.expected = rightOutput.actual;
 
                 if (input.expected != null) {
                     output.actual = input.expected;
                 }
             } else {
-                left.input.expected = promote;
+                leftInput.expected = promote;
 
                 if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
                     if (shiftDistance == long.class) {
-                        right.input.expected = int.class;
-                        right.input.explicit = true;
+                        rightInput.expected = int.class;
+                        rightInput.explicit = true;
                     } else {
-                        right.input.expected = shiftDistance;
+                        rightInput.expected = shiftDistance;
                     }
                 } else {
-                    right.input.expected = promote;
+                    rightInput.expected = promote;
                 }
             }
         }
 
-        left.cast();
-        right.cast();
-
-        return output;
-    }
+        left.cast(leftInput, leftOutput);
+        right.cast(rightInput, rightOutput);
 
-    @Override
-    BinaryMathNode write(ClassNode classNode) {
         BinaryMathNode binaryMathNode = new BinaryMathNode();
 
-        binaryMathNode.setLeftNode(left.cast(left.write(classNode)));
-        binaryMathNode.setRightNode(right.cast(right.write(classNode)));
+        binaryMathNode.setLeftNode(left.cast(leftOutput));
+        binaryMathNode.setRightNode(right.cast(rightOutput));
 
         binaryMathNode.setLocation(location);
         binaryMathNode.setExpressionType(output.actual);
         binaryMathNode.setBinaryType(promote);
         binaryMathNode.setShiftType(shiftDistance);
         binaryMathNode.setOperation(operation);
-        binaryMathNode.setCat(cat);
+        binaryMathNode.setCat(false);
         binaryMathNode.setOriginallExplicit(originallyExplicit);
 
-        return binaryMathNode;
+        output.expressionNode = binaryMathNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index f2acae9299f20..e9dac2123c6cd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -31,11 +31,11 @@
 /**
  * Represents a boolean expression.
  */
-public final class EBool extends AExpression {
+public class EBool extends AExpression {
 
-    private final Operation operation;
-    private AExpression left;
-    private AExpression right;
+    protected final Operation operation;
+    protected final AExpression left;
+    protected final AExpression right;
 
     public EBool(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
@@ -46,37 +46,33 @@ public EBool(Location location, Operation operation, AExpression left, AExpressi
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         Input leftInput = new Input();
         leftInput.expected = boolean.class;
-        left.analyze(scriptRoot, scope, leftInput);
-        left.cast();
+        Output leftOutput = left.analyze(classNode, scriptRoot, scope, leftInput);
+        left.cast(leftInput, leftOutput);
 
         Input rightInput = new Input();
         rightInput.expected = boolean.class;
-        right.analyze(scriptRoot, scope, rightInput);
-        right.cast();
+        Output rightOutput = right.analyze(classNode, scriptRoot, scope, rightInput);
+        right.cast(rightInput, rightOutput);
 
         output.actual = boolean.class;
 
-        return output;
-    }
-
-    @Override
-    BooleanNode write(ClassNode classNode) {
         BooleanNode booleanNode = new BooleanNode();
 
-        booleanNode.setLeftNode(left.cast(left.write(classNode)));
-        booleanNode.setRightNode(right.cast(right.write(classNode)));
+        booleanNode.setLeftNode(left.cast(leftOutput));
+        booleanNode.setRightNode(right.cast(rightOutput));
 
         booleanNode.setLocation(location);
         booleanNode.setExpressionType(output.actual);
         booleanNode.setOperation(operation);
 
-        return booleanNode;
+        output.expressionNode = booleanNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 6e78fb972008c..b1362e99cb2a5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -23,13 +23,12 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ConstantNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 /**
  * Represents a boolean constant.
  */
-public final class EBoolean extends AExpression {
+public class EBoolean extends AExpression {
 
     protected boolean constant;
 
@@ -40,9 +39,8 @@ public EBoolean(Location location, boolean constant) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + constant + "]."));
@@ -50,17 +48,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.actual = boolean.class;
 
-        return output;
-    }
-
-    @Override
-    ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
         constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
-        return constantNode;
+        output.expressionNode = constantNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
index ad4dce45d4d7a..c86966df03f60 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.MemberCallNode;
+import org.elasticsearch.painless.ir.FieldNode;
 import org.elasticsearch.painless.lookup.PainlessClassBinding;
 import org.elasticsearch.painless.lookup.PainlessInstanceBinding;
 import org.elasticsearch.painless.lookup.PainlessMethod;
@@ -32,35 +33,35 @@
 
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * Represents a user-defined call.
  */
-public final class ECallLocal extends AExpression {
+public class ECallLocal extends AExpression {
 
-    private final String name;
-    private final List<AExpression> arguments;
-
-    private FunctionTable.LocalFunction localFunction = null;
-    private PainlessMethod importedMethod = null;
-    private PainlessClassBinding classBinding = null;
-    private int classBindingOffset = 0;
-    private PainlessInstanceBinding instanceBinding = null;
-    private String bindingName = null;
+    protected final String name;
+    protected final List<AExpression> arguments;
 
     public ECallLocal(Location location, String name, List<AExpression> arguments) {
         super(location);
 
         this.name = Objects.requireNonNull(name);
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        FunctionTable.LocalFunction localFunction = null;
+        PainlessMethod importedMethod = null;
+        PainlessClassBinding classBinding = null;
+        int classBindingOffset = 0;
+        PainlessInstanceBinding instanceBinding = null;
+        String bindingName = null;
+
+        Output output = new Output();
 
         localFunction = scriptRoot.getFunctionTable().getFunction(name, arguments.size());
 
@@ -125,43 +126,53 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             typeParameters = new ArrayList<>(classBinding.typeParameters);
             output.actual = classBinding.returnType;
             bindingName = scriptRoot.getNextSyntheticName("class_binding");
-            scriptRoot.getClassNode().addField(new SField(location,
-                    Modifier.PRIVATE, bindingName, classBinding.javaConstructor.getDeclaringClass()));
+
+            FieldNode fieldNode = new FieldNode();
+            fieldNode.setLocation(location);
+            fieldNode.setModifiers(Modifier.PRIVATE);
+            fieldNode.setFieldType(classBinding.javaConstructor.getDeclaringClass());
+            fieldNode.setName(bindingName);
+
+            classNode.addFieldNode(fieldNode);
         } else if (instanceBinding != null) {
             typeParameters = new ArrayList<>(instanceBinding.typeParameters);
             output.actual = instanceBinding.returnType;
             bindingName = scriptRoot.getNextSyntheticName("instance_binding");
-            scriptRoot.getClassNode().addField(new SField(location, Modifier.STATIC | Modifier.PUBLIC,
-                    bindingName, instanceBinding.targetInstance.getClass()));
+
+            FieldNode fieldNode = new FieldNode();
+            fieldNode.setLocation(location);
+            fieldNode.setModifiers(Modifier.PUBLIC | Modifier.STATIC);
+            fieldNode.setFieldType(instanceBinding.targetInstance.getClass());
+            fieldNode.setName(bindingName);
+
+            classNode.addFieldNode(fieldNode);
+
             scriptRoot.addStaticConstant(bindingName, instanceBinding.targetInstance);
         } else {
             throw new IllegalStateException("Illegal tree structure.");
         }
 
+        List<Output> argumentOutputs = new ArrayList<>(arguments.size());
         // if the class binding is using an implicit this reference then the arguments counted must
         // be incremented by 1 as the this reference will not be part of the arguments passed into
         // the class binding call
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            Input expressionInput = new Input();
-            expressionInput.expected = typeParameters.get(argument + classBindingOffset);
-            expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            Input argumentInput = new Input();
+            argumentInput.expected = typeParameters.get(argument + classBindingOffset);
+            argumentInput.internal = true;
+            Output argumentOutput = expression.analyze(classNode, scriptRoot, scope, argumentInput);
+            expression.cast(argumentInput, argumentOutput);
+            argumentOutputs.add(argumentOutput);
         }
 
         output.statement = true;
 
-        return output;
-    }
-
-    @Override
-    MemberCallNode write(ClassNode classNode) {
         MemberCallNode memberCallNode = new MemberCallNode();
 
-        for (AExpression argument : arguments) {
-            memberCallNode.addArgumentNode(argument.cast(argument.write(classNode)));
+        for (int argument = 0; argument < arguments.size(); ++argument) {
+            memberCallNode.addArgumentNode(arguments.get(argument).cast(argumentOutputs.get(argument)));
         }
 
         memberCallNode.setLocation(location);
@@ -173,7 +184,9 @@ MemberCallNode write(ClassNode classNode) {
         memberCallNode.setBindingName(bindingName);
         memberCallNode.setInstanceBinding(instanceBinding);
 
-        return memberCallNode;
+        output.expressionNode = memberCallNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index fa8b25cd861f9..e5f3b11910d18 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -35,11 +35,12 @@
 /**
  * Represents a capturing function reference.  For member functions that require a this reference, ie not static.
  */
-public final class ECapturingFunctionRef extends AExpression implements ILambda {
-    private final String variable;
-    private final String call;
+public class ECapturingFunctionRef extends AExpression implements ILambda {
 
-    private FunctionRef ref;
+    protected final String variable;
+    protected final String call;
+
+    // TODO: #54015
     private Variable captured;
     private String defPointer;
 
@@ -51,9 +52,10 @@ public ECapturingFunctionRef(Location location, String variable, String call) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        FunctionRef ref = null;
+
+        Output output = new Output();
 
         captured = scope.getVariable(location, variable);
         if (input.expected == null) {
@@ -75,11 +77,6 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = input.expected;
         }
 
-        return output;
-    }
-
-    @Override
-    CapturingFuncRefNode write(ClassNode classNode) {
         CapturingFuncRefNode capturingFuncRefNode = new CapturingFuncRefNode();
 
         capturingFuncRefNode.setLocation(location);
@@ -87,9 +84,11 @@ CapturingFuncRefNode write(ClassNode classNode) {
         capturingFuncRefNode.setCapturedName(captured.getName());
         capturingFuncRefNode.setName(call);
         capturingFuncRefNode.setPointer(defPointer);
-        capturingFuncRefNode.setFuncRef(ref);;
+        capturingFuncRefNode.setFuncRef(ref);
 
-        return capturingFuncRefNode;
+        output.expressionNode = capturingFuncRefNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 32555c18625f7..359546e805a40 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -34,13 +34,11 @@
 /**
  * Represents a comparison expression.
  */
-public final class EComp extends AExpression {
+public class EComp extends AExpression {
 
-    private final Operation operation;
-    private AExpression left;
-    private AExpression right;
-
-    private Class<?> promotedType;
+    protected final Operation operation;
+    protected final AExpression left;
+    protected final AExpression right;
 
     public EComp(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
@@ -51,12 +49,16 @@ public EComp(Location location, Operation operation, AExpression left, AExpressi
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Class<?> promotedType;
+
+        Output output = new Output();
+
+        Input leftInput = new Input();
+        Output leftOutput = left.analyze(classNode, scriptRoot, scope, leftInput);
 
-        Output leftOutput = left.analyze(scriptRoot, scope, new Input());
-        Output rightOutput = right.analyze(scriptRoot, scope, new Input());
+        Input rightInput = new Input();
+        Output rightOutput = right.analyze(classNode, scriptRoot, scope, rightInput);
 
         if (operation == Operation.EQ || operation == Operation.EQR || operation == Operation.NE || operation == Operation.NER) {
             promotedType = AnalyzerCaster.promoteEquality(leftOutput.actual, rightOutput.actual);
@@ -74,11 +76,11 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
 
         if (operation != Operation.EQR && operation != Operation.NER && promotedType == def.class) {
-            left.input.expected = leftOutput.actual;
-            right.input.expected = rightOutput.actual;
+            leftInput.expected = leftOutput.actual;
+            rightInput.expected = rightOutput.actual;
         } else {
-            left.input.expected = promotedType;
-            right.input.expected = promotedType;
+            leftInput.expected = promotedType;
+            rightInput.expected = promotedType;
         }
 
         if ((operation == Operation.EQ || operation == Operation.EQR || operation == Operation.NE || operation == Operation.NER)
@@ -86,27 +88,24 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalArgumentException("extraneous comparison of [null] constants"));
         }
 
-        left.cast();
-        right.cast();
+        left.cast(leftInput, leftOutput);
+        right.cast(rightInput, rightOutput);
 
         output.actual = boolean.class;
 
-        return output;
-    }
-
-    @Override
-    ComparisonNode write(ClassNode classNode) {
         ComparisonNode comparisonNode = new ComparisonNode();
 
-        comparisonNode.setLeftNode(left.cast(left.write(classNode)));
-        comparisonNode.setRightNode(right.cast(right.write(classNode)));
+        comparisonNode.setLeftNode(left.cast(leftOutput));
+        comparisonNode.setRightNode(right.cast(rightOutput));
 
         comparisonNode.setLocation(location);
         comparisonNode.setExpressionType(output.actual);
         comparisonNode.setComparisonType(promotedType);
         comparisonNode.setOperation(operation);
 
-        return comparisonNode;
+        output.expressionNode = comparisonNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index d523838ecf66d..2b4440787347b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -32,11 +32,11 @@
 /**
  * Represents a conditional expression.
  */
-public final class EConditional extends AExpression {
+public class EConditional extends AExpression {
 
-    private AExpression condition;
-    private AExpression left;
-    private AExpression right;
+    protected final AExpression condition;
+    protected final AExpression left;
+    protected final AExpression right;
 
     public EConditional(Location location, AExpression condition, AExpression left, AExpression right) {
         super(location);
@@ -47,30 +47,28 @@ public EConditional(Location location, AExpression condition, AExpression left,
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         Input conditionInput = new Input();
         conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
-        condition.cast();
+        Output conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+        condition.cast(conditionInput, conditionOutput);
 
         Input leftInput = new Input();
         leftInput.expected = input.expected;
         leftInput.explicit = input.explicit;
         leftInput.internal = input.internal;
+        Output leftOutput = left.analyze(classNode, scriptRoot, scope, leftInput);
 
         Input rightInput = new Input();
         rightInput.expected = input.expected;
         rightInput.explicit = input.explicit;
         rightInput.internal = input.internal;
+        Output rightOutput = right.analyze(classNode, scriptRoot, scope, rightInput);
 
         output.actual = input.expected;
 
-        Output leftOutput = left.analyze(scriptRoot, scope, leftInput);
-        Output rightOutput = right.analyze(scriptRoot, scope, rightInput);
-
         if (input.expected == null) {
             Class<?> promote = AnalyzerCaster.promoteConditional(leftOutput.actual, rightOutput.actual);
 
@@ -80,29 +78,26 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                         "[" + PainlessLookupUtility.typeToCanonicalTypeName(rightOutput.actual) + "]"));
             }
 
-            left.input.expected = promote;
-            right.input.expected = promote;
+            leftInput.expected = promote;
+            rightInput.expected = promote;
             output.actual = promote;
         }
 
-        left.cast();
-        right.cast();
-
-        return output;
-    }
+        left.cast(leftInput, leftOutput);
+        right.cast(rightInput, rightOutput);
 
-    @Override
-    ConditionalNode write(ClassNode classNode) {
         ConditionalNode conditionalNode = new ConditionalNode();
 
-        conditionalNode.setLeftNode(left.cast(left.write(classNode)));
-        conditionalNode.setRightNode(right.cast(right.write(classNode)));
-        conditionalNode.setConditionNode(condition.cast(condition.write(classNode)));
+        conditionalNode.setLeftNode(left.cast(leftOutput));
+        conditionalNode.setRightNode(right.cast(rightOutput));
+        conditionalNode.setConditionNode(condition.cast(conditionOutput));
 
         conditionalNode.setLocation(location);
         conditionalNode.setExpressionType(output.actual);
 
-        return conditionalNode;
+        output.expressionNode = conditionalNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index 5caa6bca92895..317a9dae3bd3d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -30,7 +30,7 @@
  * Represents a constant inserted into the tree replacing
  * other constants during constant folding.  (Internal only.)
  */
-final class EConstant extends AExpression {
+public class EConstant extends AExpression {
 
     protected Object constant;
 
@@ -41,9 +41,8 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (constant instanceof String) {
             output.actual = String.class;
@@ -69,17 +68,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                     "for constant node"));
         }
 
-        return output;
-    }
-
-    @Override
-    ConstantNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
         constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
-        return constantNode;
+        output.expressionNode = constantNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 405f90b356f1a..35388db216cb5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ConstantNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -31,11 +30,9 @@
 /**
  * Represents a decimal constant.
  */
-public final class EDecimal extends AExpression {
+public class EDecimal extends AExpression {
 
-    private final String value;
-
-    protected Object constant;
+    protected final String value;
 
     public EDecimal(Location location, String value) {
         super(location);
@@ -44,9 +41,10 @@ public EDecimal(Location location, String value) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Object constant;
+
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + value + "]."));
@@ -72,17 +70,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             }
         }
 
-        return output;
-    }
-
-    @Override
-    ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
         constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
-        return constantNode;
+        output.expressionNode = constantNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
index 6c771af47caf4..6ccb1a3f397b8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java
@@ -33,8 +33,9 @@
  * non null. If the first expression is null then it evaluates the second expression and returns it.
  */
 public class EElvis extends AExpression {
-    private AExpression lhs;
-    private AExpression rhs;
+
+    protected AExpression lhs;
+    protected AExpression rhs;
 
     public EElvis(Location location, AExpression lhs, AExpression rhs) {
         super(location);
@@ -44,24 +45,26 @@ public EElvis(Location location, AExpression lhs, AExpression rhs) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.expected != null && input.expected.isPrimitive()) {
             throw createError(new IllegalArgumentException("Elvis operator cannot return primitives"));
         }
+
         Input leftInput = new Input();
         leftInput.expected = input.expected;
         leftInput.explicit = input.explicit;
         leftInput.internal = input.internal;
+        Output leftOutput = lhs.analyze(classNode, scriptRoot, scope, leftInput);
+
         Input rightInput = new Input();
         rightInput.expected = input.expected;
         rightInput.explicit = input.explicit;
         rightInput.internal = input.internal;
+        Output rightOutput = rhs.analyze(classNode, scriptRoot, scope, rightInput);
+
         output.actual = input.expected;
-        Output leftOutput = lhs.analyze(scriptRoot, scope, leftInput);
-        Output rightOutput = rhs.analyze(scriptRoot, scope, rightInput);
 
         if (lhs instanceof ENull) {
             throw createError(new IllegalArgumentException("Extraneous elvis operator. LHS is null."));
@@ -83,28 +86,25 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         if (input.expected == null) {
             Class<?> promote = AnalyzerCaster.promoteConditional(leftOutput.actual, rightOutput.actual);
 
-            lhs.input.expected = promote;
-            rhs.input.expected = promote;
+            leftInput.expected = promote;
+            rightInput.expected = promote;
             output.actual = promote;
         }
 
-        lhs.cast();
-        rhs.cast();
-
-        return output;
-    }
+        lhs.cast(leftInput, leftOutput);
+        rhs.cast(rightInput, rightOutput);
 
-    @Override
-    ElvisNode write(ClassNode classNode) {
         ElvisNode elvisNode = new ElvisNode();
 
-        elvisNode.setLeftNode(lhs.cast(lhs.write(classNode)));
-        elvisNode.setRightNode(rhs.cast(rhs.write(classNode)));
+        elvisNode.setLeftNode(lhs.cast(leftOutput));
+        elvisNode.setRightNode(rhs.cast(rightOutput));
 
         elvisNode.setLocation(location);
         elvisNode.setExpressionType(output.actual);
 
-        return elvisNode;
+        output.expressionNode = elvisNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index b76dcd946b3b3..532e9607fb7a6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -30,10 +29,10 @@
 /**
  * Represents an explicit cast.
  */
-public final class EExplicit extends AExpression {
+public class EExplicit extends AExpression {
 
-    private final String type;
-    private AExpression child;
+    protected final String type;
+    protected final AExpression child;
 
     public EExplicit(Location location, String type, AExpression child) {
         super(location);
@@ -43,9 +42,8 @@ public EExplicit(Location location, String type, AExpression child) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
 
@@ -56,15 +54,12 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         Input childInput = new Input();
         childInput.expected = output.actual;
         childInput.explicit = true;
-        child.analyze(scriptRoot, scope, childInput);
-        child.cast();
+        Output childOutput = child.analyze(classNode, scriptRoot, scope, childInput);
+        child.cast(childInput, childOutput);
 
-        return output;
-    }
+        output.expressionNode = child.cast(childOutput);
 
-    @Override
-    ExpressionNode write(ClassNode classNode) {
-        return child.cast(child.write(classNode));
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 62bb34c215590..a9cb7876fe11a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -33,11 +33,12 @@
 /**
  * Represents a function reference.
  */
-public final class EFunctionRef extends AExpression implements ILambda {
-    private final String type;
-    private final String call;
+public class EFunctionRef extends AExpression implements ILambda {
 
-    private FunctionRef ref;
+    protected final String type;
+    protected final String call;
+
+    // TODO: #54015
     private String defPointer;
 
     public EFunctionRef(Location location, String type, String call) {
@@ -48,9 +49,10 @@ public EFunctionRef(Location location, String type, String call) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        FunctionRef ref;
+
+        Output output = new Output();
 
         if (input.expected == null) {
             ref = null;
@@ -63,18 +65,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = input.expected;
         }
 
-        return output;
-    }
-
-    @Override
-    FuncRefNode write(ClassNode classNode) {
         FuncRefNode funcRefNode = new FuncRefNode();
 
         funcRefNode.setLocation(location);
         funcRefNode.setExpressionType(output.actual);
         funcRefNode.setFuncRef(ref);
 
-        return funcRefNode;
+        output.expressionNode = funcRefNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
index e6ee1a03602b8..9bf0a00e610ca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
@@ -33,13 +33,10 @@
  * <p>
  * Unlike java's, this works for primitive types too.
  */
-public final class EInstanceof extends AExpression {
-    private AExpression expression;
-    private final String type;
+public class EInstanceof extends AExpression {
 
-    private Class<?> resolvedType;
-    private Class<?> expressionType;
-    private boolean primitiveExpression;
+    protected final AExpression expression;
+    protected final String type;
 
     public EInstanceof(Location location, AExpression expression, String type) {
         super(location);
@@ -48,9 +45,12 @@ public EInstanceof(Location location, AExpression expression, String type) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Class<?> resolvedType;
+        Class<?> expressionType;
+        boolean primitiveExpression;
+
+        Output output = new Output();
 
         // ensure the specified type is part of the definition
         Class<?> clazz = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
@@ -64,9 +64,10 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                 PainlessLookupUtility.typeToJavaType(clazz);
 
         // analyze and cast the expression
-        Output expressionOutput = expression.analyze(scriptRoot, scope, new Input());
-        expression.input.expected = expressionOutput.actual;
-        expression.cast();
+        Input expressionInput = new Input();
+        Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+        expressionInput.expected = expressionOutput.actual;
+        expression.cast(expressionInput, expressionOutput);
 
         // record if the expression returns a primitive
         primitiveExpression = expressionOutput.actual.isPrimitive();
@@ -76,14 +77,9 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.actual = boolean.class;
 
-        return output;
-    }
-
-    @Override
-    InstanceofNode write(ClassNode classNode) {
         InstanceofNode instanceofNode = new InstanceofNode();
 
-        instanceofNode.setChildNode(expression.cast(expression.write(classNode)));
+        instanceofNode.setChildNode(expression.cast(expressionOutput));
 
         instanceofNode.setLocation(location);
         instanceofNode.setExpressionType(output.actual);
@@ -91,7 +87,9 @@ InstanceofNode write(ClassNode classNode) {
         instanceofNode.setResolvedType(resolvedType);
         instanceofNode.setPrimitiveResult(primitiveExpression);
 
-        return instanceofNode;
+        output.expressionNode = instanceofNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index a4cb19d048c62..41ffd62df044f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.Scope.LambdaScope;
 import org.elasticsearch.painless.Scope.Variable;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.FunctionNode;
 import org.elasticsearch.painless.ir.LambdaNode;
@@ -59,27 +60,16 @@
  * <br>
  * {@code sort(list, lambda$0(capture))}
  */
-public final class ELambda extends AExpression implements ILambda {
+public class ELambda extends AExpression implements ILambda {
 
-    private final List<String> paramTypeStrs;
-    private final List<String> paramNameStrs;
-    private final List<AStatement> statements;
+    protected final List<String> paramTypeStrs;
+    protected final List<String> paramNameStrs;
+    protected final List<AStatement> statements;
 
-    // captured variables
+    // TODO: #54015
     private List<Variable> captures;
-    // static parent, static lambda
-    private FunctionRef ref;
-    // dynamic parent, deferred until link time
     private String defPointer;
 
-    private String name;
-    private Class<?> returnType;
-    private List<Class<?>> typeParameters;
-    private List<String> parameterNames;
-    private SBlock block;
-    private boolean methodEscape;
-    private int maxLoopCounter;
-
     public ELambda(Location location,
                    List<String> paramTypes, List<String> paramNames,
                    List<AStatement> statements) {
@@ -91,11 +81,18 @@ public ELambda(Location location,
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
-        List<Class<?>> typeParameters = new ArrayList<>();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        String name;
+        Class<?> returnType;
+        List<Class<?>> typeParametersWithCaptures;
+        List<String> parameterNames;
+        SBlock block;
+        int maxLoopCounter;
+        FunctionRef ref;
+
+        Output output = new Output();
+
+        List<Class<?>> typeParameters;
         PainlessMethod interfaceMethod;
         // inspect the target first, set interface method if we know it.
         if (input.expected == null) {
@@ -157,8 +154,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         for (int index = 0; index < typeParameters.size(); ++index) {
             Class<?> type = typeParameters.get(index);
-            String name = paramNameStrs.get(index);
-            lambdaScope.defineVariable(location, type, name, true);
+            String paramName = paramNameStrs.get(index);
+            lambdaScope.defineVariable(location, type, paramName, true);
         }
 
         block = new SBlock(location, statements);
@@ -167,7 +164,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         }
         AStatement.Input blockInput = new AStatement.Input();
         blockInput.lastSource = true;
-        AStatement.Output blockOutput = block.analyze(scriptRoot, lambdaScope, blockInput);
+        AStatement.Output blockOutput = block.analyze(classNode, scriptRoot, lambdaScope, blockInput);
 
         if (blockOutput.methodEscape == false) {
             throw createError(new IllegalArgumentException("not all paths return a value for lambda"));
@@ -177,18 +174,18 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         // prepend capture list to lambda's arguments
         captures = new ArrayList<>(lambdaScope.getCaptures());
-        this.typeParameters = new ArrayList<>(captures.size() + typeParameters.size());
+        typeParametersWithCaptures = new ArrayList<>(captures.size() + typeParameters.size());
         parameterNames = new ArrayList<>(captures.size() + paramNameStrs.size());
         for (Variable var : captures) {
-            this.typeParameters.add(var.getType());
+            typeParametersWithCaptures.add(var.getType());
             parameterNames.add(var.getName());
         }
-        this.typeParameters.addAll(typeParameters);
+        typeParametersWithCaptures.addAll(typeParameters);
         parameterNames.addAll(paramNameStrs);
 
         // desugar lambda body into a synthetic method
         name = scriptRoot.getNextSyntheticName("lambda");
-        scriptRoot.getFunctionTable().addFunction(name, returnType, this.typeParameters, true, true);
+        scriptRoot.getFunctionTable().addFunction(name, returnType, typeParametersWithCaptures, true, true);
 
         // setup method reference to synthetic method
         if (input.expected == null) {
@@ -202,19 +199,12 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = input.expected;
         }
 
-        return output;
-    }
-
-    @Override
-    LambdaNode write(ClassNode classNode) {
         FunctionNode functionNode = new FunctionNode();
-
-        functionNode.setBlockNode(block.write(classNode));
-
+        functionNode.setBlockNode((BlockNode)blockOutput.statementNode);
         functionNode.setLocation(location);
         functionNode.setName(name);
         functionNode.setReturnType(returnType);
-        functionNode.getTypeParameters().addAll(typeParameters);
+        functionNode.getTypeParameters().addAll(typeParametersWithCaptures);
         functionNode.getParameterNames().addAll(parameterNames);
         functionNode.setStatic(true);
         functionNode.setVarArgs(false);
@@ -224,7 +214,6 @@ LambdaNode write(ClassNode classNode) {
         classNode.addFunctionNode(functionNode);
 
         LambdaNode lambdaNode = new LambdaNode();
-
         lambdaNode.setLocation(location);
         lambdaNode.setExpressionType(output.actual);
         lambdaNode.setFuncRef(ref);
@@ -233,7 +222,9 @@ LambdaNode write(ClassNode classNode) {
             lambdaNode.addCapture(capture.getName());
         }
 
-        return lambdaNode;
+        output.expressionNode = lambdaNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index cb49bce8a11c3..c60a9d1bea78b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -29,29 +29,28 @@
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
 
 /**
  * Represents a list initialization shortcut.
  */
-public final class EListInit extends AExpression {
-    private final List<AExpression> values;
+public class EListInit extends AExpression {
 
-    private PainlessConstructor constructor = null;
-    private PainlessMethod method = null;
+    protected final List<AExpression> values;
 
     public EListInit(Location location, List<AExpression> values) {
         super(location);
 
-        this.values = values;
+        this.values = Collections.unmodifiableList(Objects.requireNonNull(values));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from list initializer."));
@@ -59,38 +58,34 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.actual = ArrayList.class;
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
+        PainlessConstructor constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
                     "constructor [" + typeToCanonicalTypeName(output.actual) + ", <init>/0] not found"));
         }
 
-        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "add", 1);
+        PainlessMethod method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "add", 1);
 
         if (method == null) {
             throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(output.actual) + ", add/1] not found"));
         }
 
-        for (int index = 0; index < values.size(); ++index) {
-            AExpression expression = values.get(index);
+        List<Output> valueOutputs = new ArrayList<>(values.size());
 
+        for (AExpression expression : values) {
             Input expressionInput = new Input();
             expressionInput.expected = def.class;
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            valueOutputs.add(expressionOutput);
         }
 
-        return output;
-    }
-
-    @Override
-    ListInitializationNode write(ClassNode classNode) {
         ListInitializationNode listInitializationNode = new ListInitializationNode();
 
-        for (AExpression value : values) {
-            listInitializationNode.addArgumentNode(value.cast(value.write(classNode)));
+        for (int i = 0; i < values.size(); ++i) {
+            listInitializationNode.addArgumentNode(values.get(i).cast(valueOutputs.get(i)));
         }
 
         listInitializationNode.setLocation(location);
@@ -98,7 +93,9 @@ ListInitializationNode write(ClassNode classNode) {
         listInitializationNode.setConstructor(constructor);
         listInitializationNode.setMethod(method);
 
-        return listInitializationNode;
+        output.expressionNode = listInitializationNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index f846a18eb89f9..fbe500c027dab 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -28,32 +28,32 @@
 import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Objects;
 
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
 
 /**
  * Represents a map initialization shortcut.
  */
-public final class EMapInit extends AExpression {
-    private final List<AExpression> keys;
-    private final List<AExpression> values;
+public class EMapInit extends AExpression {
 
-    private PainlessConstructor constructor = null;
-    private PainlessMethod method = null;
+    protected final List<AExpression> keys;
+    protected final List<AExpression> values;
 
     public EMapInit(Location location, List<AExpression> keys, List<AExpression> values) {
         super(location);
 
-        this.keys = keys;
-        this.values = values;
+        this.keys = Collections.unmodifiableList(Objects.requireNonNull(keys));
+        this.values = Collections.unmodifiableList(Objects.requireNonNull(values));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from map initializer."));
@@ -61,14 +61,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.actual = HashMap.class;
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
+        PainlessConstructor constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, 0);
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
                     "constructor [" + typeToCanonicalTypeName(output.actual) + ", <init>/0] not found"));
         }
 
-        method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "put", 2);
+        PainlessMethod method = scriptRoot.getPainlessLookup().lookupPainlessMethod(output.actual, false, "put", 2);
 
         if (method == null) {
             throw createError(new IllegalArgumentException("method [" + typeToCanonicalTypeName(output.actual) + ", put/2] not found"));
@@ -78,37 +78,33 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalStateException("Illegal tree structure."));
         }
 
-        for (int index = 0; index < keys.size(); ++index) {
-            AExpression expression = keys.get(index);
+        List<Output> keyOutputs = new ArrayList<>(keys.size());
+        List<Output> valueOutputs = new ArrayList<>(values.size());
 
+        for (int i = 0; i < keys.size(); ++i) {
+            AExpression expression = keys.get(i);
             Input expressionInput = new Input();
             expressionInput.expected = def.class;
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
-        }
-
-        for (int index = 0; index < values.size(); ++index) {
-            AExpression expression = values.get(index);
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            keyOutputs.add(expressionOutput);
 
-            Input expressionInput = new Input();
+            expression = values.get(i);
+            expressionInput = new Input();
             expressionInput.expected = def.class;
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            valueOutputs.add(expressionOutput);
         }
 
-        return output;
-    }
-
-    @Override
-    MapInitializationNode write(ClassNode classNode) {
         MapInitializationNode mapInitializationNode = new MapInitializationNode();
 
-        for (int index = 0; index < keys.size(); ++index) {
+        for (int i = 0; i < keys.size(); ++i) {
             mapInitializationNode.addArgumentNode(
-                    keys.get(index).cast(keys.get(index).write(classNode)),
-                    values.get(index).cast(values.get(index).write(classNode)));
+                    keys.get(i).cast(keyOutputs.get(i)),
+                    values.get(i).cast(valueOutputs.get(i)));
         }
 
         mapInitializationNode.setLocation(location);
@@ -116,7 +112,9 @@ MapInitializationNode write(ClassNode classNode) {
         mapInitializationNode.setConstructor(constructor);
         mapInitializationNode.setMethod(method);
 
-        return mapInitializationNode;
+        output.expressionNode = mapInitializationNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
index 73603d79e6c89..a4ce05d57cf98 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java
@@ -25,30 +25,31 @@
 import org.elasticsearch.painless.ir.NewArrayNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * Represents an array instantiation.
  */
-public final class ENewArray extends AExpression {
+public class ENewArray extends AExpression {
 
-    private final String type;
-    private final List<AExpression> arguments;
-    private final boolean initialize;
+    protected final String type;
+    protected final List<AExpression> arguments;
+    protected final boolean initialize;
 
     public ENewArray(Location location, String type, List<AExpression> arguments, boolean initialize) {
         super(location);
 
         this.type = Objects.requireNonNull(type);
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
         this.initialize = initialize;
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
              throw createError(new IllegalArgumentException("A newly created array must be read from."));
@@ -60,34 +61,32 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
-        for (int argument = 0; argument < arguments.size(); ++argument) {
-            AExpression expression = arguments.get(argument);
+        List<Output> argumentOutputs = new ArrayList<>();
 
+        for (AExpression expression : arguments) {
             Input expressionInput = new Input();
             expressionInput.expected = initialize ? clazz.getComponentType() : int.class;
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            argumentOutputs.add(expressionOutput);
         }
 
         output.actual = clazz;
 
-        return output;
-    }
-
-    @Override
-    NewArrayNode write(ClassNode classNode) {
         NewArrayNode newArrayNode = new NewArrayNode();
 
-        for (AExpression argument : arguments) {
-            newArrayNode.addArgumentNode(argument.cast(argument.write(classNode)));
+        for (int i = 0; i < arguments.size(); ++ i) {
+            newArrayNode.addArgumentNode(arguments.get(i).cast(argumentOutputs.get(i)));
         }
 
         newArrayNode.setLocation(location);
         newArrayNode.setExpressionType(output.actual);
         newArrayNode.setInitialize(initialize);
 
-        return newArrayNode;
+        output.expressionNode = newArrayNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
index 0a132c12e5adb..16dc2ee729827 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
+import org.elasticsearch.painless.ir.FunctionNode;
 import org.elasticsearch.painless.ir.NewArrayFuncRefNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
@@ -34,11 +35,11 @@
 /**
  * Represents a function reference.
  */
-public final class ENewArrayFunctionRef extends AExpression implements ILambda {
-    private final String type;
+public class ENewArrayFunctionRef extends AExpression implements ILambda {
 
-    private SFunction function;
-    private FunctionRef ref;
+    protected final String type;
+
+    // TODO: #54015
     private String defPointer;
 
     public ENewArrayFunctionRef(Location location, String type) {
@@ -48,19 +49,20 @@ public ENewArrayFunctionRef(Location location, String type) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         SReturn code = new SReturn(location, new ENewArray(location, type, Arrays.asList(new EVariable(location, "size")), false));
-        function = new SFunction(
+        SFunction function = new SFunction(
                 location, type, scriptRoot.getNextSyntheticName("newarray"),
                 Collections.singletonList("int"), Collections.singletonList("size"),
                 new SBlock(location, Collections.singletonList(code)), true, true, true, false);
         function.generateSignature(scriptRoot.getPainlessLookup());
-        function.analyze(scriptRoot);
+        FunctionNode functionNode = function.writeFunction(classNode, scriptRoot);
         scriptRoot.getFunctionTable().addFunction(function.name, function.returnType, function.typeParameters, true, true);
 
+        FunctionRef ref;
+
         if (input.expected == null) {
             ref = null;
             output.actual = String.class;
@@ -72,12 +74,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = input.expected;
         }
 
-        return output;
-    }
-
-    @Override
-    NewArrayFuncRefNode write(ClassNode classNode) {
-        classNode.addFunctionNode(function.write(classNode));
+        classNode.addFunctionNode(functionNode);
 
         NewArrayFuncRefNode newArrayFuncRefNode = new NewArrayFuncRefNode();
 
@@ -85,7 +82,9 @@ NewArrayFuncRefNode write(ClassNode classNode) {
         newArrayFuncRefNode.setExpressionType(output.actual);
         newArrayFuncRefNode.setFuncRef(ref);
 
-        return newArrayFuncRefNode;
+        output.expressionNode = newArrayFuncRefNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index dda96482a472d..1659910986173 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -28,6 +28,8 @@
 import org.elasticsearch.painless.spi.annotation.NonDeterministicAnnotation;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -36,24 +38,21 @@
 /**
  * Represents and object instantiation.
  */
-public final class ENewObj extends AExpression {
+public class ENewObj extends AExpression {
 
-    private final String type;
-    private final List<AExpression> arguments;
-
-    private PainlessConstructor constructor;
+    protected final String type;
+    protected final List<AExpression> arguments;
 
     public ENewObj(Location location, String type, List<AExpression> arguments) {
         super(location);
 
         this.type = Objects.requireNonNull(type);
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
 
@@ -61,7 +60,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
-        constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, arguments.size());
+        PainlessConstructor constructor = scriptRoot.getPainlessLookup().lookupPainlessConstructor(output.actual, arguments.size());
 
         if (constructor == null) {
             throw createError(new IllegalArgumentException(
@@ -79,27 +78,25 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                     "expected [" + constructor.typeParameters.size() + "] arguments, but found [" + arguments.size() + "]."));
         }
 
-        for (int argument = 0; argument < arguments.size(); ++argument) {
-            AExpression expression = arguments.get(argument);
+        List<Output> argumentOutputs = new ArrayList<>();
+
+        for (int i = 0; i < arguments.size(); ++i) {
+            AExpression expression = arguments.get(i);
 
             Input expressionInput = new Input();
-            expressionInput.expected = types[argument];
+            expressionInput.expected = types[i];
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            argumentOutputs.add(expressionOutput);
         }
 
         output.statement = true;
 
-        return output;
-    }
-
-    @Override
-    NewObjectNode write(ClassNode classNode) {
         NewObjectNode newObjectNode = new NewObjectNode();
 
-        for (AExpression argument : arguments) {
-            newObjectNode.addArgumentNode(argument.cast(argument.write(classNode)));
+        for (int i = 0; i < arguments.size(); ++ i) {
+            newObjectNode.addArgumentNode(arguments.get(i).cast(argumentOutputs.get(i)));
         }
 
         newObjectNode.setLocation(location);
@@ -107,7 +104,9 @@ NewObjectNode write(ClassNode classNode) {
         newObjectNode.setRead(input.read);
         newObjectNode.setConstructor(constructor);
 
-        return newObjectNode;
+        output.expressionNode = newObjectNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 841bd09586bed..1b61a05a68501 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -29,16 +29,15 @@
 /**
  * Represents a null constant.
  */
-public final class ENull extends AExpression {
+public class ENull extends AExpression {
 
     public ENull(Location location) {
         super(location);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from null constant."));
@@ -55,17 +54,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.actual = Object.class;
         }
 
-        return output;
-    }
-
-    @Override
-    NullNode write(ClassNode classNode) {
         NullNode nullNode = new NullNode();
 
         nullNode.setLocation(location);
         nullNode.setExpressionType(output.actual);
 
-        return nullNode;
+        output.expressionNode = nullNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index f7cbe681b639d..1e548252f1a74 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ConstantNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -31,12 +30,10 @@
 /**
  * Represents a non-decimal numeric constant.
  */
-public final class ENumeric extends AExpression {
+public class ENumeric extends AExpression {
 
-    private final String value;
-    private int radix;
-
-    protected Object constant;
+    protected final String value;
+    protected final int radix;
 
     public ENumeric(Location location, String value, int radix) {
         super(location);
@@ -46,9 +43,10 @@ public ENumeric(Location location, String value, int radix) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Object constant;
+
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + value + "]."));
@@ -114,17 +112,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             }
         }
 
-        return output;
-    }
-
-    @Override
-    ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
         constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
-        return constantNode;
+        output.expressionNode = constantNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
index 3847397407dd8..23e92f018c371 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java
@@ -42,11 +42,10 @@
 /**
  * Represents a regex constant. All regexes are constants.
  */
-public final class ERegex extends AExpression {
+public class ERegex extends AExpression {
 
-    private final String pattern;
-    private final int flags;
-    private String name;
+    protected final String pattern;
+    protected final int flags;
 
     public ERegex(Location location, String pattern, String flagsString) {
         super(location);
@@ -63,10 +62,8 @@ public ERegex(Location location, String pattern, String flagsString) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (scriptRoot.getCompilerSettings().areRegexesEnabled() == false) {
             throw createError(new IllegalStateException("Regexes are disabled. Set [script.painless.regex.enabled] to [true] "
@@ -85,14 +82,9 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                     new IllegalArgumentException("Error compiling regex: " + e.getDescription()));
         }
 
-        name = scriptRoot.getNextSyntheticName("regex");
+        String name = scriptRoot.getNextSyntheticName("regex");
         output.actual = Pattern.class;
 
-        return output;
-    }
-
-    @Override
-    MemberFieldLoadNode write(ClassNode classNode) {
         FieldNode fieldNode = new FieldNode();
         fieldNode.setLocation(location);
         fieldNode.setModifiers(Modifier.FINAL | Modifier.STATIC | Modifier.PRIVATE);
@@ -169,7 +161,9 @@ MemberFieldLoadNode write(ClassNode classNode) {
         memberFieldLoadNode.setName(name);
         memberFieldLoadNode.setStatic(true);
 
-        return memberFieldLoadNode;
+        output.expressionNode = memberFieldLoadNode;
+
+        return output;
     }
 
     private int flagForChar(char c) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
index 0706e321e5312..b45ba741ae1a3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java
@@ -30,9 +30,9 @@
 /**
  * Represents a static type target.
  */
-public final class EStatic extends AExpression {
+public class EStatic extends AExpression {
 
-    private final String type;
+    protected final String type;
 
     public EStatic(Location location, String type) {
         super(location);
@@ -41,9 +41,8 @@ public EStatic(Location location, String type) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         output.actual = scriptRoot.getPainlessLookup().canonicalTypeNameToType(type);
 
@@ -51,17 +50,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
 
-        return output;
-    }
-
-    @Override
-    StaticNode write(ClassNode classNode) {
         StaticNode staticNode = new StaticNode();
 
         staticNode.setLocation(location);
         staticNode.setExpressionType(output.actual);
 
-        return staticNode;
+        output.expressionNode = staticNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
index 5cc5b67fe511b..77b9fb857bc08 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ConstantNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -31,7 +30,7 @@
 /**
  * Represents a string constant.
  */
-public final class EString extends AExpression {
+public class EString extends AExpression {
 
     protected String constant;
 
@@ -42,9 +41,8 @@ public EString(Location location, String string) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.read == false) {
             throw createError(new IllegalArgumentException("Must read from constant [" + constant + "]."));
@@ -52,17 +50,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.actual = String.class;
 
-        return output;
-    }
-
-    @Override
-    ExpressionNode write(ClassNode classNode) {
         ConstantNode constantNode = new ConstantNode();
         constantNode.setLocation(location);
         constantNode.setExpressionType(output.actual);
         constantNode.setConstant(constant);
 
-        return constantNode;
+        output.expressionNode = constantNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index e498ceb3578c8..cafcda3b7dd2a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.UnaryMathNode;
-import org.elasticsearch.painless.ir.UnaryNode;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -35,13 +34,10 @@
 /**
  * Represents a unary math expression.
  */
-public final class EUnary extends AExpression {
+public class EUnary extends AExpression {
 
-    private final Operation operation;
-    private AExpression child;
-
-    private Class<?> promote;
-    private boolean originallyExplicit = false; // record whether there was originally an explicit cast
+    protected final Operation operation;
+    protected final AExpression child;
 
     public EUnary(Location location, Operation operation, AExpression child) {
         super(location);
@@ -51,21 +47,24 @@ public EUnary(Location location, Operation operation, AExpression child) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        Class<?> promote = null;
+        boolean originallyExplicit = input.explicit;
 
-        originallyExplicit = input.explicit;
+        Input childInput = new Input();
+        Output childOutput;
 
         if (operation == Operation.NOT) {
-            Input childInput = new Input();
+
             childInput.expected = boolean.class;
-            child.analyze(scriptRoot, scope, childInput);
-            child.cast();
+            childOutput = child.analyze(classNode, scriptRoot, scope, childInput);
+            child.cast(childInput, childOutput);
 
             output.actual = boolean.class;
         } else if (operation == Operation.BWNOT || operation == Operation.ADD || operation == Operation.SUB) {
-            Output childOutput = child.analyze(scriptRoot, scope, new Input());
+            childOutput = child.analyze(classNode, scriptRoot, scope, new Input());
 
             promote = AnalyzerCaster.promoteNumeric(childOutput.actual, operation != Operation.BWNOT);
 
@@ -75,8 +74,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                         "[" + PainlessLookupUtility.typeToCanonicalTypeName(childOutput.actual) + "]"));
             }
 
-            child.input.expected = promote;
-            child.cast();
+            childInput.expected = promote;
+            child.cast(childInput, childOutput);
 
             if (promote == def.class && input.expected != null) {
                 output.actual = input.expected;
@@ -87,14 +86,9 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             throw createError(new IllegalStateException("unexpected unary operation [" + operation.name + "]"));
         }
 
-        return output;
-    }
-
-    @Override
-    UnaryNode write(ClassNode classNode) {
         UnaryMathNode unaryMathNode = new UnaryMathNode();
 
-        unaryMathNode.setChildNode(child.cast(child.write(classNode)));
+        unaryMathNode.setChildNode(child.cast(childOutput));
 
         unaryMathNode.setLocation(location);
         unaryMathNode.setExpressionType(output.actual);
@@ -102,7 +96,9 @@ UnaryNode write(ClassNode classNode) {
         unaryMathNode.setOperation(operation);
         unaryMathNode.setOriginallExplicit(originallyExplicit);
 
-        return unaryMathNode;
+        output.expressionNode = unaryMathNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
index 2f1a4d6f09a72..0c345dfa46735 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EVariable.java
@@ -31,9 +31,9 @@
 /**
  * Represents a variable load/store.
  */
-public final class EVariable extends AStoreable {
+public class EVariable extends AStoreable {
 
-    private final String name;
+    protected final String name;
 
     public EVariable(Location location, String name) {
         super(location);
@@ -42,20 +42,19 @@ public EVariable(Location location, String name) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
         AStoreable.Input storeableInput = new AStoreable.Input();
         storeableInput.read = input.read;
         storeableInput.expected = input.expected;
         storeableInput.explicit = input.explicit;
         storeableInput.internal = input.internal;
 
-        return analyze(scriptRoot, scope, storeableInput);
+        return analyze(classNode, scriptRoot, scope, storeableInput);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         Variable variable = scope.getVariable(location, name);
 
@@ -65,18 +64,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
 
         output.actual = variable.getType();
 
-        return output;
-    }
-
-    @Override
-    VariableNode write(ClassNode classNode) {
         VariableNode variableNode = new VariableNode();
 
         variableNode.setLocation(location);
         variableNode.setExpressionType(output.actual);
         variableNode.setName(name);
 
-        return variableNode;
+        output.expressionNode = variableNode;
+
+        return output;
     }
 
     @Override
@@ -84,11 +80,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalArgumentException("Illegal tree structure.");
-    }
-
     @Override
     public String toString() {
         return singleLineToString(name);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
index 684b015aa835e..664a9e1c0048a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
@@ -34,11 +34,12 @@
 /**
  * Represents an array load/store and defers to a child subnode.
  */
-public final class PBrace extends AStoreable {
+public class PBrace extends AStoreable {
 
-    private AExpression index;
+    protected final AExpression index;
 
-    private AStoreable sub = null;
+    // TODO: #54015
+    private boolean isDefOptimized = false;
 
     public PBrace(Location location, AExpression prefix, AExpression index) {
         super(location, prefix);
@@ -47,24 +48,26 @@ public PBrace(Location location, AExpression prefix, AExpression index) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
         AStoreable.Input storeableInput = new AStoreable.Input();
         storeableInput.read = input.read;
         storeableInput.expected = input.expected;
         storeableInput.explicit = input.explicit;
         storeableInput.internal = input.internal;
 
-        return analyze(scriptRoot, scope, storeableInput);
+        return analyze(classNode, scriptRoot, scope, storeableInput);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
-        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
-        prefix.input.expected = prefixOutput.actual;
-        prefix.cast();
+        Input prefixInput = new Input();
+        Output prefixOutput = prefix.analyze(classNode, scriptRoot, scope, prefixInput);
+        prefixInput.expected = prefixOutput.actual;
+        prefix.cast(prefixInput, prefixOutput);
+
+        AStoreable sub;
 
         if (prefixOutput.actual.isArray()) {
             sub = new PSubBrace(location, prefixOutput.actual, index);
@@ -79,39 +82,32 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
                     "[" + PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual) + "]."));
         }
 
+        isDefOptimized = sub.isDefOptimized();
+
         Input subInput = new Input();
         subInput.write = input.write;
         subInput.read = input.read;
         subInput.expected = input.expected;
         subInput.explicit = input.explicit;
-        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
+        Output subOutput = sub.analyze(classNode, scriptRoot, scope, subInput);
         output.actual = subOutput.actual;
 
-        return output;
-    }
-
-    @Override
-    BraceNode write(ClassNode classNode) {
         BraceNode braceNode = new BraceNode();
 
-        braceNode.setLeftNode(prefix.cast(prefix.write(classNode)));
-        braceNode.setRightNode(sub.write(classNode));
+        braceNode.setLeftNode(prefix.cast(prefixOutput));
+        braceNode.setRightNode(subOutput.expressionNode);
 
         braceNode.setLocation(location);
         braceNode.setExpressionType(output.actual);
 
-        return braceNode;
-    }
+        output.expressionNode = braceNode;
 
-    @Override
-    boolean isDefOptimized() {
-        return sub.isDefOptimized();
+        return output;
     }
 
     @Override
-    void updateActual(Class<?> actual) {
-        sub.updateActual(actual);
-        this.output.actual = actual;
+    boolean isDefOptimized() {
+        return isDefOptimized;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index 4a0254ef96253..036c04f3878b5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.painless.spi.annotation.NonDeterministicAnnotation;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -36,30 +37,30 @@
 /**
  * Represents a method call and defers to a child subnode.
  */
-public final class PCallInvoke extends AExpression {
+public class PCallInvoke extends AExpression {
 
-    private final String name;
-    private final boolean nullSafe;
-    private final List<AExpression> arguments;
-
-    private AExpression sub = null;
+    protected final String name;
+    protected final boolean nullSafe;
+    protected final List<AExpression> arguments;
 
     public PCallInvoke(Location location, AExpression prefix, String name, boolean nullSafe, List<AExpression> arguments) {
         super(location, prefix);
 
         this.name = Objects.requireNonNull(name);
         this.nullSafe = nullSafe;
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        Input prefixInput = new Input();
+        Output prefixOutput = prefix.analyze(classNode, scriptRoot, scope, prefixInput);
+        prefixInput.expected = prefixOutput.actual;
+        prefix.cast(prefixInput, prefixOutput);
 
-        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
-        prefix.input.expected = prefixOutput.actual;
-        prefix.cast();
+        AExpression sub;
 
         if (prefixOutput.actual == def.class) {
             sub = new PSubDefCall(location, name, arguments);
@@ -84,25 +85,22 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         Input subInput = new Input();
         subInput.expected = input.expected;
         subInput.explicit = input.explicit;
-        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
+        Output subOutput = sub.analyze(classNode, scriptRoot, scope, subInput);
         output.actual = subOutput.actual;
 
         output.statement = true;
 
-        return output;
-    }
-
-    @Override
-    CallNode write(ClassNode classNode) {
         CallNode callNode = new CallNode();
 
-        callNode.setLeftNode(prefix.cast(prefix.write(classNode)));
-        callNode.setRightNode(sub.write(classNode));
+        callNode.setLeftNode(prefix.cast(prefixOutput));
+        callNode.setRightNode(subOutput.expressionNode);
 
         callNode.setLocation(location);
         callNode.setExpressionType(output.actual);
 
-        return callNode;
+        output.expressionNode = callNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index 2380aa536edde..9751786e1515b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -38,12 +38,13 @@
 /**
  * Represents a field load/store and defers to a child subnode.
  */
-public final class PField extends AStoreable {
+public class PField extends AStoreable {
 
-    private final boolean nullSafe;
-    private final String value;
+    protected final boolean nullSafe;
+    protected final String value;
 
-    private AStoreable sub = null;
+    // TODO: #54015
+    private boolean isDefOptimized;
 
     public PField(Location location, AExpression prefix, boolean nullSafe, String value) {
         super(location, prefix);
@@ -53,24 +54,26 @@ public PField(Location location, AExpression prefix, boolean nullSafe, String va
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AExpression.Input input) {
         AStoreable.Input storeableInput = new AStoreable.Input();
         storeableInput.read = input.read;
         storeableInput.expected = input.expected;
         storeableInput.explicit = input.explicit;
         storeableInput.internal = input.internal;
 
-        return analyze(scriptRoot, scope, storeableInput);
+        return analyze(classNode, scriptRoot, scope, storeableInput);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
-        Output prefixOutput = prefix.analyze(scriptRoot, scope, new Input());
-        prefix.input.expected = prefixOutput.actual;
-        prefix.cast();
+        Input prefixInput = new Input();
+        Output prefixOutput = prefix.analyze(classNode, scriptRoot, scope, prefixInput);
+        prefixInput.expected = prefixOutput.actual;
+        prefix.cast(prefixInput, prefixOutput);
+
+        AStoreable sub = null;
 
         if (prefixOutput.actual.isArray()) {
             sub = new PSubArrayLength(location, PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual), value);
@@ -99,7 +102,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
                             location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefixOutput.actual), getter, setter);
                 } else {
                     EConstant index = new EConstant(location, value);
-                    index.analyze(scriptRoot, scope, new Input());
+                    index.analyze(classNode, scriptRoot, scope, new Input());
 
                     if (Map.class.isAssignableFrom(prefixOutput.actual)) {
                         sub = new PSubMapShortcut(location, prefixOutput.actual, index);
@@ -119,6 +122,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             }
         }
 
+        isDefOptimized = sub.isDefOptimized();
+
         if (nullSafe) {
             sub = new PSubNullSafeField(location, sub);
         }
@@ -128,34 +133,25 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
         subInput.read = input.read;
         subInput.expected = input.expected;
         subInput.explicit = input.explicit;
-        Output subOutput = sub.analyze(scriptRoot, scope, subInput);
+        Output subOutput = sub.analyze(classNode, scriptRoot, scope, subInput);
         output.actual = subOutput.actual;
 
-        return output;
-    }
-
-    @Override
-    DotNode write(ClassNode classNode) {
         DotNode dotNode = new DotNode();
 
-        dotNode.setLeftNode(prefix.cast(prefix.write(classNode)));
-        dotNode.setRightNode(sub.write(classNode));
+        dotNode.setLeftNode(prefix.cast(prefixOutput));
+        dotNode.setRightNode(subOutput.expressionNode);
 
         dotNode.setLocation(location);
         dotNode.setExpressionType(output.actual);
 
-        return dotNode;
-    }
+        output.expressionNode = dotNode;
 
-    @Override
-    boolean isDefOptimized() {
-        return sub.isDefOptimized();
+        return output;
     }
 
     @Override
-    void updateActual(Class<?> actual) {
-        sub.updateActual(actual);
-        this.output.actual = actual;
+    boolean isDefOptimized() {
+        return isDefOptimized;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
index 43cdba046bd54..aaefa0717d3ba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubArrayLength.java
@@ -30,10 +30,10 @@
 /**
  * Represents an array length field load.
  */
-final class PSubArrayLength extends AStoreable {
+public class PSubArrayLength extends AStoreable {
 
-    private final String type;
-    private final String value;
+    protected final String type;
+    protected final String value;
 
     PSubArrayLength(Location location, String type, String value) {
         super(location);
@@ -43,9 +43,8 @@ final class PSubArrayLength extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         if ("length".equals(value)) {
             if (input.write) {
@@ -57,27 +56,19 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Field [" + value + "] does not exist for type [" + type + "]."));
         }
 
-        return output;
-    }
-
-    @Override
-    DotSubArrayLengthNode write(ClassNode classNode) {
         DotSubArrayLengthNode dotSubArrayLengthNode = new DotSubArrayLengthNode();
 
         dotSubArrayLengthNode.setLocation(location);
         dotSubArrayLengthNode.setExpressionType(output.actual);
 
-        return dotSubArrayLengthNode;
-    }
+        output.expressionNode = dotSubArrayLengthNode;
 
-    @Override
-    boolean isDefOptimized() {
-        throw new IllegalStateException("Illegal tree structure.");
+        return output;
     }
 
     @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalStateException("Illegal tree structure.");
+    boolean isDefOptimized() {
+        return false;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
index 0303ce051f7f5..2a285d49acbe2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubBrace.java
@@ -30,10 +30,10 @@
 /**
  * Represents an array load/store.
  */
-final class PSubBrace extends AStoreable {
+public class PSubBrace extends AStoreable {
 
-    private final Class<?> clazz;
-    private AExpression index;
+    protected final Class<?> clazz;
+    protected final AExpression index;
 
     PSubBrace(Location location, Class<?> clazz, AExpression index) {
         super(location);
@@ -43,29 +43,26 @@ final class PSubBrace extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         Input indexInput = new Input();
         indexInput.expected = int.class;
-        index.analyze(scriptRoot, scope, indexInput);
-        index.cast();
+        Output indexOutput = index.analyze(classNode, scriptRoot, scope, indexInput);
+        index.cast(indexInput, indexOutput);
 
         output.actual = clazz.getComponentType();
 
-        return output;
-    }
-
-    BraceSubNode write(ClassNode classNode) {
         BraceSubNode braceSubNode = new BraceSubNode();
 
-        braceSubNode.setChildNode(index.cast(index.write(classNode)));
+        braceSubNode.setChildNode(index.cast(indexOutput));
 
         braceSubNode.setLocation(location);
         braceSubNode.setExpressionType(output.actual);
 
-        return braceSubNode;
+        output.expressionNode = braceSubNode;
+
+        return output;
     }
 
     @Override
@@ -73,11 +70,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw createError(new IllegalStateException("Illegal tree structure."));
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, index);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
index 7e509682f7d47..721e76cd3e500 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
@@ -26,30 +26,33 @@
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * Represents a method call.
  */
-final class PSubCallInvoke extends AExpression {
+public class PSubCallInvoke extends AExpression {
 
-    private final PainlessMethod method;
-    private final Class<?> box;
-    private final List<AExpression> arguments;
+    protected final PainlessMethod method;
+    protected final Class<?> box;
+    protected final List<AExpression> arguments;
 
     PSubCallInvoke(Location location, PainlessMethod method, Class<?> box, List<AExpression> arguments) {
         super(location);
 
         this.method = Objects.requireNonNull(method);
         this.box = box;
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        List<Output> argumentOutputs = new ArrayList<>();
 
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
@@ -57,22 +60,18 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             Input expressionInput = new Input();
             expressionInput.expected = method.typeParameters.get(argument);
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
+            argumentOutputs.add(expressionOutput);
         }
 
         output.statement = true;
         output.actual = method.returnType;
 
-        return output;
-    }
-
-    @Override
-    CallSubNode write(ClassNode classNode) {
         CallSubNode callSubNode = new CallSubNode();
 
-        for (AExpression argument : arguments) {
-            callSubNode.addArgumentNode(argument.cast(argument.write(classNode)));
+        for (int argument = 0; argument < arguments.size(); ++ argument) {
+            callSubNode.addArgumentNode(arguments.get(argument).cast(argumentOutputs.get(argument)));
         }
 
         callSubNode.setLocation(location);
@@ -80,7 +79,9 @@ CallSubNode write(ClassNode classNode) {
         callSubNode.setMethod(method);
         callSubNode .setBox(box);
 
-        return callSubNode;
+        output.expressionNode = callSubNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
index 060102ae8593f..00f09a493c0e9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
@@ -32,8 +32,9 @@
 /**
  * Represents an array load/store or shortcut on a def type.  (Internal only.)
  */
-final class PSubDefArray extends AStoreable {
-    private AExpression index;
+public class PSubDefArray extends AStoreable {
+
+    protected AExpression index;
 
     PSubDefArray(Location location, AExpression index) {
         super(location);
@@ -42,30 +43,27 @@ final class PSubDefArray extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
-        Output indexOutput = index.analyze(scriptRoot, scope, new Input());
-        index.input.expected = indexOutput.actual;
-        index.cast();
+        Input indexInput = new Input();
+        Output indexOutput = index.analyze(classNode, scriptRoot, scope, indexInput);
+        indexInput.expected = indexOutput.actual;
+        index.cast(indexInput, indexOutput);
 
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
         output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
 
-        return output;
-    }
-
-    @Override
-    BraceSubDefNode write(ClassNode classNode) {
         BraceSubDefNode braceSubDefNode = new BraceSubDefNode();
 
-        braceSubDefNode.setChildNode(index.cast(index.write(classNode)));
+        braceSubDefNode.setChildNode(index.cast(indexOutput));
 
         braceSubDefNode.setLocation(location);
         braceSubDefNode.setExpressionType(output.actual);
 
-        return braceSubDefNode;
+        output.expressionNode = braceSubDefNode;
+
+        return output;
     }
 
     @Override
@@ -73,11 +71,6 @@ boolean isDefOptimized() {
         return true;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        this.output.actual = actual;
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, index);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
index f810f1a634b6e..8339d161f2a84 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
@@ -28,51 +28,55 @@
 
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 /**
  * Represents a method call made on a def type. (Internal only.)
  */
-final class PSubDefCall extends AExpression {
+public class PSubDefCall extends AExpression {
 
-    private final String name;
-    private final List<AExpression> arguments;
-
-    private final StringBuilder recipe = new StringBuilder();
-    private final List<String> pointers = new ArrayList<>();
-    private final List<Class<?>> parameterTypes = new ArrayList<>();
+    protected final String name;
+    protected final List<AExpression> arguments;
 
     PSubDefCall(Location location, String name, List<AExpression> arguments) {
         super(location);
 
         this.name = Objects.requireNonNull(name);
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(Objects.requireNonNull(arguments));
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        StringBuilder recipe = new StringBuilder();
+        List<String> pointers = new ArrayList<>();
+        List<Class<?>> parameterTypes = new ArrayList<>();
 
         parameterTypes.add(Object.class);
         int totalCaptures = 0;
 
+        List<Output> argumentOutputs = new ArrayList<>(arguments.size());
+
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
             Input expressionInput = new Input();
             expressionInput.internal = true;
-            Output expressionOutput = expression.analyze(scriptRoot, scope, expressionInput);
+            Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            argumentOutputs.add(expressionOutput);
 
             if (expressionOutput.actual == void.class) {
                 throw createError(new IllegalArgumentException("Argument(s) cannot be of [void] type when calling method [" + name + "]."));
             }
 
-            expression.input.expected = expressionOutput.actual;
-            expression.cast();
+            expressionInput.expected = expressionOutput.actual;
+            expression.cast(expressionInput, expressionOutput);
             parameterTypes.add(expressionOutput.actual);
 
+            // TODO: #54015
             if (expression instanceof ILambda) {
                 ILambda lambda = (ILambda) expression;
                 pointers.add(lambda.getPointer());
@@ -87,15 +91,10 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
         output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
 
-        return output;
-    }
-
-    @Override
-    CallSubDefNode write(ClassNode classNode) {
         CallSubDefNode callSubDefNode = new CallSubDefNode();
 
-        for (AExpression argument : arguments) {
-            callSubDefNode.addArgumentNode(argument.cast(argument.write(classNode)));
+        for (int argument = 0; argument < arguments.size(); ++ argument) {
+            callSubDefNode.addArgumentNode(arguments.get(argument).cast(argumentOutputs.get(argument)));
         }
 
         callSubDefNode.setLocation(location);
@@ -105,7 +104,9 @@ CallSubDefNode write(ClassNode classNode) {
         callSubDefNode.getPointers().addAll(pointers);
         callSubDefNode.getTypeParameters().addAll(parameterTypes);
 
-        return callSubDefNode;
+        output.expressionNode = callSubDefNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
index 75bfa5c5755cc..1cb8b2a0119e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
@@ -32,9 +32,9 @@
 /**
  * Represents a field load/store or shortcut on a def type.  (Internal only.)
  */
-final class PSubDefField extends AStoreable {
+public class PSubDefField extends AStoreable {
 
-    private final String value;
+    protected final String value;
 
     PSubDefField(Location location, String value) {
         super(location);
@@ -43,25 +43,21 @@ final class PSubDefField extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         // TODO: remove ZonedDateTime exception when JodaCompatibleDateTime is removed
         output.actual = input.expected == null || input.expected == ZonedDateTime.class || input.explicit ? def.class : input.expected;
 
-        return output;
-    }
-
-    @Override
-    DotSubDefNode write(ClassNode classNode) {
         DotSubDefNode dotSubDefNode = new DotSubDefNode();
 
         dotSubDefNode.setLocation(location);
         dotSubDefNode.setExpressionType(output.actual);
         dotSubDefNode.setValue(value);
 
-        return dotSubDefNode;
+        output.expressionNode = dotSubDefNode;
+
+        return output;
     }
 
     @Override
@@ -69,11 +65,6 @@ boolean isDefOptimized() {
         return true;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        this.output.actual = actual;
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, value);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
index c9a438571bc36..26d2858f54afb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
@@ -33,9 +33,9 @@
 /**
  * Represents a field load/store.
  */
-final class PSubField extends AStoreable {
+public class PSubField extends AStoreable {
 
-    private final PainlessField field;
+    protected final PainlessField field;
 
     PSubField(Location location, PainlessField field) {
         super(location);
@@ -44,9 +44,8 @@ final class PSubField extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
          if (input.write && Modifier.isFinal(field.javaField.getModifiers())) {
              throw createError(new IllegalArgumentException("Cannot write to read-only field [" + field.javaField.getName() + "] " +
@@ -55,18 +54,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
 
          output.actual = field.typeParameter;
 
-         return output;
-    }
-
-    @Override
-    DotSubNode write(ClassNode classNode) {
         DotSubNode dotSubNode = new DotSubNode();
 
         dotSubNode.setLocation(location);
         dotSubNode.setExpressionType(output.actual);
         dotSubNode.setField(field);
 
-        return dotSubNode;
+        output.expressionNode = dotSubNode;
+
+        return output;
     }
 
     @Override
@@ -74,11 +70,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalArgumentException("Illegal tree structure.");
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, field.javaField.getName());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
index 4dd3b46ceaa10..d23039453b3f8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
@@ -32,13 +32,10 @@
 /**
  * Represents a list load/store shortcut.  (Internal only.)
  */
-final class PSubListShortcut extends AStoreable {
+public class PSubListShortcut extends AStoreable {
 
-    private final Class<?> targetClass;
-    private AExpression index;
-
-    private PainlessMethod getter;
-    private PainlessMethod setter;
+    protected final Class<?> targetClass;
+    protected final AExpression index;
 
     PSubListShortcut(Location location, Class<?> targetClass, AExpression index) {
         super(location);
@@ -48,14 +45,13 @@ final class PSubListShortcut extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
 
-        getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
-        setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "set", 2);
+        PainlessMethod getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
+        PainlessMethod setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "set", 2);
 
         if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1 ||
             getter.typeParameters.get(0) != int.class)) {
@@ -71,32 +67,31 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
+        Output indexOutput = new Output();
+
         if ((input.read || input.write) && (input.read == false || getter != null) && (input.write == false || setter != null)) {
             Input indexInput = new Input();
             indexInput.expected = int.class;
-            index.analyze(scriptRoot, scope, indexInput);
-            index.cast();
+            indexOutput = index.analyze(classNode, scriptRoot, scope, indexInput);
+            index.cast(indexInput, indexOutput);
 
             output.actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal list shortcut for type [" + canonicalClassName + "]."));
         }
 
-        return output;
-    }
-
-    @Override
-    ListSubShortcutNode write(ClassNode classNode) {
         ListSubShortcutNode listSubShortcutNode = new ListSubShortcutNode();
 
-        listSubShortcutNode.setChildNode(index.cast(index.write(classNode)));
+        listSubShortcutNode.setChildNode(index.cast(indexOutput));
 
         listSubShortcutNode.setLocation(location);
         listSubShortcutNode.setExpressionType(output.actual);
         listSubShortcutNode.setGetter(getter);
         listSubShortcutNode.setSetter(setter);
 
-        return listSubShortcutNode;
+        output.expressionNode = listSubShortcutNode;
+
+        return output;
     }
 
     @Override
@@ -104,11 +99,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalArgumentException("Illegal tree structure.");
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, index);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
index 707609e5a4947..4105bcec8b261 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
@@ -32,13 +32,10 @@
 /**
  * Represents a map load/store shortcut. (Internal only.)
  */
-final class PSubMapShortcut extends AStoreable {
+public class PSubMapShortcut extends AStoreable {
 
-    private final Class<?> targetClass;
-    private AExpression index;
-
-    private PainlessMethod getter;
-    private PainlessMethod setter;
+    protected final Class<?> targetClass;
+    protected final AExpression index;
 
     PSubMapShortcut(Location location, Class<?> targetClass, AExpression index) {
         super(location);
@@ -48,14 +45,13 @@ final class PSubMapShortcut extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
 
-        getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
-        setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "put", 2);
+        PainlessMethod getter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "get", 1);
+        PainlessMethod setter = scriptRoot.getPainlessLookup().lookupPainlessMethod(targetClass, false, "put", 2);
 
         if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1)) {
             throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + canonicalClassName + "]."));
@@ -70,32 +66,31 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
+        Output indexOutput;
+
         if ((input.read || input.write) && (input.read == false || getter != null) && (input.write == false || setter != null)) {
             Input indexInput = new Input();
             indexInput.expected = setter != null ? setter.typeParameters.get(0) : getter.typeParameters.get(0);
-            index.analyze(scriptRoot, scope, indexInput);
-            index.cast();
+            indexOutput = index.analyze(classNode, scriptRoot, scope, indexInput);
+            index.cast(indexInput, indexOutput);
 
             output.actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal map shortcut for type [" + canonicalClassName + "]."));
         }
 
-        return output;
-    }
-
-    @Override
-    MapSubShortcutNode write(ClassNode classNode) {
         MapSubShortcutNode mapSubShortcutNode = new MapSubShortcutNode();
 
-        mapSubShortcutNode.setChildNode(index.cast(index.write(classNode)));
+        mapSubShortcutNode.setChildNode(index.cast(indexOutput));
 
         mapSubShortcutNode.setLocation(location);
         mapSubShortcutNode.setExpressionType(output.actual);
         mapSubShortcutNode.setGetter(getter);
         mapSubShortcutNode.setSetter(setter);
 
-        return mapSubShortcutNode;
+        output.expressionNode = mapSubShortcutNode;
+
+        return output;
     }
 
     @Override
@@ -103,11 +98,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalArgumentException("Illegal tree structure.");
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, index);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
index 83d5fded5649d..bc9a259a2e96c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeCallInvoke.java
@@ -31,10 +31,11 @@
  * Implements a call who's value is null if the prefix is null rather than throwing an NPE.
  */
 public class PSubNullSafeCallInvoke extends AExpression {
+
     /**
-     * The expression gaurded by the null check. Required at construction time and replaced at analysis time.
+     * The expression guarded by the null check. Required at construction time and replaced at analysis time.
      */
-    private AExpression guarded;
+    protected final AExpression guarded;
 
     public PSubNullSafeCallInvoke(Location location, AExpression guarded) {
         super(location);
@@ -42,29 +43,25 @@ public PSubNullSafeCallInvoke(Location location, AExpression guarded) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
-        Output guardedOutput = guarded.analyze(scriptRoot, scope, new Input());
+        Output guardedOutput = guarded.analyze(classNode, scriptRoot, scope, new Input());
         output.actual = guardedOutput.actual;
         if (output.actual.isPrimitive()) {
             throw new IllegalArgumentException("Result of null safe operator must be nullable");
         }
 
-        return output;
-    }
-
-    @Override
-    NullSafeSubNode write(ClassNode classNode) {
         NullSafeSubNode nullSafeSubNode = new NullSafeSubNode();
 
-        nullSafeSubNode.setChildNode(guarded.write(classNode));
+        nullSafeSubNode.setChildNode(guardedOutput.expressionNode);
 
         nullSafeSubNode.setLocation(location);
         nullSafeSubNode.setExpressionType(output.actual);
 
-        return nullSafeSubNode;
+        output.expressionNode = nullSafeSubNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
index 536c8b15e83c6..22d4c91945273 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubNullSafeField.java
@@ -29,7 +29,8 @@
  * Implements a field who's value is null if the prefix is null rather than throwing an NPE.
  */
 public class PSubNullSafeField extends AStoreable {
-    private AStoreable guarded;
+
+    protected final AStoreable guarded;
 
     public PSubNullSafeField(Location location, AStoreable guarded) {
         super(location);
@@ -37,44 +38,37 @@ public PSubNullSafeField(Location location, AStoreable guarded) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         if (input.write) {
             throw createError(new IllegalArgumentException("Can't write to null safe reference"));
         }
+
         Input guardedInput = new Input();
         guardedInput.read = input.read;
-        Output guardedOutput = guarded.analyze(scriptRoot, scope, guardedInput);
+        Output guardedOutput = guarded.analyze(classNode, scriptRoot, scope, guardedInput);
         output.actual = guardedOutput.actual;
+
         if (output.actual.isPrimitive()) {
             throw new IllegalArgumentException("Result of null safe operator must be nullable");
         }
 
-        return output;
-    }
-
-    @Override
-    boolean isDefOptimized() {
-        return guarded.isDefOptimized();
-    }
-
-    @Override
-    void updateActual(Class<?> actual) {
-        guarded.updateActual(actual);
-    }
-
-    @Override
-    NullSafeSubNode write(ClassNode classNode) {
         NullSafeSubNode nullSafeSubNode = new NullSafeSubNode();
 
-        nullSafeSubNode.setChildNode(guarded.write(classNode));
+        nullSafeSubNode.setChildNode(guardedOutput.expressionNode);
 
         nullSafeSubNode.setLocation(location);
         nullSafeSubNode.setExpressionType(output.actual);
 
-        return nullSafeSubNode;
+        output.expressionNode = nullSafeSubNode;
+
+        return output;
+    }
+
+    @Override
+    boolean isDefOptimized() {
+        return false;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
index e88fbeaac0765..a55a5556bafca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
@@ -29,12 +29,12 @@
 /**
  * Represents a field load/store shortcut.  (Internal only.)
  */
-final class PSubShortcut extends AStoreable {
+public class PSubShortcut extends AStoreable {
 
-    private final String value;
-    private final String type;
-    private final PainlessMethod getter;
-    private final PainlessMethod setter;
+    protected final String value;
+    protected final String type;
+    protected final PainlessMethod getter;
+    protected final PainlessMethod setter;
 
     PSubShortcut(Location location, String value, String type, PainlessMethod getter, PainlessMethod setter) {
         super(location);
@@ -46,9 +46,8 @@ final class PSubShortcut extends AStoreable {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
+        Output output = new Output();
 
         if (getter != null && (getter.returnType == void.class || !getter.typeParameters.isEmpty())) {
             throw createError(new IllegalArgumentException(
@@ -70,11 +69,6 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, AStoreable.Input input) {
             throw createError(new IllegalArgumentException("Illegal shortcut on field [" + value + "] for type [" + type + "]."));
         }
 
-        return output;
-    }
-
-    @Override
-    DotSubShortcutNode write(ClassNode classNode) {
         DotSubShortcutNode dotSubShortcutNode = new DotSubShortcutNode();
 
         dotSubShortcutNode.setLocation(location);
@@ -82,7 +76,9 @@ DotSubShortcutNode write(ClassNode classNode) {
         dotSubShortcutNode.setGetter(getter);
         dotSubShortcutNode.setSetter(setter);
 
-        return dotSubShortcutNode;
+        output.expressionNode = dotSubShortcutNode;
+
+        return output;
     }
 
     @Override
@@ -90,11 +86,6 @@ boolean isDefOptimized() {
         return false;
     }
 
-    @Override
-    void updateActual(Class<?> actual) {
-        throw new IllegalArgumentException("Illegal tree structure.");
-    }
-
     @Override
     public String toString() {
         return singleLineToString(prefix, value);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 8e6db345e0b30..f662976c2e4b3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -33,9 +34,9 @@
 /**
  * Represents a set of statements as a branch of control-flow.
  */
-public final class SBlock extends AStatement {
+public class SBlock extends AStatement {
 
-    final List<AStatement> statements;
+    protected final List<AStatement> statements;
 
     public SBlock(Location location, List<AStatement> statements) {
         super(location);
@@ -44,9 +45,8 @@ public SBlock(Location location, List<AStatement> statements) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (statements == null || statements.isEmpty()) {
             throw createError(new IllegalArgumentException("A block must contain at least one statement."));
@@ -54,6 +54,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         AStatement last = statements.get(statements.size() - 1);
 
+        List<Output> statementOutputs = new ArrayList<>(statements.size());
+
         for (AStatement statement : statements) {
             // Note that we do not need to check after the last statement because
             // there is no statement that can be unreachable after the last.
@@ -65,7 +67,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             statementInput.inLoop = input.inLoop;
             statementInput.lastSource = input.lastSource && statement == last;
             statementInput.lastLoop = (input.beginLoop || input.lastLoop) && statement == last;
-            Output statementOutput = statement.analyze(scriptRoot, scope, statementInput);
+
+            Output statementOutput = statement.analyze(classNode, scriptRoot, scope, statementInput);
 
             output.methodEscape = statementOutput.methodEscape;
             output.loopEscape = statementOutput.loopEscape;
@@ -73,24 +76,23 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.anyContinue |= statementOutput.anyContinue;
             output.anyBreak |= statementOutput.anyBreak;
             output.statementCount += statementOutput.statementCount;
-        }
 
-        return output;
-    }
+            statementOutputs.add(statementOutput);
+        }
 
-    @Override
-    BlockNode write(ClassNode classNode) {
         BlockNode blockNode = new BlockNode();
 
-        for (AStatement statement : statements) {
-            blockNode.addStatementNode(statement.write(classNode));
+        for (Output statementOutput : statementOutputs) {
+            blockNode.addStatementNode(statementOutput.statementNode);
         }
 
         blockNode.setLocation(location);
         blockNode.setAllEscape(output.allEscape);
         blockNode.setStatementCount(output.statementCount);
 
-        return blockNode;
+        output.statementNode = blockNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 4b036153319ea..55d22a2097a9f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -28,16 +28,15 @@
 /**
  * Represents a break statement.
  */
-public final class SBreak extends AStatement {
+public class SBreak extends AStatement {
 
     public SBreak(Location location) {
         super(location);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.inLoop == false) {
             throw createError(new IllegalArgumentException("Break statement outside of a loop."));
@@ -48,16 +47,12 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         output.anyBreak = true;
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    BreakNode write(ClassNode classNode) {
         BreakNode breakNode = new BreakNode();
-
         breakNode.setLocation(location);
 
-        return breakNode;
+        output.statementNode = breakNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index f00c0b18a70fc..06dc1a06514fd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -21,8 +21,10 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.CatchNode;
 import org.elasticsearch.painless.ir.ClassNode;
+import org.elasticsearch.painless.ir.DeclarationNode;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
@@ -31,11 +33,11 @@
 /**
  * Represents a catch block as part of a try-catch block.
  */
-public final class SCatch extends AStatement {
+public class SCatch extends AStatement {
 
-    private final DType baseException;
-    private final SDeclaration declaration;
-    private final SBlock block;
+    protected final DType baseException;
+    protected final SDeclaration declaration;
+    protected final SBlock block;
 
     public SCatch(Location location, DType baseException, SDeclaration declaration, SBlock block) {
         super(location);
@@ -46,11 +48,10 @@ public SCatch(Location location, DType baseException, SDeclaration declaration,
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
-        declaration.analyze(scriptRoot, scope, new Input());
+        Output declarationOutput = declaration.analyze(classNode, scriptRoot, scope, new Input());
 
         Class<?> baseType = baseException.resolveType(scriptRoot.getPainlessLookup()).getType();
         Class<?> type = scope.getVariable(location, declaration.name).getType();
@@ -61,12 +62,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
                     "to [" + PainlessLookupUtility.typeToCanonicalTypeName(baseType) + "]"));
         }
 
+        Output blockOutput = null;
+
         if (block != null) {
             Input blockInput = new Input();
             blockInput.lastSource = input.lastSource;
             blockInput.inLoop = input.inLoop;
             blockInput.lastLoop = input.lastLoop;
-            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+            blockOutput = block.analyze(classNode, scriptRoot, scope, blockInput);
 
             output.methodEscape = blockOutput.methodEscape;
             output.loopEscape = blockOutput.loopEscape;
@@ -76,19 +79,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.statementCount = blockOutput.statementCount;
         }
 
-        return output;
-    }
-
-    @Override
-    CatchNode write(ClassNode classNode) {
         CatchNode catchNode = new CatchNode();
-
-        catchNode.setDeclarationNode(declaration.write(classNode));
-        catchNode.setBlockNode(block == null ? null : block.write(classNode));
-
+        catchNode.setDeclarationNode((DeclarationNode)declarationOutput.statementNode);
+        catchNode.setBlockNode(blockOutput == null ? null : (BlockNode)blockOutput.statementNode);
         catchNode.setLocation(location);
 
-        return catchNode;
+        output.statementNode = catchNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java
index d946b9b1c69b5..1c03994d55d06 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.ScriptClassInfo;
 import org.elasticsearch.painless.ir.ClassNode;
-import org.elasticsearch.painless.ir.StatementNode;
 import org.elasticsearch.painless.symbol.FunctionTable;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 import org.objectweb.asm.util.Printer;
@@ -36,37 +35,25 @@
 /**
  * The root of all Painless trees.  Contains a series of statements.
  */
-public final class SClass extends ANode {
+public class SClass extends ANode {
 
-    private final ScriptClassInfo scriptClassInfo;
-    private final String name;
-    private final Printer debugStream;
-    private final List<SFunction> functions = new ArrayList<>();
-    private final List<SField> fields = new ArrayList<>();
-
-    private ScriptRoot scriptRoot;
-    private final String sourceText;
+    protected final ScriptClassInfo scriptClassInfo;
+    protected final String name;
+    protected final String sourceText;
+    protected final Printer debugStream;
+    protected final List<SFunction> functions = new ArrayList<>();
 
     public SClass(ScriptClassInfo scriptClassInfo, String name, String sourceText, Printer debugStream,
             Location location, List<SFunction> functions) {
         super(location);
         this.scriptClassInfo = Objects.requireNonNull(scriptClassInfo);
         this.name = Objects.requireNonNull(name);
+        this.sourceText = Objects.requireNonNull(sourceText);
         this.debugStream = debugStream;
         this.functions.addAll(Objects.requireNonNull(functions));
-        this.sourceText = Objects.requireNonNull(sourceText);
-    }
-
-    void addFunction(SFunction function) {
-        functions.add(function);
     }
 
-    void addField(SField field) {
-        fields.add(field);
-    }
-
-    public ScriptRoot analyze(ScriptRoot scriptRoot) {
-        this.scriptRoot = scriptRoot;
+    public ClassNode writeClass(ScriptRoot scriptRoot) {
         scriptRoot.addStaticConstant("$NAME", name);
         scriptRoot.addStaticConstant("$SOURCE", sourceText);
 
@@ -83,31 +70,10 @@ public ScriptRoot analyze(ScriptRoot scriptRoot) {
                     function.name, function.returnType, function.typeParameters, function.isInternal, function.isStatic);
         }
 
-        // copy protection is required because synthetic functions are
-        // added for lambdas/method references and analysis here is
-        // only for user-defined functions
-        List<SFunction> functions = new ArrayList<>(this.functions);
-        for (SFunction function : functions) {
-            function.analyze(scriptRoot);
-        }
-
-        return scriptRoot;
-    }
-
-    @Override
-    public StatementNode write(ClassNode classNode) {
-        throw new UnsupportedOperationException();
-    }
-
-    public ClassNode writeClass() {
         ClassNode classNode = new ClassNode();
 
-        for (SField field : fields) {
-            classNode.addFieldNode(field.write(classNode));
-        }
-
         for (SFunction function : functions) {
-            classNode.addFunctionNode(function.write(classNode));
+            classNode.addFunctionNode(function.writeFunction(classNode, scriptRoot));
         }
 
         classNode.setLocation(location);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 1f5c752f8ccef..830455274f9e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -28,16 +28,15 @@
 /**
  * Represents a continue statement.
  */
-public final class SContinue extends AStatement {
+public class SContinue extends AStatement {
 
     public SContinue(Location location) {
         super(location);
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (input.inLoop == false) {
             throw createError(new IllegalArgumentException("Continue statement outside of a loop."));
@@ -51,16 +50,12 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         output.anyContinue = true;
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    ContinueNode write(ClassNode classNode) {
         ContinueNode continueNode = new ContinueNode();
-
         continueNode.setLocation(location);
 
-        return continueNode;
+        output.statementNode = continueNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 47bbe0d123ecb..4bbfcf6d1d505 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -23,8 +23,10 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.DeclarationBlockNode;
+import org.elasticsearch.painless.ir.DeclarationNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -33,9 +35,9 @@
 /**
  * Represents a series of declarations.
  */
-public final class SDeclBlock extends AStatement {
+public class SDeclBlock extends AStatement {
 
-    private final List<SDeclaration> declarations;
+    protected final List<SDeclaration> declarations;
 
     public SDeclBlock(Location location, List<SDeclaration> declarations) {
         super(location);
@@ -44,30 +46,28 @@ public SDeclBlock(Location location, List<SDeclaration> declarations) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        List<Output> declarationOutputs = new ArrayList<>(declarations.size());
 
         for (SDeclaration declaration : declarations) {
-            declaration.analyze(scriptRoot, scope, new Input());
+            declarationOutputs.add(declaration.analyze(classNode, scriptRoot, scope, new Input()));
         }
 
         output.statementCount = declarations.size();
 
-        return output;
-    }
-
-    @Override
-    DeclarationBlockNode write(ClassNode classNode) {
         DeclarationBlockNode declarationBlockNode = new DeclarationBlockNode();
 
-        for (SDeclaration declaration : declarations) {
-            declarationBlockNode.addDeclarationNode(declaration.write(classNode));
+        for (Output declarationOutput : declarationOutputs) {
+            declarationBlockNode.addDeclarationNode((DeclarationNode)declarationOutput.statementNode);
         }
 
         declarationBlockNode.setLocation(location);
 
-        return declarationBlockNode;
+        output.statementNode = declarationBlockNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 2a28b4d029558..22b71eb931a79 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.DeclarationNode;
-import org.elasticsearch.painless.node.AExpression.Input;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -31,12 +30,12 @@
 /**
  * Represents a single variable declaration.
  */
-public final class SDeclaration extends AStatement {
+public class SDeclaration extends AStatement {
 
-    private DType type;
+    protected final DType type;
     protected final String name;
     protected final boolean requiresDefault;
-    private AExpression expression;
+    protected final AExpression expression;
 
     public SDeclaration(Location location, DType type, String name, boolean requiresDefault, AExpression expression) {
         super(location);
@@ -48,37 +47,32 @@ public SDeclaration(Location location, DType type, String name, boolean requires
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         DResolvedType resolvedType = type.resolveType(scriptRoot.getPainlessLookup());
-        type = resolvedType;
+
+        AExpression.Output expressionOutput = null;
 
         if (expression != null) {
             AExpression.Input expressionInput = new AExpression.Input();
             expressionInput.expected = resolvedType.getType();
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
         }
 
         scope.defineVariable(location, resolvedType.getType(), name, false);
 
-        return output;
-    }
-
-    @Override
-    DeclarationNode write(ClassNode classNode) {
         DeclarationNode declarationNode = new DeclarationNode();
-
-        declarationNode.setExpressionNode(expression == null ? null : expression.cast(expression.write(classNode)));
-
+        declarationNode.setExpressionNode(expression == null ? null : expression.cast(expressionOutput));
         declarationNode.setLocation(location);
-        declarationNode.setDeclarationType(((DResolvedType)type).getType());
+        declarationNode.setDeclarationType(resolvedType.getType());
         declarationNode.setName(name);
         declarationNode.setRequiresDefault(requiresDefault);
 
-        return declarationNode;
+        output.statementNode = declarationNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 8d31f325a049f..f5f4b0636a370 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.DoWhileLoopNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -30,12 +31,10 @@
 /**
  * Represents a do-while loop.
  */
-public final class SDo extends AStatement {
+public class SDo extends AStatement {
 
-    private final SBlock block;
-    private AExpression condition;
-
-    private boolean continuous = false;
+    protected final SBlock block;
+    protected final AExpression condition;
 
     public SDo(Location location, SBlock block, AExpression condition) {
         super(location);
@@ -45,10 +44,8 @@ public SDo(Location location, SBlock block, AExpression condition) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
         scope = scope.newLocalScope();
 
         if (block == null) {
@@ -58,7 +55,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         Input blockInput = new Input();
         blockInput.beginLoop = true;
         blockInput.inLoop = true;
-        Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+        Output blockOutput = block.analyze(classNode, scriptRoot, scope, blockInput);
 
         if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
@@ -66,8 +63,10 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         AExpression.Input conditionInput = new AExpression.Input();
         conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
-        condition.cast();
+        AExpression.Output conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+        condition.cast(conditionInput, conditionOutput);
+
+        boolean continuous = false;
 
         if (condition instanceof EBoolean) {
             continuous = ((EBoolean)condition).constant;
@@ -84,20 +83,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    DoWhileLoopNode write(ClassNode classNode) {
         DoWhileLoopNode doWhileLoopNode = new DoWhileLoopNode();
-
-        doWhileLoopNode.setConditionNode(condition.cast(condition.write(classNode)));
-        doWhileLoopNode.setBlockNode(block.write(classNode));
-
+        doWhileLoopNode.setConditionNode(condition.cast(conditionOutput));
+        doWhileLoopNode.setBlockNode((BlockNode)blockOutput.statementNode);
         doWhileLoopNode.setLocation(location);
         doWhileLoopNode.setContinuous(continuous);
 
-        return doWhileLoopNode;
+        output.statementNode = doWhileLoopNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index b5c439233b64d..ac8741cc551ec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -36,12 +36,10 @@
  */
 public class SEach extends AStatement {
 
-    private final String type;
-    private final String name;
-    private AExpression expression;
-    private final SBlock block;
-
-    private AStatement sub = null;
+    protected final String type;
+    protected final String name;
+    protected final AExpression expression;
+    protected final SBlock block;
 
     public SEach(Location location, String type, String name, AExpression expression, SBlock block) {
         super(location);
@@ -53,13 +51,14 @@ public SEach(Location location, String type, String name, AExpression expression
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
-        AExpression.Output expressionOutput = expression.analyze(scriptRoot, scope, new AExpression.Input());
-        expression.input.expected = expressionOutput.actual;
-        expression.cast();
+        AExpression.Input expressionInput = new AExpression.Input();
+        AExpression.Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+        // TODO: no need to cast here
+        expressionInput.expected = expressionOutput.actual;
+        expression.cast(expressionInput, expressionOutput);
 
         Class<?> clazz = scriptRoot.getPainlessLookup().canonicalTypeNameToType(this.type);
 
@@ -70,17 +69,6 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         scope = scope.newLocalScope();
         Variable variable = scope.defineVariable(location, clazz, name, true);
 
-        if (expressionOutput.actual.isArray()) {
-            sub = new SSubEachArray(location, variable, expression, block);
-        } else if (expressionOutput.actual == def.class || Iterable.class.isAssignableFrom(expressionOutput.actual)) {
-            sub = new SSubEachIterable(location, variable, expression, block);
-        } else {
-            throw createError(new IllegalArgumentException("Illegal for each type " +
-                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(expressionOutput.actual) + "]."));
-        }
-
-        sub.analyze(scriptRoot, scope, input);
-
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous for each loop."));
         }
@@ -88,27 +76,36 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         Input blockInput = new Input();
         blockInput.beginLoop = true;
         blockInput.inLoop = true;
-        Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+        Output blockOutput = block.analyze(classNode, scriptRoot, scope, blockInput);
         blockOutput.statementCount = Math.max(1, blockOutput.statementCount);
 
         if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
             throw createError(new IllegalArgumentException("Extraneous for loop."));
         }
 
-        output.statementCount = 1;
+        AStatement sub;
 
-        return output;
-    }
+        if (expressionOutput.actual.isArray()) {
+            sub = new SSubEachArray(location, variable, expressionOutput, blockOutput);
+        } else if (expressionOutput.actual == def.class || Iterable.class.isAssignableFrom(expressionOutput.actual)) {
+            sub = new SSubEachIterable(location, variable, expressionOutput, blockOutput);
+        } else {
+            throw createError(new IllegalArgumentException("Illegal for each type " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(expressionOutput.actual) + "]."));
+        }
 
-    @Override
-    ForEachLoopNode write(ClassNode classNode) {
-        ForEachLoopNode forEachLoopNode = new ForEachLoopNode();
+        Output subOutput = sub.analyze(classNode, scriptRoot, scope, input);
 
-        forEachLoopNode.setConditionNode((ConditionNode)sub.write(classNode));
+        output.statementCount = 1;
 
+        ForEachLoopNode forEachLoopNode = new ForEachLoopNode();
+        forEachLoopNode.setConditionNode((ConditionNode)subOutput.statementNode);
         forEachLoopNode.setLocation(location);
 
-        return forEachLoopNode;
+        output.statementNode = forEachLoopNode;
+
+        return output;
+
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index f04cefa622acd..ace1be4b96736 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.ir.ReturnNode;
 import org.elasticsearch.painless.ir.StatementExpressionNode;
-import org.elasticsearch.painless.ir.StatementNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
 import java.util.Objects;
@@ -33,9 +32,9 @@
 /**
  * Represents the top-level node for an expression as a statement.
  */
-public final class SExpression extends AStatement {
+public class SExpression extends AStatement {
 
-    private AExpression expression;
+    protected final AExpression expression;
 
     public SExpression(Location location, AExpression expression) {
         super(location);
@@ -44,15 +43,13 @@ public SExpression(Location location, AExpression expression) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
         Class<?> rtnType = scope.getReturnType();
         boolean isVoid = rtnType == void.class;
 
         AExpression.Input expressionInput = new AExpression.Input();
         expressionInput.read = input.lastSource && !isVoid;
-        AExpression.Output expressionOutput = expression.analyze(scriptRoot, scope, expressionInput);
+        AExpression.Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
 
         if ((input.lastSource == false || isVoid) && expressionOutput.statement == false) {
             throw createError(new IllegalArgumentException("Not a statement."));
@@ -60,40 +57,33 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         boolean rtn = input.lastSource && isVoid == false && expressionOutput.actual != void.class;
 
-        expression.input.expected = rtn ? rtnType : expressionOutput.actual;
-        expression.input.internal = rtn;
-        expression.cast();
+        expressionInput.expected = rtn ? rtnType : expressionOutput.actual;
+        expressionInput.internal = rtn;
+        expression.cast(expressionInput, expressionOutput);
 
-        output = new Output();
+        Output output = new Output();
         output.methodEscape = rtn;
         output.loopEscape = rtn;
         output.allEscape = rtn;
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    StatementNode write(ClassNode classNode) {
-        ExpressionNode expressionNode = expression.cast(expression.write(classNode));
+        ExpressionNode expressionNode = expression.cast(expressionOutput);
 
         if (output.methodEscape) {
             ReturnNode returnNode = new ReturnNode();
-
             returnNode.setExpressionNode(expressionNode);
-
             returnNode.setLocation(location);
 
-            return returnNode;
+            output.statementNode = returnNode;
         } else {
             StatementExpressionNode statementExpressionNode = new StatementExpressionNode();
-
             statementExpressionNode.setExpressionNode(expressionNode);
-
             statementExpressionNode.setLocation(location);
 
-            return statementExpressionNode;
+            output.statementNode = statementExpressionNode;
         }
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SField.java
deleted file mode 100644
index 944802fb7ac39..0000000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SField.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless.node;
-
-import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.ir.ClassNode;
-import org.elasticsearch.painless.ir.FieldNode;
-
-/**
- * Represents a member field for its parent class (internal only).
- */
-public class SField extends ANode {
-
-    private final int modifiers;
-    private final String name;
-    private final Class<?> type;
-
-    /**
-     * Standard constructor.
-     * @param location original location in the source
-     * @param modifiers java modifiers for the field
-     * @param name name of the field
-     * @param type type of the field
-     */
-    public SField(Location location, int modifiers, String name, Class<?> type) {
-        super(location);
-
-        this.modifiers = modifiers;
-        this.name = name;
-        this.type = type;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    @Override
-    FieldNode write(ClassNode classNode) {
-        FieldNode fieldNode = new FieldNode();
-
-        fieldNode.setLocation(location);
-        fieldNode.setModifiers(modifiers);
-        fieldNode.setName(name);
-        fieldNode.setFieldType(type);
-
-        return fieldNode;
-    }
-
-    @Override
-    public String toString() {
-        return singleLineToString(name, type);
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 241c50b8fe2b9..ce43e9a434f1a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -21,8 +21,8 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
-import org.elasticsearch.painless.ir.ExpressionNode;
 import org.elasticsearch.painless.ir.ForLoopNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
@@ -33,14 +33,12 @@
 /**
  * Represents a for loop.
  */
-public final class SFor extends AStatement {
+public class SFor extends AStatement {
 
-    private ANode initializer;
-    private AExpression condition;
-    private AExpression afterthought;
-    private final SBlock block;
-
-    private boolean continuous = false;
+    protected final ANode initializer;
+    protected final AExpression condition;
+    protected final AExpression afterthought;
+    protected final SBlock block;
 
     public SFor(Location location, ANode initializer, AExpression condition, AExpression afterthought, SBlock block) {
         super(location);
@@ -52,37 +50,42 @@ public SFor(Location location, ANode initializer, AExpression condition, AExpres
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
         scope = scope.newLocalScope();
 
+        Output initializerStatementOutput = null;
+        AExpression.Output initializerExpressionOutput = null;
+
         if (initializer != null) {
             if (initializer instanceof SDeclBlock) {
-                ((SDeclBlock)initializer).analyze(scriptRoot, scope, new Input());
+                initializerStatementOutput = ((SDeclBlock)initializer).analyze(classNode, scriptRoot, scope, new Input());
             } else if (initializer instanceof AExpression) {
                 AExpression initializer = (AExpression)this.initializer;
 
                 AExpression.Input initializerInput = new AExpression.Input();
                 initializerInput.read = false;
-                AExpression.Output initializerOutput = initializer.analyze(scriptRoot, scope, initializerInput);
+                initializerExpressionOutput = initializer.analyze(classNode, scriptRoot, scope, initializerInput);
 
-                if (initializerOutput.statement == false) {
+                if (initializerExpressionOutput.statement == false) {
                     throw createError(new IllegalArgumentException("Not a statement."));
                 }
 
-                initializer.input.expected = initializerOutput.actual;
-                initializer.cast();
+                initializerInput.expected = initializerExpressionOutput.actual;
+                initializer.cast(initializerInput, initializerExpressionOutput);
             } else {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
+        boolean continuous = false;
+
+        AExpression.Output conditionOutput = null;
+
         if (condition != null) {
             AExpression.Input conditionInput = new AExpression.Input();
             conditionInput.expected = boolean.class;
-            condition.analyze(scriptRoot, scope, conditionInput);
-            condition.cast();
+            conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+            condition.cast(conditionInput, conditionOutput);
 
             if (condition instanceof EBoolean) {
                 continuous = ((EBoolean)condition).constant;
@@ -99,27 +102,30 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             continuous = true;
         }
 
+        AExpression.Output afterthoughtOutput = null;
+
         if (afterthought != null) {
             AExpression.Input afterthoughtInput = new AExpression.Input();
             afterthoughtInput.read = false;
-            AExpression.Output afterthoughtOutput = afterthought.analyze(scriptRoot, scope, afterthoughtInput);
+            afterthoughtOutput = afterthought.analyze(classNode, scriptRoot, scope, afterthoughtInput);
 
             if (afterthoughtOutput.statement == false) {
                 throw createError(new IllegalArgumentException("Not a statement."));
             }
 
-            afterthought.input.expected = afterthoughtOutput.actual;
-            afterthought.cast();
+            afterthoughtInput.expected = afterthoughtOutput.actual;
+            afterthought.cast(afterthoughtInput, afterthoughtOutput);
         }
 
-        output = new Output();
+        Output output = new Output();
+        Output blockOutput = null;
 
         if (block != null) {
             Input blockInput = new Input();
             blockInput.beginLoop = true;
             blockInput.inLoop = true;
 
-            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+            blockOutput = block.analyze(classNode, scriptRoot, scope, blockInput);
 
             if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
                 throw createError(new IllegalArgumentException("Extraneous for loop."));
@@ -135,24 +141,19 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    ForLoopNode write(ClassNode classNode) {
         ForLoopNode forLoopNode = new ForLoopNode();
-
         forLoopNode.setInitialzerNode(initializer == null ? null : initializer instanceof AExpression ?
-                ((AExpression)initializer).cast((ExpressionNode)initializer.write(classNode)) :
-                initializer.write(classNode));
-        forLoopNode.setConditionNode(condition == null ? null : condition.cast(condition.write(classNode)));
-        forLoopNode.setAfterthoughtNode(afterthought == null ? null : afterthought.cast(afterthought.write(classNode)));
-        forLoopNode.setBlockNode(block == null ? null : block.write(classNode));
-
+                ((AExpression)initializer).cast(initializerExpressionOutput) :
+                initializerStatementOutput.statementNode);
+        forLoopNode.setConditionNode(condition == null ? null : condition.cast(conditionOutput));
+        forLoopNode.setAfterthoughtNode(afterthought == null ? null : afterthought.cast(afterthoughtOutput));
+        forLoopNode.setBlockNode(blockOutput == null ? null : (BlockNode)blockOutput.statementNode);
         forLoopNode.setLocation(location);
         forLoopNode.setContinuous(continuous);
 
-        return forLoopNode;
+        output.statementNode = forLoopNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 96bf040099a17..fad71982b4772 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -46,32 +46,28 @@
 /**
  * Represents a user-defined function.
  */
-public final class SFunction extends ANode {
+public class SFunction extends ANode {
 
-    private final String rtnTypeStr;
-    public final String name;
-    private final List<String> paramTypeStrs;
-    private final List<String> paramNameStrs;
-    private final SBlock block;
-    public final boolean isInternal;
-    public final boolean isStatic;
-    public final boolean synthetic;
+    protected final String rtnTypeStr;
+    protected final String name;
+    protected final List<String> paramTypeStrs;
+    protected final List<String> paramNameStrs;
+    protected final SBlock block;
+    protected final boolean isInternal;
+    protected final boolean isStatic;
+    protected final boolean synthetic;
 
     /**
      * If set to {@code true} default return values are inserted if
      * not all paths return a value.
      */
-    public final boolean isAutoReturnEnabled;
+    protected final boolean isAutoReturnEnabled;
 
-    private int maxLoopCounter;
+    protected Class<?> returnType;
+    protected List<Class<?>> typeParameters;
+    protected MethodType methodType;
 
-    Class<?> returnType;
-    List<Class<?>> typeParameters;
-    MethodType methodType;
-
-    org.objectweb.asm.commons.Method method;
-
-    private boolean methodEscape;
+    protected org.objectweb.asm.commons.Method method;
 
     public SFunction(Location location, String rtnType, String name,
             List<String> paramTypes, List<String> paramNames,
@@ -89,6 +85,7 @@ public SFunction(Location location, String rtnType, String name,
         this.isAutoReturnEnabled = isAutoReturnEnabled;
     }
 
+    // TODO: do this in class on add to remove need for mutable state
     void generateSignature(PainlessLookup painlessLookup) {
         returnType = painlessLookup.canonicalTypeNameToType(rtnTypeStr);
 
@@ -121,7 +118,7 @@ void generateSignature(PainlessLookup painlessLookup) {
                 PainlessLookupUtility.typeToJavaType(returnType), paramClasses).toMethodDescriptorString());
     }
 
-    void analyze(ScriptRoot scriptRoot) {
+    FunctionNode writeFunction(ClassNode classNode, ScriptRoot scriptRoot) {
         FunctionScope functionScope = newFunctionScope(returnType);
 
         for (int index = 0; index < typeParameters.size(); ++index) {
@@ -130,7 +127,7 @@ void analyze(ScriptRoot scriptRoot) {
             functionScope.defineVariable(location, typeParameter, parameterName, false);
         }
 
-        maxLoopCounter = scriptRoot.getCompilerSettings().getMaxLoopCounter();
+        int maxLoopCounter = scriptRoot.getCompilerSettings().getMaxLoopCounter();
 
         if (block.statements.isEmpty()) {
             throw createError(new IllegalArgumentException("Cannot generate an empty function [" + name + "]."));
@@ -138,8 +135,8 @@ void analyze(ScriptRoot scriptRoot) {
 
         Input blockInput = new Input();
         blockInput.lastSource = true;
-        Output blockOutput = block.analyze(scriptRoot, functionScope.newLocalScope(), blockInput);
-        methodEscape = blockOutput.methodEscape;
+        Output blockOutput = block.analyze(classNode, scriptRoot, functionScope.newLocalScope(), blockInput);
+        boolean methodEscape = blockOutput.methodEscape;
 
         if (methodEscape == false && isAutoReturnEnabled == false && returnType != void.class) {
             throw createError(new IllegalArgumentException("not all paths provide a return value " +
@@ -152,11 +149,8 @@ void analyze(ScriptRoot scriptRoot) {
             scriptRoot.setUsedVariables(functionScope.getUsedVariables());
         }
         // TODO: end
-    }
 
-    @Override
-    public FunctionNode write(ClassNode classNode) {
-        BlockNode blockNode = block.write(classNode);
+        BlockNode blockNode = (BlockNode)blockOutput.statementNode;
 
         if (methodEscape == false) {
             ExpressionNode expressionNode;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index a7d738670ec6f..44b6e0d6105c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.IfNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -30,10 +31,10 @@
 /**
  * Represents an if block.
  */
-public final class SIf extends AStatement {
+public class SIf extends AStatement {
 
-    AExpression condition;
-    final SBlock ifblock;
+    protected final AExpression condition;
+    protected final SBlock ifblock;
 
     public SIf(Location location, AExpression condition, SBlock ifblock) {
         super(location);
@@ -43,14 +44,13 @@ public SIf(Location location, AExpression condition, SBlock ifblock) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         AExpression.Input conditionInput = new AExpression.Input();
         conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
-        condition.cast();
+        AExpression.Output conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+        condition.cast(conditionInput, conditionOutput);
 
         if (condition instanceof EBoolean) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
@@ -65,25 +65,20 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         ifblockInput.inLoop = input.inLoop;
         ifblockInput.lastLoop = input.lastLoop;
 
-        Output ifblockOutput = ifblock.analyze(scriptRoot, scope.newLocalScope(), ifblockInput);
+        Output ifblockOutput = ifblock.analyze(classNode, scriptRoot, scope.newLocalScope(), ifblockInput);
 
         output.anyContinue = ifblockOutput.anyContinue;
         output.anyBreak = ifblockOutput.anyBreak;
         output.statementCount = ifblockOutput.statementCount;
 
-        return output;
-    }
-
-    @Override
-    IfNode write(ClassNode classNode) {
         IfNode ifNode = new IfNode();
-
-        ifNode.setConditionNode(condition.cast(condition.write(classNode)));
-        ifNode.setBlockNode(ifblock.write(classNode));
-
+        ifNode.setConditionNode(condition.cast(conditionOutput));
+        ifNode.setBlockNode((BlockNode)ifblockOutput.statementNode);
         ifNode.setLocation(location);
 
-        return ifNode;
+        output.statementNode = ifNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index f433db860ccfb..6ae366ad8428a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.IfElseNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -33,11 +34,11 @@
 /**
  * Represents an if/else block.
  */
-public final class SIfElse extends AStatement {
+public class SIfElse extends AStatement {
 
-    private AExpression condition;
-    private final SBlock ifblock;
-    private final SBlock elseblock;
+    protected final AExpression condition;
+    protected final SBlock ifblock;
+    protected final SBlock elseblock;
 
     public SIfElse(Location location, AExpression condition, SBlock ifblock, SBlock elseblock) {
         super(location);
@@ -48,14 +49,13 @@ public SIfElse(Location location, AExpression condition, SBlock ifblock, SBlock
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         AExpression.Input conditionInput = new AExpression.Input();
         conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
-        condition.cast();
+        AExpression.Output conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+        condition.cast(conditionInput, conditionOutput);
 
         if (condition instanceof EBoolean) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
@@ -70,7 +70,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         ifblockInput.inLoop = input.inLoop;
         ifblockInput.lastLoop = input.lastLoop;
 
-        Output ifblockOutput = ifblock.analyze(scriptRoot, scope.newLocalScope(), ifblockInput);
+        Output ifblockOutput = ifblock.analyze(classNode, scriptRoot, scope.newLocalScope(), ifblockInput);
 
         output.anyContinue = ifblockOutput.anyContinue;
         output.anyBreak = ifblockOutput.anyBreak;
@@ -85,7 +85,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         elseblockInput.inLoop = input.inLoop;
         elseblockInput.lastLoop = input.lastLoop;
 
-        Output elseblockOutput = elseblock.analyze(scriptRoot, scope.newLocalScope(), elseblockInput);
+        Output elseblockOutput = elseblock.analyze(classNode, scriptRoot, scope.newLocalScope(), elseblockInput);
 
         output.methodEscape = ifblockOutput.methodEscape && elseblockOutput.methodEscape;
         output.loopEscape = ifblockOutput.loopEscape && elseblockOutput.loopEscape;
@@ -94,20 +94,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         output.anyBreak |= elseblockOutput.anyBreak;
         output.statementCount = Math.max(ifblockOutput.statementCount, elseblockOutput.statementCount);
 
-        return output;
-    }
-
-    @Override
-    IfElseNode write(ClassNode classNode) {
         IfElseNode ifElseNode = new IfElseNode();
-
-        ifElseNode.setConditionNode(condition.cast(condition.write(classNode)));
-        ifElseNode.setBlockNode(ifblock.write(classNode));
-        ifElseNode.setElseBlockNode(elseblock.write(classNode));
-
+        ifElseNode.setConditionNode(condition.cast(conditionOutput));
+        ifElseNode.setBlockNode((BlockNode)ifblockOutput.statementNode);
+        ifElseNode.setElseBlockNode((BlockNode)elseblockOutput.statementNode);
         ifElseNode.setLocation(location);
 
-        return ifElseNode;
+        output.statementNode = ifElseNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index 23a5d4183edbc..6ab6c6590e350 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -29,9 +29,9 @@
 /**
  * Represents a return statement.
  */
-public final class SReturn extends AStatement {
+public class SReturn extends AStatement {
 
-    private AExpression expression;
+    protected final AExpression expression;
 
     public SReturn(Location location, AExpression expression) {
         super(location);
@@ -40,9 +40,10 @@ public SReturn(Location location, AExpression expression) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
+
+        AExpression.Output expressionOutput = null;
 
         if (expression == null) {
             if (scope.getReturnType() != void.class) {
@@ -54,8 +55,8 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             AExpression.Input expressionInput = new AExpression.Input();
             expressionInput.expected = scope.getReturnType();
             expressionInput.internal = true;
-            expression.analyze(scriptRoot, scope, expressionInput);
-            expression.cast();
+            expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+            expression.cast(expressionInput, expressionOutput);
         }
 
         output.methodEscape = true;
@@ -64,18 +65,13 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    ReturnNode write(ClassNode classNode) {
         ReturnNode returnNode = new ReturnNode();
-
-        returnNode.setExpressionNode(expression == null ? null : expression.cast(expression.write(classNode)));
-
+        returnNode.setExpressionNode(expression == null ? null : expression.cast(expressionOutput));
         returnNode.setLocation(location);
 
-        return returnNode;
+        output.statementNode = returnNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
index 5e1846457d515..fef81dfed6968 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.Scope.Variable;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ForEachSubArrayNode;
 import org.elasticsearch.painless.lookup.PainlessCast;
@@ -33,46 +34,34 @@
 /**
  * Represents a for-each loop for arrays.
  */
-final class SSubEachArray extends AStatement {
-    private final Variable variable;
-    private AExpression expression;
-    private final SBlock block;
+public class SSubEachArray extends AStatement {
 
-    private PainlessCast cast = null;
-    private Variable array = null;
-    private Variable index = null;
-    private Class<?> indexed = null;
+    protected final Variable variable;
+    protected final AExpression.Output expressionOutput;
+    protected final Output blockOutput;
 
-    SSubEachArray(Location location, Variable variable, AExpression expression, SBlock block) {
+    SSubEachArray(Location location, Variable variable, AExpression.Output expressionOutput, Output blockOutput) {
         super(location);
 
         this.variable = Objects.requireNonNull(variable);
-        this.expression = Objects.requireNonNull(expression);
-        this.block = block;
+        this.expressionOutput = Objects.requireNonNull(expressionOutput);
+        this.blockOutput = blockOutput;
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         // We must store the array and index as variables for securing slots on the stack, and
         // also add the location offset to make the names unique in case of nested for each loops.
-        array = scope.defineInternalVariable(location, expression.output.actual, "array" + location.getOffset(), true);
-        index = scope.defineInternalVariable(location, int.class, "index" + location.getOffset(), true);
-        indexed = expression.output.actual.getComponentType();
-        cast = AnalyzerCaster.getLegalCast(location, indexed, variable.getType(), true, true);
+        Variable array = scope.defineVariable(location, expressionOutput.actual, "#array" + location.getOffset(), true);
+        Variable index = scope.defineVariable(location, int.class, "#index" + location.getOffset(), true);
+        Class<?> indexed = expressionOutput.actual.getComponentType();
+        PainlessCast cast = AnalyzerCaster.getLegalCast(location, indexed, variable.getType(), true, true);
 
-        return output;
-    }
-
-    @Override
-    ForEachSubArrayNode write(ClassNode classNode) {
         ForEachSubArrayNode forEachSubArrayNode = new ForEachSubArrayNode();
-
-        forEachSubArrayNode.setConditionNode(expression.write(classNode));
-        forEachSubArrayNode.setBlockNode(block.write(classNode));
-
+        forEachSubArrayNode.setConditionNode(expressionOutput.expressionNode);
+        forEachSubArrayNode.setBlockNode((BlockNode)blockOutput.statementNode);
         forEachSubArrayNode.setLocation(location);
         forEachSubArrayNode.setVariableType(variable.getType());
         forEachSubArrayNode.setVariableName(variable.getName());
@@ -84,11 +73,14 @@ ForEachSubArrayNode write(ClassNode classNode) {
         forEachSubArrayNode.setIndexedType(indexed);
         forEachSubArrayNode.setContinuous(false);
 
-        return forEachSubArrayNode;
+        output.statementNode = forEachSubArrayNode;
+
+        return output;
     }
 
     @Override
     public String toString() {
-        return singleLineToString(variable.getCanonicalTypeName(), variable.getName(), expression, block);
+        //return singleLineToString(variable.getCanonicalTypeName(), variable.getName(), expression, block);
+        return null;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index 4198452ff8e79..83561431db1b1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
 import org.elasticsearch.painless.Scope.Variable;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.ForEachSubIterableNode;
 import org.elasticsearch.painless.lookup.PainlessCast;
@@ -38,56 +39,46 @@
 /**
  * Represents a for-each loop for iterables.
  */
-final class SSubEachIterable extends AStatement {
+public class SSubEachIterable extends AStatement {
 
-    private AExpression expression;
-    private final SBlock block;
-    private final Variable variable;
+    protected final Variable variable;
+    protected final AExpression.Output expressionOutput;
+    protected final Output blockOutput;
 
-    private PainlessCast cast = null;
-    private Variable iterator = null;
-    private PainlessMethod method = null;
-
-    SSubEachIterable(Location location, Variable variable, AExpression expression, SBlock block) {
+    SSubEachIterable(Location location, Variable variable, AExpression.Output expressionOutput, Output blockOutput) {
         super(location);
 
         this.variable = Objects.requireNonNull(variable);
-        this.expression = Objects.requireNonNull(expression);
-        this.block = block;
+        this.expressionOutput = Objects.requireNonNull(expressionOutput);
+        this.blockOutput = blockOutput;
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         // We must store the iterator as a variable for securing a slot on the stack, and
         // also add the location offset to make the name unique in case of nested for each loops.
-        iterator = scope.defineInternalVariable(location, Iterator.class, "itr" + location.getOffset(), true);
+        Variable iterator = scope.defineVariable(location, Iterator.class, "#itr" + location.getOffset(), true);
+
+        PainlessMethod method;
 
-        if (expression.output.actual == def.class) {
+        if (expressionOutput.actual == def.class) {
             method = null;
         } else {
-            method = scriptRoot.getPainlessLookup().lookupPainlessMethod(expression.output.actual, false, "iterator", 0);
+            method = scriptRoot.getPainlessLookup().lookupPainlessMethod(expressionOutput.actual, false, "iterator", 0);
 
             if (method == null) {
                     throw createError(new IllegalArgumentException(
-                            "method [" + typeToCanonicalTypeName(expression.output.actual) + ", iterator/0] not found"));
+                            "method [" + typeToCanonicalTypeName(expressionOutput.actual) + ", iterator/0] not found"));
             }
         }
 
-        cast = AnalyzerCaster.getLegalCast(location, def.class, variable.getType(), true, true);
-
-        return output;
-    }
+        PainlessCast cast = AnalyzerCaster.getLegalCast(location, def.class, variable.getType(), true, true);
 
-    @Override
-    ForEachSubIterableNode write(ClassNode classNode) {
         ForEachSubIterableNode forEachSubIterableNode = new ForEachSubIterableNode();
-
-        forEachSubIterableNode.setConditionNode(expression.write(classNode));
-        forEachSubIterableNode.setBlockNode(block.write(classNode));
-
+        forEachSubIterableNode.setConditionNode(expressionOutput.expressionNode);
+        forEachSubIterableNode.setBlockNode((BlockNode)blockOutput.statementNode);
         forEachSubIterableNode.setLocation(location);
         forEachSubIterableNode.setVariableType(variable.getType());
         forEachSubIterableNode.setVariableName(variable.getName());
@@ -97,11 +88,14 @@ ForEachSubIterableNode write(ClassNode classNode) {
         forEachSubIterableNode.setMethod(method);
         forEachSubIterableNode.setContinuous(false);
 
-        return forEachSubIterableNode;
+        output.statementNode = forEachSubIterableNode;
+
+        return output;
     }
 
     @Override
     public String toString() {
-        return singleLineToString(variable.getCanonicalTypeName(), variable.getName(), expression, block);
+        //return singleLineToString(variable.getCanonicalTypeName(), variable.getName(), expression, block);
+        return null;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index 86b40fafa378b..3fff9615b12d9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -30,9 +30,9 @@
 /**
  * Represents a throw statement.
  */
-public final class SThrow extends AStatement {
+public class SThrow extends AStatement {
 
-    private AExpression expression;
+    protected final AExpression expression;
 
     public SThrow(Location location, AExpression expression) {
         super(location);
@@ -41,32 +41,26 @@ public SThrow(Location location, AExpression expression) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         AExpression.Input expressionInput = new AExpression.Input();
         expressionInput.expected = Exception.class;
-        expression.analyze(scriptRoot, scope, expressionInput);
-        expression.cast();
+        AExpression.Output expressionOutput = expression.analyze(classNode, scriptRoot, scope, expressionInput);
+        expression.cast(expressionInput, expressionOutput);
 
         output.methodEscape = true;
         output.loopEscape = true;
         output.allEscape = true;
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    ThrowNode write(ClassNode classNode) {
         ThrowNode throwNode = new ThrowNode();
-
-        throwNode.setExpressionNode(expression.cast(expression.write(classNode)));
-
+        throwNode.setExpressionNode(expression.cast(expressionOutput));
         throwNode.setLocation(location);
 
-        return throwNode;
+        output.statementNode = throwNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 7798fd13b13a2..b9fd59f6b3fc1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -21,10 +21,13 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
+import org.elasticsearch.painless.ir.CatchNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.TryNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -33,10 +36,10 @@
 /**
  * Represents the try block as part of a try-catch block.
  */
-public final class STry extends AStatement {
+public class STry extends AStatement {
 
-    private final SBlock block;
-    private final List<SCatch> catches;
+    protected final SBlock block;
+    protected final List<SCatch> catches;
 
     public STry(Location location, SBlock block, List<SCatch> catches) {
         super(location);
@@ -46,9 +49,8 @@ public STry(Location location, SBlock block, List<SCatch> catches) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
 
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous try statement."));
@@ -59,7 +61,7 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
         blockInput.inLoop = input.inLoop;
         blockInput.lastLoop = input.lastLoop;
 
-        Output blockOutput = block.analyze(scriptRoot, scope.newLocalScope(), blockInput);
+        Output blockOutput = block.analyze(classNode, scriptRoot, scope.newLocalScope(), blockInput);
 
         output.methodEscape = blockOutput.methodEscape;
         output.loopEscape = blockOutput.loopEscape;
@@ -69,13 +71,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         int statementCount = 0;
 
+        List<Output> catchOutputs = new ArrayList<>();
+
         for (SCatch catc : catches) {
             Input catchInput = new Input();
             catchInput.lastSource = input.lastSource;
             catchInput.inLoop = input.inLoop;
             catchInput.lastLoop = input.lastLoop;
 
-            Output catchOutput = catc.analyze(scriptRoot, scope.newLocalScope(), catchInput);
+            Output catchOutput = catc.analyze(classNode, scriptRoot, scope.newLocalScope(), catchInput);
 
             output.methodEscape &= catchOutput.methodEscape;
             output.loopEscape &= catchOutput.loopEscape;
@@ -83,27 +87,25 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             output.anyContinue |= catchOutput.anyContinue;
             output.anyBreak |= catchOutput.anyBreak;
 
+            catchOutputs.add(catchOutput);
+
             statementCount = Math.max(statementCount, catchOutput.statementCount);
         }
 
         output.statementCount = blockOutput.statementCount + statementCount;
 
-        return output;
-    }
-
-    @Override
-    TryNode write(ClassNode classNode) {
         TryNode tryNode = new TryNode();
 
-        for (SCatch catc : catches) {
-            tryNode.addCatchNode(catc.write(classNode));
+        for (Output catchOutput : catchOutputs) {
+            tryNode.addCatchNode((CatchNode)catchOutput.statementNode);
         }
 
-        tryNode.setBlockNode(block.write(classNode));
-
+        tryNode.setBlockNode((BlockNode)blockOutput.statementNode);
         tryNode.setLocation(location);
 
-        return tryNode;
+        output.statementNode = tryNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index eead51786e2d3..9fa3139b87eed 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Scope;
+import org.elasticsearch.painless.ir.BlockNode;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.ir.WhileNode;
 import org.elasticsearch.painless.symbol.ScriptRoot;
@@ -30,12 +31,10 @@
 /**
  * Represents a while loop.
  */
-public final class SWhile extends AStatement {
+public class SWhile extends AStatement {
 
-    private AExpression condition;
-    private final SBlock block;
-
-    private boolean continuous = false;
+    protected final AExpression condition;
+    protected final SBlock block;
 
     public SWhile(Location location, AExpression condition, SBlock block) {
         super(location);
@@ -45,16 +44,16 @@ public SWhile(Location location, AExpression condition, SBlock block) {
     }
 
     @Override
-    Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
-        this.input = input;
-        output = new Output();
-
+    Output analyze(ClassNode classNode, ScriptRoot scriptRoot, Scope scope, Input input) {
+        Output output = new Output();
         scope = scope.newLocalScope();
 
         AExpression.Input conditionInput = new AExpression.Input();
         conditionInput.expected = boolean.class;
-        condition.analyze(scriptRoot, scope, conditionInput);
-        condition.cast();
+        AExpression.Output conditionOutput = condition.analyze(classNode, scriptRoot, scope, conditionInput);
+        condition.cast(conditionInput, conditionOutput);
+
+        boolean continuous = false;
 
         if (condition instanceof EBoolean) {
             continuous = ((EBoolean)condition).constant;
@@ -68,12 +67,14 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
             }
         }
 
+        Output blockOutput = null;
+
         if (block != null) {
             Input blockInput = new Input();
             blockInput.beginLoop = true;
             blockInput.inLoop = true;
 
-            Output blockOutput = block.analyze(scriptRoot, scope, blockInput);
+            blockOutput = block.analyze(classNode, scriptRoot, scope, blockInput);
 
             if (blockOutput.loopEscape && blockOutput.anyContinue == false) {
                 throw createError(new IllegalArgumentException("Extraneous while loop."));
@@ -89,20 +90,15 @@ Output analyze(ScriptRoot scriptRoot, Scope scope, Input input) {
 
         output.statementCount = 1;
 
-        return output;
-    }
-
-    @Override
-    WhileNode write(ClassNode classNode) {
         WhileNode whileNode = new WhileNode();
-
-        whileNode.setConditionNode(condition.cast(condition.write(classNode)));
-        whileNode.setBlockNode(block == null ? null : block.write(classNode));
-
+        whileNode.setConditionNode(condition.cast(conditionOutput));
+        whileNode.setBlockNode(blockOutput == null ? null : (BlockNode)blockOutput.statementNode);
         whileNode.setLocation(location);
         whileNode.setContinuous(continuous);
 
-        return whileNode;
+        output.statementNode = whileNode;
+
+        return output;
     }
 
     @Override
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index 0a7c0cdcd0d7e..0cee95adef5fd 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 public class DefOptimizationTests extends ScriptTestCase {
+
     public void testIntBraceArrayOptiLoad() {
         final String script = "int x = 0; def y = new int[1]; y[0] = 5; x = y[0]; return x;";
         assertBytecodeExists(script, "INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I");
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
index bdc683125eb68..70e754a4168f5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.painless.FeatureTestObject;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
-import org.elasticsearch.painless.Scope.Variable;
 import org.elasticsearch.painless.ScriptClassInfo;
 import org.elasticsearch.painless.action.PainlessExecuteAction;
 import org.elasticsearch.painless.antlr.Walker;
@@ -749,30 +748,6 @@ public void testSIfElse() {
               + "}");
     }
 
-    public void testSSubEachArray() {
-        Location l = new Location(getTestName(), 0);
-        Variable v = new Variable(int.class, "test", false);
-        AExpression e = new ENewArray(l, "int", Arrays.asList(new EConstant(l, 1), new EConstant(l, 2), new EConstant(l, 3)), true);
-        SBlock b = new SBlock(l, singletonList(new SReturn(l, new EConstant(l, 5))));
-        SSubEachArray node = new SSubEachArray(l, v, e, b);
-        assertEquals(
-                "(SSubEachArray int test (ENewArray int init (Args (EConstant Integer 1) (EConstant Integer 2) (EConstant Integer 3))) "
-              + "(SBlock (SReturn (EConstant Integer 5))))",
-                node.toString());
-    }
-
-    public void testSSubEachIterable() {
-        Location l = new Location(getTestName(), 0);
-        Variable v = new Variable(int.class, "test", false);
-        AExpression e = new EListInit(l, Arrays.asList(new EConstant(l, 1), new EConstant(l, 2), new EConstant(l, 3)));
-        SBlock b = new SBlock(l, singletonList(new SReturn(l, new EConstant(l, 5))));
-        SSubEachIterable node = new SSubEachIterable(l, v, e, b);
-        assertEquals(
-                  "(SSubEachIterable int test (EListInit (EConstant Integer 1) (EConstant Integer 2) (EConstant Integer 3)) (SBlock "
-                + "(SReturn (EConstant Integer 5))))",
-                  node.toString());
-    }
-
     public void testSThrow() {
         assertToString("(SClass (SThrow (ENewObj RuntimeException)))", "throw new RuntimeException()");
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index 2839f8a822cda..d5afc38e702d5 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -247,16 +247,16 @@ public void start() {
     void onScrollResponse(ScrollableHitSource.AsyncResponse asyncResponse) {
         // lastBatchStartTime is essentially unused (see WorkerBulkByScrollTaskState.throttleWaitTime. Leaving it for now, since it seems
         // like a bug?
-        onScrollResponse(new TimeValue(System.nanoTime()), this.lastBatchSize, asyncResponse);
+        onScrollResponse(System.nanoTime(), this.lastBatchSize, asyncResponse);
     }
 
     /**
      * Process a scroll response.
-     * @param lastBatchStartTime the time when the last batch started. Used to calculate the throttling delay.
+     * @param lastBatchStartTimeNS the time when the last batch started. Used to calculate the throttling delay.
      * @param lastBatchSize the size of the last batch. Used to calculate the throttling delay.
      * @param asyncResponse the response to process from ScrollableHitSource
      */
-    void onScrollResponse(TimeValue lastBatchStartTime, int lastBatchSize, ScrollableHitSource.AsyncResponse asyncResponse) {
+    void onScrollResponse(long lastBatchStartTimeNS, int lastBatchSize, ScrollableHitSource.AsyncResponse asyncResponse) {
         ScrollableHitSource.Response response = asyncResponse.response();
         logger.debug("[{}]: got scroll response with [{}] hits", task.getId(), response.getHits().size());
         if (task.isCancelled()) {
@@ -284,7 +284,7 @@ protected void doRun() throws Exception {
                  * It is important that the batch start time be calculated from here, scroll response to scroll response. That way the time
                  * waiting on the scroll doesn't count against this batch in the throttle.
                  */
-                prepareBulkRequest(timeValueNanos(System.nanoTime()), asyncResponse);
+                prepareBulkRequest(System.nanoTime(), asyncResponse);
             }
 
             @Override
@@ -293,7 +293,7 @@ public void onFailure(Exception e) {
             }
         };
         prepareBulkRequestRunnable = (AbstractRunnable) threadPool.getThreadContext().preserveContext(prepareBulkRequestRunnable);
-        worker.delayPrepareBulkRequest(threadPool, lastBatchStartTime, lastBatchSize, prepareBulkRequestRunnable);
+        worker.delayPrepareBulkRequest(threadPool, lastBatchStartTimeNS, lastBatchSize, prepareBulkRequestRunnable);
     }
 
     /**
@@ -301,7 +301,7 @@ public void onFailure(Exception e) {
      * delay has been slept. Uses the generic thread pool because reindex is rare enough not to need its own thread pool and because the
      * thread may be blocked by the user script.
      */
-    void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.AsyncResponse asyncResponse) {
+    void prepareBulkRequest(long thisBatchStartTimeNS, ScrollableHitSource.AsyncResponse asyncResponse) {
         ScrollableHitSource.Response response = asyncResponse.response();
         logger.debug("[{}]: preparing bulk request", task.getId());
         if (task.isCancelled()) {
@@ -327,12 +327,12 @@ void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.AsyncR
             /*
              * If we noop-ed the entire batch then just skip to the next batch or the BulkRequest would fail validation.
              */
-            notifyDone(thisBatchStartTime, asyncResponse, 0);
+            notifyDone(thisBatchStartTimeNS, asyncResponse, 0);
             return;
         }
         request.timeout(mainRequest.getTimeout());
         request.waitForActiveShards(mainRequest.getWaitForActiveShards());
-        sendBulkRequest(request, () -> notifyDone(thisBatchStartTime, asyncResponse, request.requests().size()));
+        sendBulkRequest(request, () -> notifyDone(thisBatchStartTimeNS, asyncResponse, request.requests().size()));
     }
 
     /**
@@ -418,14 +418,14 @@ void onBulkResponse(BulkResponse response, Runnable onSuccess) {
         }
     }
 
-    void notifyDone(TimeValue thisBatchStartTime, ScrollableHitSource.AsyncResponse asyncResponse, int batchSize) {
+    void notifyDone(long thisBatchStartTimeNS, ScrollableHitSource.AsyncResponse asyncResponse, int batchSize) {
         if (task.isCancelled()) {
             logger.debug("[{}]: finishing early because the task was cancelled", task.getId());
             finishHim(null);
             return;
         }
         this.lastBatchSize = batchSize;
-        asyncResponse.done(worker.throttleWaitTime(thisBatchStartTime, timeValueNanos(System.nanoTime()), batchSize));
+        asyncResponse.done(worker.throttleWaitTime(thisBatchStartTimeNS, System.nanoTime(), batchSize));
     }
 
     private void recordFailure(Failure failure, List<Failure> failures) {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index 9a0565561b713..1217a7cd252f1 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -111,7 +111,6 @@
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
 import static org.elasticsearch.action.bulk.BackoffPolicy.constantBackoff;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
-import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
@@ -255,7 +254,7 @@ public void testScrollResponseSetsTotal() {
 
         long total = randomIntBetween(0, Integer.MAX_VALUE);
         ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), total, emptyList(), null);
-        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueSeconds(0), 0, response);
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), 0, 0, response);
         assertEquals(total, testTask.getStatus().getTotal());
     }
 
@@ -268,7 +267,7 @@ public void testScrollResponseBatchingBehavior() throws Exception {
             Hit hit = new ScrollableHitSource.BasicHit("index", "id", 0);
             ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 1, singletonList(hit), null);
             DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
-            simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 0, response);
+            simulateScrollResponse(action, System.nanoTime(), 0, response);
 
             // Use assert busy because the update happens on another thread
             final int expectedBatches = batches;
@@ -354,7 +353,7 @@ public ScheduledCancellable schedule(Runnable command, TimeValue delay, String n
             }
         });
         ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 0, emptyList(), null);
-        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 10, response);
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), System.nanoTime(), 10, response);
         ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
         assertThat(e.getCause(), instanceOf(EsRejectedExecutionException.class));
         assertThat(e.getCause(), hasToString(containsString("test")));
@@ -372,7 +371,7 @@ public void testShardFailuresAbortRequest() throws Exception {
         SearchFailure shardFailure = new SearchFailure(new RuntimeException("test"));
         ScrollableHitSource.Response scrollResponse = new ScrollableHitSource.Response(false, singletonList(shardFailure), 0,
                 emptyList(), null);
-        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), System.nanoTime(), 0, scrollResponse);
         BulkByScrollResponse response = listener.get();
         assertThat(response.getBulkFailures(), empty());
         assertThat(response.getSearchFailures(), contains(shardFailure));
@@ -386,7 +385,7 @@ public void testShardFailuresAbortRequest() throws Exception {
      */
     public void testSearchTimeoutsAbortRequest() throws Exception {
         ScrollableHitSource.Response scrollResponse = new ScrollableHitSource.Response(true, emptyList(), 0, emptyList(), null);
-        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), System.nanoTime(), 0, scrollResponse);
         BulkByScrollResponse response = listener.get();
         assertThat(response.getBulkFailures(), empty());
         assertThat(response.getSearchFailures(), empty());
@@ -423,7 +422,7 @@ protected AbstractAsyncBulkByScrollAction.RequestWrapper<?> buildRequest(Hit doc
         ScrollableHitSource.BasicHit hit = new ScrollableHitSource.BasicHit("index", "id", 0);
         hit.setSource(new BytesArray("{}"), XContentType.JSON);
         ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 1, singletonList(hit), null);
-        simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 0, response);
+        simulateScrollResponse(action, System.nanoTime(), 0, response);
         ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
         assertThat(e.getCause(), instanceOf(RuntimeException.class));
         assertThat(e.getCause().getMessage(), equalTo("surprise"));
@@ -509,7 +508,7 @@ protected RequestWrapper<?> buildRequest(Hit doc) {
             new TotalHits(0, TotalHits.Relation.EQUAL_TO),0);
         InternalSearchResponse internalResponse = new InternalSearchResponse(hits, null, null, null, false, false, 1);
         SearchResponse searchResponse = new SearchResponse(internalResponse, scrollId(), 5, 4, 0, randomLong(), null,
-            SearchResponse.Clusters.EMPTY);
+            SearchResponse.Clusters.EMPTY, null);
 
         client.lastSearch.get().listener.onResponse(searchResponse);
 
@@ -619,7 +618,7 @@ public void testCancelBeforeInitialSearch() throws Exception {
     }
 
     public void testCancelBeforeScrollResponse() throws Exception {
-        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 1,
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> simulateScrollResponse(action, System.nanoTime(), 1,
                 new ScrollableHitSource.Response(false, emptyList(), between(1, 100000), emptyList(), null)));
     }
 
@@ -634,7 +633,7 @@ public void testCancelBeforeOnBulkResponse() throws Exception {
     }
 
     public void testCancelBeforeStartNextScroll() throws Exception {
-        TimeValue now = timeValueNanos(System.nanoTime());
+        long now = System.nanoTime();
         cancelTaskCase((DummyAsyncBulkByScrollAction action) -> action.notifyDone(now, null, 0));
     }
 
@@ -683,7 +682,7 @@ public ScheduledCancellable schedule(Runnable command, TimeValue delay, String n
         ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), total, emptyList(), null);
         // Use a long delay here so the test will time out if the cancellation doesn't reschedule the throttled task
         worker.rethrottle(1);
-        simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 1000, response);
+        simulateScrollResponse(action, System.nanoTime(), 1000, response);
 
         // Now that we've got our cancel we'll just verify that it all came through all right
         assertEquals(reason, listener.get(10, TimeUnit.SECONDS).getReasonCancelled());
@@ -712,7 +711,7 @@ private void cancelTaskCase(Consumer<DummyAsyncBulkByScrollAction> testMe) throw
     /**
      * Simulate a scroll response by setting the scroll id and firing the onScrollResponse method.
      */
-    private void simulateScrollResponse(DummyAsyncBulkByScrollAction action, TimeValue lastBatchTime, int lastBatchSize,
+    private void simulateScrollResponse(DummyAsyncBulkByScrollAction action, long lastBatchTime, int lastBatchSize,
             ScrollableHitSource.Response response) {
         action.setScroll(scrollId());
         action.onScrollResponse(lastBatchTime, lastBatchSize, new ScrollableHitSource.AsyncResponse() {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ClientScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ClientScrollableHitSourceTests.java
index d9e39b17a251d..75aa764aae9cf 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ClientScrollableHitSourceTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ClientScrollableHitSourceTests.java
@@ -164,7 +164,7 @@ private SearchResponse createSearchResponse() {
             new TotalHits(0, TotalHits.Relation.EQUAL_TO),0);
         InternalSearchResponse internalResponse = new InternalSearchResponse(hits, null, null, null, false, false, 1);
         return new SearchResponse(internalResponse, randomSimpleString(random(), 1, 10), 5, 4, 0, randomLong(), null,
-            SearchResponse.Clusters.EMPTY);
+            SearchResponse.Clusters.EMPTY, null);
     }
 
     private void assertSameHits(List<? extends ScrollableHitSource.Hit> actual, SearchHit[] expected) {
diff --git a/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java b/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java
index 0467b9419c778..b7d63991877db 100644
--- a/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java
+++ b/modules/tasks/src/main/java/org/elasticsearch/tasksplugin/TasksPlugin.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.tasksplugin;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
@@ -35,7 +34,7 @@
 public class TasksPlugin extends Plugin implements SystemIndexPlugin {
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.singletonList(new SystemIndexDescriptor(TASK_INDEX, this.getClass().getSimpleName()));
     }
 }
diff --git a/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java b/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java
index 23b873e377eb3..48ec1e06098f3 100644
--- a/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java
+++ b/modules/tasks/src/test/java/org/elasticsearch/tasksplugin/TasksPluginTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.tasksplugin;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
@@ -28,6 +27,6 @@ public class TasksPluginTests extends ESTestCase {
     public void testDummy() {
         // This is a dummy test case to satisfy the conventions
         TasksPlugin plugin = new TasksPlugin();
-        assertThat(plugin.getSystemIndexDescriptors(Settings.EMPTY), Matchers.hasSize(1));
+        assertThat(plugin.getSystemIndexDescriptors(), Matchers.hasSize(1));
     }
 }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java
deleted file mode 100644
index 3f7ff0d6e2a0f..0000000000000
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.transport.netty4;
-
-import io.netty.buffer.ByteBuf;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.bytes.AbstractBytesReference;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-
-final class ByteBufBytesReference extends AbstractBytesReference {
-
-    private final ByteBuf buffer;
-    private final int length;
-    private final int offset;
-
-    ByteBufBytesReference(ByteBuf buffer, int length) {
-        this.buffer = buffer;
-        this.length = length;
-        this.offset = buffer.readerIndex();
-        assert length <= buffer.readableBytes() : "length[" + length +"] > " + buffer.readableBytes();
-    }
-
-    @Override
-    public byte get(int index) {
-        return buffer.getByte(offset + index);
-    }
-
-    @Override
-    public int getInt(int index) {
-        return buffer.getInt(offset + index);
-    }
-
-    @Override
-    public int indexOf(byte marker, int from) {
-        final int start = offset + from;
-        return buffer.forEachByte(start, length - start, value -> value != marker);
-    }
-
-    @Override
-    public int length() {
-        return length;
-    }
-
-    @Override
-    public BytesReference slice(int from, int length) {
-        return new ByteBufBytesReference(buffer.slice(offset + from, length), length);
-    }
-
-    @Override
-    public StreamInput streamInput() {
-        return new ByteBufStreamInput(buffer.duplicate(), length);
-    }
-
-    @Override
-    public void writeTo(OutputStream os) throws IOException {
-        buffer.getBytes(offset, os, length);
-    }
-
-    ByteBuf toByteBuf() {
-        return buffer.duplicate();
-    }
-
-    @Override
-    public String utf8ToString() {
-        return buffer.toString(offset, length, StandardCharsets.UTF_8);
-    }
-
-    @Override
-    public BytesRef toBytesRef() {
-        if (buffer.hasArray()) {
-            return new BytesRef(buffer.array(), buffer.arrayOffset() + offset, length);
-        }
-        final byte[] copy = new byte[length];
-        buffer.getBytes(offset, copy);
-        return new BytesRef(copy);
-    }
-
-    @Override
-    public long ramBytesUsed() {
-        return buffer.capacity();
-    }
-
-}
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
index ba72f52b32a57..0f29bb0c139a9 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
@@ -80,28 +81,24 @@ public static ByteBuf toByteBuf(final BytesReference reference) {
         if (reference.length() == 0) {
             return Unpooled.EMPTY_BUFFER;
         }
-        if (reference instanceof ByteBufBytesReference) {
-            return ((ByteBufBytesReference) reference).toByteBuf();
-        } else {
-            final BytesRefIterator iterator = reference.iterator();
-            // usually we have one, two, or three components from the header, the message, and a buffer
-            final List<ByteBuf> buffers = new ArrayList<>(3);
-            try {
-                BytesRef slice;
-                while ((slice = iterator.next()) != null) {
-                    buffers.add(Unpooled.wrappedBuffer(slice.bytes, slice.offset, slice.length));
-                }
+        final BytesRefIterator iterator = reference.iterator();
+        // usually we have one, two, or three components from the header, the message, and a buffer
+        final List<ByteBuf> buffers = new ArrayList<>(3);
+        try {
+            BytesRef slice;
+            while ((slice = iterator.next()) != null) {
+                buffers.add(Unpooled.wrappedBuffer(slice.bytes, slice.offset, slice.length));
+            }
 
-                if (buffers.size() == 1) {
-                    return buffers.get(0);
-                } else {
-                    CompositeByteBuf composite = Unpooled.compositeBuffer(buffers.size());
-                    composite.addComponents(true, buffers);
-                    return composite;
-                }
-            } catch (IOException ex) {
-                throw new AssertionError("no IO happens here", ex);
+            if (buffers.size() == 1) {
+                return buffers.get(0);
+            } else {
+                CompositeByteBuf composite = Unpooled.compositeBuffer(buffers.size());
+                composite.addComponents(true, buffers);
+                return composite;
             }
+        } catch (IOException ex) {
+            throw new AssertionError("no IO happens here", ex);
         }
     }
 
@@ -112,7 +109,9 @@ public static BytesReference toBytesReference(final ByteBuf buffer) {
         final int readableBytes = buffer.readableBytes();
         if (readableBytes == 0) {
             return BytesArray.EMPTY;
+        } else {
+            final ByteBuffer[] byteBuffers = buffer.nioBuffers();
+            return BytesReference.fromByteBuffers(byteBuffers);
         }
-        return new ByteBufBytesReference(buffer, buffer.readableBytes());
     }
 }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java
deleted file mode 100644
index 4a41aaec952a0..0000000000000
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.transport.netty4;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
-
-import java.io.IOException;
-
-public class ByteBufBytesReferenceTests extends AbstractBytesReferenceTestCase {
-
-    @Override
-    protected BytesReference newBytesReference(int length) throws IOException {
-        return newBytesReferenceWithOffsetOfZero(length);
-    }
-
-    @Override
-    protected BytesReference newBytesReferenceWithOffsetOfZero(int length) throws IOException {
-        ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
-        for (int i = 0; i < length; i++) {
-            out.writeByte((byte) random().nextInt(1 << 8));
-        }
-        assertEquals(out.size(), length);
-        BytesReference ref = out.bytes();
-        assertEquals(ref.length(), length);
-        BytesRef bytesRef = ref.toBytesRef();
-        final ByteBuf buffer = Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset, bytesRef.length);
-        return Netty4Utils.toBytesReference(buffer);
-    }
-
-    public void testSliceOnAdvancedBuffer() throws IOException {
-        BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE));
-        BytesRef bytesRef = bytesReference.toBytesRef();
-        ByteBuf channelBuffer = Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset,
-            bytesRef.length);
-        int numBytesToRead = randomIntBetween(1, 5);
-        for (int i = 0; i < numBytesToRead; i++) {
-            channelBuffer.readByte();
-        }
-        BytesReference other = Netty4Utils.toBytesReference(channelBuffer);
-        BytesReference slice = bytesReference.slice(numBytesToRead, bytesReference.length() - numBytesToRead);
-        assertEquals(other, slice);
-        assertEquals(other.slice(3, 1), slice.slice(3, 1));
-    }
-
-    public void testImmutable() throws IOException {
-        BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE));
-        BytesRef bytesRef = BytesRef.deepCopyOf(bytesReference.toBytesRef());
-        ByteBuf channelBuffer = Unpooled.wrappedBuffer(bytesRef.bytes, bytesRef.offset,
-            bytesRef.length);
-        ByteBufBytesReference byteBufBytesReference = new ByteBufBytesReference(channelBuffer, bytesRef.length);
-        assertEquals(byteBufBytesReference, bytesReference);
-        channelBuffer.readInt(); // this advances the index of the channel buffer
-        assertEquals(byteBufBytesReference, bytesReference);
-        assertEquals(bytesRef, byteBufBytesReference.toBytesRef());
-
-        BytesRef unicodeBytes = new BytesRef(randomUnicodeOfCodepointLength(100));
-        channelBuffer = Unpooled.wrappedBuffer(unicodeBytes.bytes, unicodeBytes.offset, unicodeBytes.length);
-        byteBufBytesReference = new ByteBufBytesReference(channelBuffer, unicodeBytes.length);
-        String utf8ToString = byteBufBytesReference.utf8ToString();
-        channelBuffer.readInt(); // this advances the index of the channel buffer
-        assertEquals(utf8ToString, byteBufBytesReference.utf8ToString());
-    }
-}
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java
index 1fbbbe93e47d0..2b31cf68f01ee 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java
@@ -69,9 +69,7 @@ public void testToChannelBuffer() throws IOException {
         BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE));
         ByteBuf buffer = Netty4Utils.toByteBuf(ref);
         BytesReference bytesReference = Netty4Utils.toBytesReference(buffer);
-        if (ref instanceof ByteBufBytesReference) {
-            assertEquals(buffer, ((ByteBufBytesReference) ref).toByteBuf());
-        } else if (AbstractBytesReferenceTestCase.getNumPages(ref) > 1) { // we gather the buffers into a channel buffer
+        if (AbstractBytesReferenceTestCase.getNumPages(ref) > 1) { // we gather the buffers into a channel buffer
             assertTrue(buffer instanceof CompositeByteBuf);
         }
         assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference));
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index f639c17a35f5a..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b6f880fa08a44fcb2d50808f9eeb6189a293ce27
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..c2f0eb182c26e
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0.jar.sha1
@@ -0,0 +1 @@
+0697a7b06e4447be330f093e62d863deaadabc8c
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 5c1e0b2f6fa43..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9c5b8619795f69c225b5ec37b87cb34de0feccd4
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..9f20b12df3341
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0.jar.sha1
@@ -0,0 +1 @@
+b269efbdd16c28525942a27f5738c9d1348b6301
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 656f63c68d194..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-421e13b9fe09523e094ac708204d62d4ea5b6618
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..4ea0c218177ec
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0.jar.sha1
@@ -0,0 +1 @@
+056eef8f0a64a70cd9af070ecd8e17d33e55cb75
\ No newline at end of file
diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriNumberFilterFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriNumberFilterFactory.java
new file mode 100644
index 0000000000000..54a5ab9c2124e
--- /dev/null
+++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriNumberFilterFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.ko.KoreanNumberFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+
+public class NoriNumberFilterFactory extends AbstractTokenFilterFactory {
+
+    public NoriNumberFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new KoreanNumberFilter(tokenStream);
+    }
+}
diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java
index bac5dd2a77065..9680d6fd5f80a 100644
--- a/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java
+++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java
@@ -39,11 +39,13 @@ public class NoriTokenizerFactory extends AbstractTokenizerFactory {
 
     private final UserDictionary userDictionary;
     private final KoreanTokenizer.DecompoundMode decompoundMode;
+    private final boolean discardPunctuation;
 
     public NoriTokenizerFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, settings, name);
         decompoundMode = getMode(settings);
         userDictionary = getUserDictionary(env, settings);
+        discardPunctuation = settings.getAsBoolean("discard_punctuation", true);
     }
 
     public static UserDictionary getUserDictionary(Environment env, Settings settings) {
@@ -77,7 +79,8 @@ public static KoreanTokenizer.DecompoundMode getMode(Settings settings) {
 
     @Override
     public Tokenizer create() {
-        return new KoreanTokenizer(KoreanTokenizer.DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary, decompoundMode, false);
+        return new KoreanTokenizer(KoreanTokenizer.DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary, decompoundMode, false,
+            discardPunctuation);
     }
 
 }
diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java
index 6e9baa7acd26c..72097e2e83472 100644
--- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java
+++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.index.analysis.AnalyzerProvider;
 import org.elasticsearch.index.analysis.NoriAnalyzerProvider;
+import org.elasticsearch.index.analysis.NoriNumberFilterFactory;
 import org.elasticsearch.index.analysis.NoriPartOfSpeechStopFilterFactory;
 import org.elasticsearch.index.analysis.NoriReadingFormFilterFactory;
 import org.elasticsearch.index.analysis.NoriTokenizerFactory;
@@ -42,6 +43,7 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
         Map<String, AnalysisProvider<TokenFilterFactory>> extra = new HashMap<>();
         extra.put("nori_part_of_speech", NoriPartOfSpeechStopFilterFactory::new);
         extra.put("nori_readingform", NoriReadingFormFilterFactory::new);
+        extra.put("nori_number", NoriNumberFilterFactory::new);
         return extra;
     }
 
diff --git a/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/AnalysisNoriFactoryTests.java b/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/AnalysisNoriFactoryTests.java
index 1677ba94b8783..de70e26fc6f01 100644
--- a/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/AnalysisNoriFactoryTests.java
+++ b/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/AnalysisNoriFactoryTests.java
@@ -43,6 +43,7 @@ protected Map<String, Class<?>> getTokenFilters() {
         Map<String, Class<?>> filters = new HashMap<>(super.getTokenFilters());
         filters.put("koreanpartofspeechstop", NoriPartOfSpeechStopFilterFactory.class);
         filters.put("koreanreadingform", NoriReadingFormFilterFactory.class);
+        filters.put("koreannumber", NoriNumberFilterFactory.class);
         return filters;
     }
 }
diff --git a/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/NoriAnalysisTests.java b/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/NoriAnalysisTests.java
index 051a2f3e4dc32..87c78c7f981b9 100644
--- a/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/NoriAnalysisTests.java
+++ b/plugins/analysis-nori/src/test/java/org/elasticsearch/index/analysis/NoriAnalysisTests.java
@@ -54,6 +54,9 @@ public void testDefaultsNoriAnalysis() throws IOException {
         filterFactory = analysis.tokenFilter.get("nori_readingform");
         assertThat(filterFactory, instanceOf(NoriReadingFormFilterFactory.class));
 
+        filterFactory = analysis.tokenFilter.get("nori_number");
+        assertThat(filterFactory, instanceOf(NoriNumberFilterFactory.class));
+
         IndexAnalyzers indexAnalyzers = analysis.indexAnalyzers;
         NamedAnalyzer analyzer = indexAnalyzers.get("nori");
         assertThat(analyzer.analyzer(), instanceOf(KoreanAnalyzer.class));
@@ -130,6 +133,33 @@ public void testNoriTokenizer() throws Exception {
         assertTokenStreamContents(tokenizer, new String[] {"뿌리", "가", "깊", "은", "나무"});
         tokenizer.setReader(new StringReader("가늠표"));
         assertTokenStreamContents(tokenizer, new String[] {"가늠표", "가늠", "표"});
+        // discard_punctuation default(true)
+        tokenizer.setReader(new StringReader("3.2개"));
+        assertTokenStreamContents(tokenizer, new String[] {"3", "2", "개"});
+    }
+
+    public void testNoriTokenizerDiscardPunctuationOptionTrue() throws Exception {
+        Settings settings = createDiscardPunctuationOption("true");
+        TestAnalysis analysis = createTestAnalysis(settings);
+        Tokenizer tokenizer = analysis.tokenizer.get("my_tokenizer").create();
+        tokenizer.setReader(new StringReader("3.2개"));
+        assertTokenStreamContents(tokenizer, new String[] {"3", "2", "개"});
+    }
+
+    public void testNoriTokenizerDiscardPunctuationOptionFalse() throws Exception {
+        Settings settings = createDiscardPunctuationOption("false");
+        TestAnalysis analysis = createTestAnalysis(settings);
+        Tokenizer tokenizer = analysis.tokenizer.get("my_tokenizer").create();
+        tokenizer.setReader(new StringReader("3.2개"));
+        assertTokenStreamContents(tokenizer, new String[] {"3", ".", "2", "개"});
+    }
+
+    public void testNoriTokenizerInvalidDiscardPunctuationOption() {
+        String wrongOption = "wrong";
+        Settings settings = createDiscardPunctuationOption(wrongOption);
+        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings));
+        assertThat(exc.getMessage(), containsString("Failed to parse value [" + wrongOption
+            + "] as only [true] or [false] are allowed."));
     }
 
     public void testNoriPartOfSpeech() throws IOException {
@@ -159,6 +189,27 @@ public void testNoriReadingForm() throws IOException {
         assertTokenStreamContents(stream, new String[] {"향가"});
     }
 
+    public void testNoriNumber() throws IOException {
+        Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put("index.analysis.filter.my_filter.type", "nori_number")
+            .build();
+        TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new AnalysisNoriPlugin());
+        TokenFilterFactory factory = analysis.tokenFilter.get("my_filter");
+        Tokenizer tokenizer = new KoreanTokenizer();
+        tokenizer.setReader(new StringReader("오늘 십만이천오백원짜리 와인 구입"));
+        TokenStream stream = factory.create(tokenizer);
+        assertTokenStreamContents(stream, new String[] {"오늘", "102500", "원", "짜리", "와인", "구입"});
+    }
+
+    private Settings createDiscardPunctuationOption(String option) {
+        return Settings.builder()
+            .put("index.analysis.tokenizer.my_tokenizer.type", "nori_tokenizer")
+            .put("index.analysis.tokenizer.my_tokenizer.discard_punctuation", option)
+            .build();
+    }
+
     private TestAnalysis createTestAnalysis(Settings analysisSettings) throws IOException {
         InputStream dict = NoriAnalysisTests.class.getResourceAsStream("user_dict.txt");
         Path home = createTempDir();
diff --git a/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/10_basic.yml b/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/10_basic.yml
index a5aa9998da6ba..523874f5743bb 100644
--- a/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/10_basic.yml
+++ b/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/10_basic.yml
@@ -46,3 +46,20 @@
             filter:       [nori_readingform]
     - length: { tokens: 1 }
     - match:  { tokens.0.token: 향가 }
+---
+"Number filter":
+    - do:
+        indices.analyze:
+          body:
+            text:                   십만이천오백과 ３.２천
+            tokenizer:
+              type:                 nori_tokenizer
+              discard_punctuation:  false
+            filter:
+              - type:               nori_part_of_speech
+                stoptags:           ["SP"]
+              - type:               nori_number
+    - length: { tokens: 3 }
+    - match:  { tokens.0.token: "102500"}
+    - match:  { tokens.1.token: 과}
+    - match:  { tokens.2.token: "3200"}
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 951d97032f9fb..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ff4ae9f3f3b0bc497f98c9bc47e943525669fc99
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..b0e0f037d5571
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0.jar.sha1
@@ -0,0 +1 @@
+356e39f7b4b0cf8bc6a766a54ab6b93b11c89f6d
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 19846e43636c3..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd6430c037566cd3852b73b2ec31e59de24cfe58
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..faa11d7a87fb3
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0.jar.sha1
@@ -0,0 +1 @@
+6139d1cc3f51d6f5fc5ab9976dc5376b682d8332
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 147bbb3192d24..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd4ca22b151a98a21e255bc1c54f0fadfee5ca4d
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..62fe533fde854
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0.jar.sha1
@@ -0,0 +1 @@
+f6c1757e23f0cbad2e4d2eb39a12d9deff2e802b
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 86cee0ecdd039..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e85f94d2747ddb560af0bc4d15f0cde45cf3ff30
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..52115dc4ff819
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0.jar.sha1
@@ -0,0 +1 @@
+e647c3158a092df07ba4ac1b827623f45176ef48
\ No newline at end of file
diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle
index 668c4c397e814..32ea5b8134c19 100644
--- a/plugins/ingest-attachment/build.gradle
+++ b/plugins/ingest-attachment/build.gradle
@@ -25,9 +25,9 @@ esplugin {
 }
 
 versions << [
-  'tika'  : '1.22',
-  'pdfbox': '2.0.16',
-  'poi'   : '4.0.1',
+  'tika'  : '1.24',
+  'pdfbox': '2.0.19',
+  'poi'   : '4.1.2',
   'mime4j': '0.8.3'
 ]
 
@@ -64,17 +64,21 @@ dependencies {
   // MS Office
   compile "org.apache.poi:poi-scratchpad:${versions.poi}"
   // Apple iWork
-  compile 'org.apache.commons:commons-compress:1.18'
+  compile 'org.apache.commons:commons-compress:1.19'
   // Outlook documents
   compile "org.apache.james:apache-mime4j-core:${versions.mime4j}"
   compile "org.apache.james:apache-mime4j-dom:${versions.mime4j}"
   // EPUB books
   compile 'org.apache.commons:commons-lang3:3.9'
+  // Microsoft Word files with visio diagrams
+  compile 'org.apache.commons:commons-math3:3.6.1'
+  // POIs dependency
+  compile 'com.zaxxer:SparseBitSet:1.2'
 }
 
 restResources {
   restApi {
-    includeCore '_common', 'cluster', 'nodes', 'ingest', 'index', 'get' 
+    includeCore '_common', 'cluster', 'nodes', 'ingest', 'index', 'get'
   }
 }
 
diff --git a/plugins/ingest-attachment/licenses/SparseBitSet-1.2.jar.sha1 b/plugins/ingest-attachment/licenses/SparseBitSet-1.2.jar.sha1
new file mode 100644
index 0000000000000..5f1d015b87ac7
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/SparseBitSet-1.2.jar.sha1
@@ -0,0 +1 @@
+8467c813d442837fcaeddbc42cf5c5359fab4933
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/SparseBitSet-LICENSE.txt b/plugins/ingest-attachment/licenses/SparseBitSet-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/SparseBitSet-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/ingest-attachment/licenses/SparseBitSet-NOTICE.txt b/plugins/ingest-attachment/licenses/SparseBitSet-NOTICE.txt
new file mode 100644
index 0000000000000..d3d6e140ce4f3
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/SparseBitSet-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache Commons Logging
+Copyright 2003-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/plugins/ingest-attachment/licenses/commons-compress-1.18.jar.sha1 b/plugins/ingest-attachment/licenses/commons-compress-1.18.jar.sha1
deleted file mode 100644
index b4d4d68d9dc1a..0000000000000
--- a/plugins/ingest-attachment/licenses/commons-compress-1.18.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1191f9f2bc0c47a8cce69193feb1ff0a8bcb37d5
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-compress-1.19.jar.sha1 b/plugins/ingest-attachment/licenses/commons-compress-1.19.jar.sha1
new file mode 100644
index 0000000000000..045fc5c1a22e2
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-compress-1.19.jar.sha1
@@ -0,0 +1 @@
+7e65777fb451ddab6a9c054beb879e521b7eab78
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-math3-3.6.1.jar.sha1 b/plugins/ingest-attachment/licenses/commons-math3-3.6.1.jar.sha1
new file mode 100644
index 0000000000000..72975be4c8851
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-math3-3.6.1.jar.sha1
@@ -0,0 +1 @@
+e4ba98f1d4b3c80ec46392f25e094a6a2e58fcbf
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-math3-LICENSE.txt b/plugins/ingest-attachment/licenses/commons-math3-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-math3-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/ingest-attachment/licenses/commons-math3-NOTICE.txt b/plugins/ingest-attachment/licenses/commons-math3-NOTICE.txt
new file mode 100644
index 0000000000000..d3d6e140ce4f3
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-math3-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache Commons Logging
+Copyright 2003-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/plugins/ingest-attachment/licenses/fontbox-2.0.16.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-2.0.16.jar.sha1
deleted file mode 100644
index 9d3ceaff68341..0000000000000
--- a/plugins/ingest-attachment/licenses/fontbox-2.0.16.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3f7819279a0b90a01b07a870d1d27dffd8de24db
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/fontbox-2.0.19.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-2.0.19.jar.sha1
new file mode 100644
index 0000000000000..ac83b56e463ad
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/fontbox-2.0.19.jar.sha1
@@ -0,0 +1 @@
+b8b3873aacde51f1a3f4a052b236de828867d000
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/pdfbox-2.0.16.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-2.0.16.jar.sha1
deleted file mode 100644
index fd4245823e59d..0000000000000
--- a/plugins/ingest-attachment/licenses/pdfbox-2.0.16.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5dce5e41fc472d02800df5ef060a1f3a58c36902
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/pdfbox-2.0.19.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-2.0.19.jar.sha1
new file mode 100644
index 0000000000000..14757fabc207c
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/pdfbox-2.0.19.jar.sha1
@@ -0,0 +1 @@
+8dfcfc278b4b66e1a8b0e9e681b84ffe48da2c21
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-4.0.1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-4.0.1.jar.sha1
deleted file mode 100644
index bce6d5f9d9ab2..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-4.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d8eaa341687a7dc48048d964d0d53238959ca9b5
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-4.1.2.jar.sha1 b/plugins/ingest-attachment/licenses/poi-4.1.2.jar.sha1
new file mode 100644
index 0000000000000..dae936314bde1
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-4.1.2.jar.sha1
@@ -0,0 +1 @@
+964bf41cf68bce08e4ef6b2279b559fdf8d454f4
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-4.0.1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-4.0.1.jar.sha1
deleted file mode 100644
index 1edaed3b1fa37..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-ooxml-4.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9ec84728bf4236b8f9ec7fef3fe1b705eef2b408
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-4.1.2.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-4.1.2.jar.sha1
new file mode 100644
index 0000000000000..b42c94ac3b33e
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-ooxml-4.1.2.jar.sha1
@@ -0,0 +1 @@
+87d9a22aa9a7dd26e80c360e709f7ee02e32ab3b
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-4.0.1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-4.0.1.jar.sha1
deleted file mode 100644
index d3022b4052d53..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-4.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d2a066340008d36cb289b71f0f7b6ad562940644
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-4.1.2.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-4.1.2.jar.sha1
new file mode 100644
index 0000000000000..852e018a5970d
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-4.1.2.jar.sha1
@@ -0,0 +1 @@
+550cc22a598c0b0a51d1f55f8371e83c1229802d
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-scratchpad-4.0.1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-scratchpad-4.0.1.jar.sha1
deleted file mode 100644
index 5bf9aedc34cdd..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-scratchpad-4.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-89b1ce1b932338204ffa3fab225b65b5d33dab5d
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-scratchpad-4.1.2.jar.sha1 b/plugins/ingest-attachment/licenses/poi-scratchpad-4.1.2.jar.sha1
new file mode 100644
index 0000000000000..ed3d36c955a9e
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-scratchpad-4.1.2.jar.sha1
@@ -0,0 +1 @@
+1be379e91d3d3fb0cd11425451acdbfb0d2264e7
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-core-1.22.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-1.22.jar.sha1
deleted file mode 100644
index d988ea2ebd192..0000000000000
--- a/plugins/ingest-attachment/licenses/tika-core-1.22.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b193f1f977e64ff77025a4cecd7997cff344c4bc
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-core-1.24.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-1.24.jar.sha1
new file mode 100644
index 0000000000000..6579753564ddd
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-core-1.24.jar.sha1
@@ -0,0 +1 @@
+ed2e85f034743ea8e7ff25ab594caf361c408a44
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parsers-1.22.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-1.22.jar.sha1
deleted file mode 100644
index 893a7115e7658..0000000000000
--- a/plugins/ingest-attachment/licenses/tika-parsers-1.22.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b8a823128f6165882ae41de3ded8655609d62d88
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parsers-1.24.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-1.24.jar.sha1
new file mode 100644
index 0000000000000..6ce939379f830
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parsers-1.24.jar.sha1
@@ -0,0 +1 @@
+282c0ecd31cb235f7d96bef18b8cdcb56573a195
\ No newline at end of file
diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
index 3e5cd7758d22c..b594ff3b172f2 100644
--- a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
+++ b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
@@ -88,5 +88,7 @@ testClusters.integTest {
     // in a hacky way to change the protocol and endpoint. We must fix that.
     setting 'azure.client.integration_test.endpoint_suffix',
       { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=${-> azureAddress()}" }, IGNORE_VALUE
+    String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0)
+    setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE
   }
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
index 6bd480f7923a9..2093139e115a3 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
@@ -23,12 +23,17 @@
 import com.microsoft.azure.storage.StorageException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.DeleteResult;
 import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -37,18 +42,20 @@
 import java.nio.file.NoSuchFileException;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
+import java.util.concurrent.ExecutorService;
 
 public class AzureBlobContainer extends AbstractBlobContainer {
 
     private final Logger logger = LogManager.getLogger(AzureBlobContainer.class);
     private final AzureBlobStore blobStore;
+    private final ThreadPool threadPool;
     private final String keyPath;
 
-    AzureBlobContainer(BlobPath path, AzureBlobStore blobStore) {
+    AzureBlobContainer(BlobPath path, AzureBlobStore blobStore, ThreadPool threadPool) {
         super(path);
         this.blobStore = blobStore;
         this.keyPath = path.buildAsString();
+        this.threadPool = threadPool;
     }
 
     private boolean blobExists(String blobName) {
@@ -105,7 +112,7 @@ public void writeBlobAtomic(String blobName, InputStream inputStream, long blobS
     @Override
     public DeleteResult delete() throws IOException {
         try {
-            return blobStore.deleteBlobDirectory(keyPath);
+            return blobStore.deleteBlobDirectory(keyPath, threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME));
         } catch (URISyntaxException | StorageException e) {
             throw new IOException(e);
         }
@@ -113,9 +120,33 @@ public DeleteResult delete() throws IOException {
 
     @Override
     public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) throws IOException {
+        final PlainActionFuture<Void> result = PlainActionFuture.newFuture();
+        if (blobNames.isEmpty()) {
+            result.onResponse(null);
+        } else {
+            final GroupedActionListener<Void> listener =
+                new GroupedActionListener<>(ActionListener.map(result, v -> null), blobNames.size());
+            final ExecutorService executor = threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME);
+            // Executing deletes in parallel since Azure SDK 8 is using blocking IO while Azure does not provide a bulk delete API endpoint
+            // TODO: Upgrade to newer non-blocking Azure SDK 11 and execute delete requests in parallel that way.
+            for (String blobName : blobNames) {
+                executor.execute(ActionRunnable.run(listener, () -> {
+                    logger.trace("deleteBlob({})", blobName);
+                    try {
+                        blobStore.deleteBlob(buildKey(blobName));
+                    } catch (StorageException e) {
+                        if (e.getHttpStatusCode() != HttpURLConnection.HTTP_NOT_FOUND) {
+                            throw new IOException(e);
+                        }
+                    } catch (URISyntaxException e) {
+                        throw new IOException(e);
+                    }
+                }));
+            }
+        }
         try {
-            blobStore.deleteBlobsIgnoringIfNotExists(blobNames.stream().map(this::buildKey).collect(Collectors.toList()));
-        } catch (URISyntaxException | StorageException e) {
+            result.actionGet();
+        } catch (Exception e) {
             throw new IOException("Exception during bulk delete", e);
         }
     }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
index 173f13d801ff3..714e29edea29d 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
@@ -28,13 +28,14 @@
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.DeleteResult;
 import org.elasticsearch.repositories.azure.AzureRepository.Repository;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URISyntaxException;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import java.util.concurrent.Executor;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -43,15 +44,17 @@
 public class AzureBlobStore implements BlobStore {
 
     private final AzureStorageService service;
+    private final ThreadPool threadPool;
 
     private final String clientName;
     private final String container;
     private final LocationMode locationMode;
 
-    public AzureBlobStore(RepositoryMetaData metadata, AzureStorageService service) {
+    public AzureBlobStore(RepositoryMetaData metadata, AzureStorageService service, ThreadPool threadPool) {
         this.container = Repository.CONTAINER_SETTING.get(metadata.settings());
         this.clientName = Repository.CLIENT_NAME.get(metadata.settings());
         this.service = service;
+        this.threadPool = threadPool;
         // locationMode is set per repository, not per client
         this.locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
         final Map<String, AzureStorageSettings> prevSettings = this.service.refreshAndClearCache(emptyMap());
@@ -77,7 +80,7 @@ public LocationMode getLocationMode() {
 
     @Override
     public BlobContainer blobContainer(BlobPath path) {
-        return new AzureBlobContainer(path, this);
+        return new AzureBlobContainer(path, this, threadPool);
     }
 
     @Override
@@ -88,12 +91,13 @@ public boolean blobExists(String blob) throws URISyntaxException, StorageExcepti
         return service.blobExists(clientName, container, blob);
     }
 
-    public void deleteBlobsIgnoringIfNotExists(Collection<String> blobs) throws URISyntaxException, StorageException {
-        service.deleteBlobsIgnoringIfNotExists(clientName, container, blobs);
+    public void deleteBlob(String blob) throws URISyntaxException, StorageException, IOException {
+        service.deleteBlob(clientName, container, blob);
     }
 
-    public DeleteResult deleteBlobDirectory(String path) throws URISyntaxException, StorageException, IOException {
-        return service.deleteBlobDirectory(clientName, container, path);
+    public DeleteResult deleteBlobDirectory(String path, Executor executor)
+            throws URISyntaxException, StorageException, IOException {
+        return service.deleteBlobDirectory(clientName, container, path, executor);
     }
 
     public InputStream getInputStream(String blob) throws URISyntaxException, StorageException, IOException {
@@ -107,7 +111,7 @@ public Map<String, BlobMetaData> listBlobsByPrefix(String keyPath, String prefix
 
     public Map<String, BlobContainer> children(BlobPath path) throws URISyntaxException, StorageException, IOException {
         return Collections.unmodifiableMap(service.children(clientName, container, path).stream().collect(
-            Collectors.toMap(Function.identity(), name -> new AzureBlobContainer(path.add(name), this))));
+            Collectors.toMap(Function.identity(), name -> new AzureBlobContainer(path.add(name), this, threadPool))));
     }
 
     public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists)
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index a6a393bbb47c6..7b7a9108c1ef5 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -115,7 +115,7 @@ protected BlobStore getBlobStore() {
 
     @Override
     protected AzureBlobStore createBlobStore() {
-        final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService);
+        final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService, threadPool);
 
         logger.debug(() -> new ParameterizedMessage(
             "using container [{}], chunk_size [{}], compress [{}], base_path [{}]",
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
index ae1258a73b4f5..d98a7c3cbd717 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
@@ -23,12 +23,16 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.Repository;
+import org.elasticsearch.threadpool.ExecutorBuilder;
+import org.elasticsearch.threadpool.ScalingExecutorBuilder;
+
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -39,6 +43,8 @@
  */
 public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin, ReloadablePlugin {
 
+    public static final String REPOSITORY_THREAD_POOL_NAME = "repository_azure";
+
     // protected for testing
     final AzureStorageService azureStoreService;
 
@@ -74,6 +80,15 @@ public List<Setting<?>> getSettings() {
         );
     }
 
+    @Override
+    public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
+        return Collections.singletonList(executorBuilder());
+    }
+
+    public static ExecutorBuilder<?> executorBuilder() {
+        return new ScalingExecutorBuilder(REPOSITORY_THREAD_POOL_NAME, 0, 32, TimeValue.timeValueSeconds(30L));
+    }
+
     @Override
     public void reload(Settings settings) {
         // secure settings should be readable
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
index c15af49578386..26ade5bdec624 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
@@ -20,16 +20,13 @@
 package org.elasticsearch.repositories.azure;
 
 import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.BatchException;
 import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.Constants;
 import com.microsoft.azure.storage.OperationContext;
 import com.microsoft.azure.storage.RetryExponentialRetry;
 import com.microsoft.azure.storage.RetryPolicy;
 import com.microsoft.azure.storage.RetryPolicyFactory;
 import com.microsoft.azure.storage.StorageErrorCodeStrings;
 import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobDeleteBatchOperation;
 import com.microsoft.azure.storage.blob.BlobInputStream;
 import com.microsoft.azure.storage.blob.BlobListingDetails;
 import com.microsoft.azure.storage.blob.BlobProperties;
@@ -44,6 +41,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.DeleteResult;
@@ -53,6 +51,7 @@
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -63,13 +62,13 @@
 import java.security.InvalidKeyException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 
@@ -188,61 +187,72 @@ public boolean blobExists(String account, String container, String blob) throws
         });
     }
 
-    public void deleteBlobsIgnoringIfNotExists(String account, String container, Collection<String> blobs)
-            throws URISyntaxException, StorageException {
-        logger.trace(() -> new ParameterizedMessage("delete blobs for container [{}], blob [{}]", container, blobs));
+    public void deleteBlob(String account, String container, String blob) throws URISyntaxException, StorageException {
         final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client(account);
         // Container name must be lower case.
         final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
-        final Iterator<String> blobIterator = blobs.iterator();
-        int currentBatchSize = 0;
-        while (blobIterator.hasNext()) {
-            final BlobDeleteBatchOperation batchDeleteOp = new BlobDeleteBatchOperation();
-            do {
-                batchDeleteOp.addSubOperation(blobContainer.getBlockBlobReference(blobIterator.next()),
-                    DeleteSnapshotsOption.NONE, null, null);
-                ++currentBatchSize;
-            } while (blobIterator.hasNext() && currentBatchSize < Constants.BATCH_MAX_REQUESTS);
-            currentBatchSize = 0;
-            try {
-                SocketAccess.doPrivilegedVoidException(() -> blobContainer.getServiceClient().executeBatch(batchDeleteOp));
-            } catch (BatchException e) {
-                for (StorageException ex : e.getExceptions().values()) {
-                    if (ex.getHttpStatusCode() != HttpURLConnection.HTTP_NOT_FOUND) {
-                        logger.error("Batch exceptions [{}]", e.getExceptions());
-                        throw e;
-                    }
-                }
-            }
-        }
+        logger.trace(() -> new ParameterizedMessage("delete blob for container [{}], blob [{}]", container, blob));
+        SocketAccess.doPrivilegedVoidException(() -> {
+            final CloudBlockBlob azureBlob = blobContainer.getBlockBlobReference(blob);
+            logger.trace(() -> new ParameterizedMessage("container [{}]: blob [{}] found. removing.", container, blob));
+            azureBlob.delete(DeleteSnapshotsOption.NONE, null, null, client.v2().get());
+        });
     }
 
-    DeleteResult deleteBlobDirectory(String account, String container, String path)
+    DeleteResult deleteBlobDirectory(String account, String container, String path, Executor executor)
             throws URISyntaxException, StorageException, IOException {
         final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client(account);
         final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
+        final Collection<Exception> exceptions = Collections.synchronizedList(new ArrayList<>());
+        final AtomicLong outstanding = new AtomicLong(1L);
+        final PlainActionFuture<Void> result = PlainActionFuture.newFuture();
         final AtomicLong blobsDeleted = new AtomicLong();
         final AtomicLong bytesDeleted = new AtomicLong();
-        final List<String> blobsToDelete = new ArrayList<>();
         SocketAccess.doPrivilegedVoidException(() -> {
-            for (ListBlobItem blobItem : blobContainer.listBlobs(path, true)) {
+            for (final ListBlobItem blobItem : blobContainer.listBlobs(path, true)) {
                 // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
                 // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /
                 final String blobPath = blobItem.getUri().getPath().substring(1 + container.length() + 1);
-                final long len;
-                if (blobItem instanceof CloudBlob) {
-                    len = ((CloudBlob) blobItem).getProperties().getLength();
-                } else {
-                    len = -1L;
-                }
-                blobsToDelete.add(blobPath);
-                blobsDeleted.incrementAndGet();
-                if (len >= 0) {
-                    bytesDeleted.addAndGet(len);
-                }
+                outstanding.incrementAndGet();
+                executor.execute(new AbstractRunnable() {
+                    @Override
+                    protected void doRun() throws Exception {
+                        final long len;
+                        if (blobItem instanceof CloudBlob) {
+                            len = ((CloudBlob) blobItem).getProperties().getLength();
+                        } else {
+                            len = -1L;
+                        }
+                        deleteBlob(account, container, blobPath);
+                        blobsDeleted.incrementAndGet();
+                        if (len >= 0) {
+                            bytesDeleted.addAndGet(len);
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        exceptions.add(e);
+                    }
+
+                    @Override
+                    public void onAfter() {
+                        if (outstanding.decrementAndGet() == 0) {
+                            result.onResponse(null);
+                        }
+                    }
+                });
             }
         });
-        deleteBlobsIgnoringIfNotExists(account, container, blobsToDelete);
+        if (outstanding.decrementAndGet() == 0) {
+            result.onResponse(null);
+        }
+        result.actionGet();
+        if (exceptions.isEmpty() == false) {
+            final IOException ex = new IOException("Deleting directory [" + path + "] failed");
+            exceptions.forEach(ex::addSuppressed);
+            throw ex;
+        }
         return new DeleteResult(blobsDeleted.get(), bytesDeleted.get());
     }
 
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java
index b8ec6115b4d17..ce3cba065c35b 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java
@@ -44,6 +44,8 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
 
@@ -62,6 +64,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Matcher;
@@ -88,9 +91,11 @@
 public class AzureBlobContainerRetriesTests extends ESTestCase {
 
     private HttpServer httpServer;
+    private ThreadPool threadPool;
 
     @Before
     public void setUp() throws Exception {
+        threadPool = new TestThreadPool(getTestClass().getName(), AzureRepositoryPlugin.executorBuilder());
         httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
         httpServer.start();
         super.setUp();
@@ -100,6 +105,7 @@ public void setUp() throws Exception {
     public void tearDown() throws Exception {
         httpServer.stop(0);
         super.tearDown();
+        ThreadPool.terminate(threadPool, 10L, TimeUnit.SECONDS);
     }
 
     private BlobContainer createBlobContainer(final int maxRetries) {
@@ -139,7 +145,7 @@ BlobRequestOptions getBlobRequestOptionsForWriteBlob() {
                 .put(ACCOUNT_SETTING.getKey(), clientName)
                 .build());
 
-        return new AzureBlobContainer(BlobPath.cleanPath(), new AzureBlobStore(repositoryMetaData, service));
+        return new AzureBlobContainer(BlobPath.cleanPath(), new AzureBlobStore(repositoryMetaData, service, threadPool), threadPool);
     }
 
     public void testReadNonexistentBlobThrowsNoSuchFileException() {
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
index 47703b90e43cc..b23693fd268d4 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
@@ -64,7 +64,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
     @Override
     protected Map<String, HttpHandler> createHttpHandlers() {
-        return Collections.singletonMap("/", new AzureBlobStoreHttpHandler("container"));
+        return Collections.singletonMap("/container", new AzureBlobStoreHttpHandler("container"));
     }
 
     @Override
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
index 9af1ae73327ef..43bfc08d96df3 100644
--- a/plugins/repository-gcs/build.gradle
+++ b/plugins/repository-gcs/build.gradle
@@ -23,8 +23,8 @@ esplugin {
 }
 
 dependencies {
-  compile 'com.google.cloud:google-cloud-storage:1.104.0'
-  compile 'com.google.cloud:google-cloud-core:1.92.5'
+  compile 'com.google.cloud:google-cloud-storage:1.106.0'
+  compile 'com.google.cloud:google-cloud-core:1.93.3'
   compile 'com.google.guava:guava:26.0-jre'
   compile 'com.google.http-client:google-http-client:1.34.2'
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
@@ -33,25 +33,25 @@ dependencies {
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile 'com.google.api:api-common:1.8.1'
-  compile 'com.google.api:gax:1.45.0'
-  compile 'org.threeten:threetenbp:1.3.3'
+  compile 'com.google.api:gax:1.54.0'
+  compile 'org.threeten:threetenbp:1.4.1'
   compile 'com.google.protobuf:protobuf-java-util:3.11.3'
   compile 'com.google.protobuf:protobuf-java:3.11.3'
   compile 'com.google.code.gson:gson:2.7'
   compile 'com.google.api.grpc:proto-google-common-protos:1.16.0'
   compile 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
-  compile 'com.google.cloud:google-cloud-core-http:1.77.0'
+  compile 'com.google.cloud:google-cloud-core-http:1.93.3'
   compile 'com.google.auth:google-auth-library-credentials:0.20.0'
   compile 'com.google.auth:google-auth-library-oauth2-http:0.20.0'
   compile 'com.google.oauth-client:google-oauth-client:1.28.0'
-  compile 'com.google.api-client:google-api-client:1.30.8'
-  compile 'com.google.http-client:google-http-client-appengine:1.29.2'
-  compile 'com.google.http-client:google-http-client-jackson2:1.29.2'
+  compile 'com.google.api-client:google-api-client:1.30.9'
+  compile 'com.google.http-client:google-http-client-appengine:1.34.2'
+  compile 'com.google.http-client:google-http-client-jackson2:1.34.2'
   compile 'com.google.api:gax-httpjson:0.62.0'
   compile 'io.grpc:grpc-context:1.12.0'
   compile 'io.opencensus:opencensus-api:0.18.0'
   compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
-  compile 'com.google.apis:google-api-services-storage:v1-rev20191011-1.30.3'
+  compile 'com.google.apis:google-api-services-storage:v1-rev20200226-1.30.9'
 
   testCompile project(':test:fixtures:gcs-fixture')
 }
diff --git a/plugins/repository-gcs/licenses/gax-1.45.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-1.45.0.jar.sha1
deleted file mode 100644
index 8c0cbc659aa60..0000000000000
--- a/plugins/repository-gcs/licenses/gax-1.45.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2ade3e3502f9d14e3731347a82ea02372094211f
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/gax-1.54.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-1.54.0.jar.sha1
new file mode 100644
index 0000000000000..ed63c084f4edc
--- /dev/null
+++ b/plugins/repository-gcs/licenses/gax-1.54.0.jar.sha1
@@ -0,0 +1 @@
+1f1668868b8b3fd5fc248d80c16dd9f09afc9180
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-api-client-1.30.8.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-1.30.8.jar.sha1
deleted file mode 100644
index 9c9e79f4a4744..0000000000000
--- a/plugins/repository-gcs/licenses/google-api-client-1.30.8.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b8664bfb9287f269228a4bbc36988174d4f311eb
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-api-client-1.30.9.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-1.30.9.jar.sha1
new file mode 100644
index 0000000000000..cda9d1bd0ec87
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-api-client-1.30.9.jar.sha1
@@ -0,0 +1 @@
+8fc8641740ee0c26715738fb4d902779b96217d6
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20191011-1.30.3.jar.sha1 b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20191011-1.30.3.jar.sha1
deleted file mode 100644
index 36d83bd3ca4d1..0000000000000
--- a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20191011-1.30.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-53fc173da380f9b65caad69912223db128dbf7ac
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20200226-1.30.9.jar.sha1 b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20200226-1.30.9.jar.sha1
new file mode 100644
index 0000000000000..28b49b7f52e0b
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20200226-1.30.9.jar.sha1
@@ -0,0 +1 @@
+053c15bb46f6f3f623e94c587a67d975ab140bc8
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-core-1.92.5.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-1.92.5.jar.sha1
deleted file mode 100644
index f00050ffc77eb..0000000000000
--- a/plugins/repository-gcs/licenses/google-cloud-core-1.92.5.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-97ca5c35f34ee67dde8f2d4e9b940f0a96393c66
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-core-1.93.3.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-1.93.3.jar.sha1
new file mode 100644
index 0000000000000..fbbcf804bc541
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-cloud-core-1.93.3.jar.sha1
@@ -0,0 +1 @@
+6fc017c0908456c0867d21c85d1b906662d4b5f1
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-1.77.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-1.77.0.jar.sha1
deleted file mode 100644
index 7efc3167589eb..0000000000000
--- a/plugins/repository-gcs/licenses/google-cloud-core-http-1.77.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e16acbc935a7762ba9b220860ae45c2c67d17d8c
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-1.93.3.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-1.93.3.jar.sha1
new file mode 100644
index 0000000000000..0518072447569
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-cloud-core-http-1.93.3.jar.sha1
@@ -0,0 +1 @@
+2589aa6a4b6c49811c08ec2803c8e9c79c410bc5
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-storage-1.104.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-storage-1.104.0.jar.sha1
deleted file mode 100644
index b729579599b60..0000000000000
--- a/plugins/repository-gcs/licenses/google-cloud-storage-1.104.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8a083661f3732b21c0addd4fe56fd3f05a9c5a84
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-storage-1.106.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-storage-1.106.0.jar.sha1
new file mode 100644
index 0000000000000..6b1d3969c81a7
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-cloud-storage-1.106.0.jar.sha1
@@ -0,0 +1 @@
+08b9c0a551b02028fe6c4c666236b21796099238
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-appengine-1.29.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-appengine-1.29.2.jar.sha1
deleted file mode 100644
index 6973b62e9281c..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-appengine-1.29.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d93f4d1d8c2496d75221e53173e4c503b7096a4d
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-appengine-1.34.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-appengine-1.34.2.jar.sha1
new file mode 100644
index 0000000000000..884514d21e510
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-appengine-1.34.2.jar.sha1
@@ -0,0 +1 @@
+dbcfa59e3de5f29ed5ab36dfc9ec8f9dbf279ef9
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.29.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.29.2.jar.sha1
deleted file mode 100644
index 5a44d18d2aa1b..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.29.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d67891f5a438e1f339387a09628e0ab0af8b612a
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.34.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.34.2.jar.sha1
new file mode 100644
index 0000000000000..6c680aef39894
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.34.2.jar.sha1
@@ -0,0 +1 @@
+fc5cd5e7b6252bae6a65ef2a1091476afb25b7c7
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/threetenbp-1.3.3.jar.sha1 b/plugins/repository-gcs/licenses/threetenbp-1.3.3.jar.sha1
deleted file mode 100644
index 9273043e14520..0000000000000
--- a/plugins/repository-gcs/licenses/threetenbp-1.3.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3ea31c96676ff12ab56be0b1af6fff61d1a4f1f2
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/threetenbp-1.4.1.jar.sha1 b/plugins/repository-gcs/licenses/threetenbp-1.4.1.jar.sha1
new file mode 100644
index 0000000000000..682387ad67c56
--- /dev/null
+++ b/plugins/repository-gcs/licenses/threetenbp-1.4.1.jar.sha1
@@ -0,0 +1 @@
+8e0de17c5077523503812837fa069459103fb714
\ No newline at end of file
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
index 24ed35e550491..ac8e9ede430e0 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
@@ -28,7 +28,6 @@
 import fixture.gcs.GoogleCloudStorageHttpHandler;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
@@ -66,7 +65,6 @@
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository.CLIENT_NAME;
 
 @SuppressForbidden(reason = "this test uses a HttpServer to emulate a Google Cloud Storage endpoint")
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/52906")
 public class GoogleCloudStorageBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryIntegTestCase {
 
     @Override
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index bf570b8e0abb6..02af59e418531 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -35,7 +35,7 @@ esplugin {
 }
 
 versions << [
-  'hadoop2': '2.8.1'
+  'hadoop2': '2.8.5'
 ]
 
 testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs"
diff --git a/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1
deleted file mode 100644
index bb000d4ddb025..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-335a867cf42bf789919bfc3229ff26747124e8f1
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.5.jar.sha1
new file mode 100644
index 0000000000000..a808179106498
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.5.jar.sha1
@@ -0,0 +1 @@
+3a301159daf9368b05364577c985498857b5c48f
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1
deleted file mode 100644
index 67a1c4f3fa546..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-688ccccc0e0739d8737a93b0039a4a661e52084b
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-auth-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-auth-2.8.5.jar.sha1
new file mode 100644
index 0000000000000..2d9064f4965eb
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-auth-2.8.5.jar.sha1
@@ -0,0 +1 @@
+63fe1f9d9ef6bdf2cb52dfeb28ed8faf78e4b85c
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1
deleted file mode 100644
index e7be8d468ddfc..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4812f251f8100fd4722c3cec5d7353f71f69cda9
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-client-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-2.8.5.jar.sha1
new file mode 100644
index 0000000000000..83d232c5a295d
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-client-2.8.5.jar.sha1
@@ -0,0 +1 @@
+45e11f6004581e53959bc3d38c4d01dbeb5f4b22
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1
deleted file mode 100644
index 6e6c625b7d6c2..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a4df18b79e4d0349ce4b58a52d314e7ae1d6be99
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-common-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-common-2.8.5.jar.sha1
new file mode 100644
index 0000000000000..26b2752840515
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-common-2.8.5.jar.sha1
@@ -0,0 +1 @@
+376e807b4f9df627b03aeaf85999cd1ffe496f05
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1
deleted file mode 100644
index e419a3124f66b..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a378f4bc8e6cd779d779c9f512e0e31edd771633
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.5.jar.sha1
new file mode 100644
index 0000000000000..241a3ea92ac2b
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.5.jar.sha1
@@ -0,0 +1 @@
+36914392fd3e77d46e54c3bb092dfc32d3f4a32b
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1
deleted file mode 100644
index 20c7963cd3d1b..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6b0100e4f58ecf7ce75817fce1ffdfbec947337a
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.5.jar.sha1
new file mode 100644
index 0000000000000..2a05eca7320a8
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.5.jar.sha1
@@ -0,0 +1 @@
+9d72fa62b01c32f1c0587d53c5005fc49f2bd11c
\ No newline at end of file
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/ByteBufUtils.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/ByteBufUtils.java
index 0408290b039d9..6e041b354ec75 100644
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/ByteBufUtils.java
+++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/ByteBufUtils.java
@@ -23,15 +23,11 @@
 import io.netty.buffer.Unpooled;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
-import org.elasticsearch.common.bytes.AbstractBytesReference;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
 
-import java.io.EOFException;
 import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -45,266 +41,34 @@ static ByteBuf toByteBuf(final BytesReference reference) {
         if (reference.length() == 0) {
             return Unpooled.EMPTY_BUFFER;
         }
-        if (reference instanceof ByteBufBytesReference) {
-            return ((ByteBufBytesReference) reference).toByteBuf();
-        } else {
-            final BytesRefIterator iterator = reference.iterator();
-            // usually we have one, two, or three components from the header, the message, and a buffer
-            final List<ByteBuf> buffers = new ArrayList<>(3);
-            try {
-                BytesRef slice;
-                while ((slice = iterator.next()) != null) {
-                    buffers.add(Unpooled.wrappedBuffer(slice.bytes, slice.offset, slice.length));
-                }
+        final BytesRefIterator iterator = reference.iterator();
+        // usually we have one, two, or three components from the header, the message, and a buffer
+        final List<ByteBuf> buffers = new ArrayList<>(3);
+        try {
+            BytesRef slice;
+            while ((slice = iterator.next()) != null) {
+                buffers.add(Unpooled.wrappedBuffer(slice.bytes, slice.offset, slice.length));
+            }
 
-                if (buffers.size() == 1) {
-                    return buffers.get(0);
-                } else {
-                    CompositeByteBuf composite = Unpooled.compositeBuffer(buffers.size());
-                    composite.addComponents(true, buffers);
-                    return composite;
-                }
-            } catch (IOException ex) {
-                throw new AssertionError("no IO happens here", ex);
+            if (buffers.size() == 1) {
+                return buffers.get(0);
+            } else {
+                CompositeByteBuf composite = Unpooled.compositeBuffer(buffers.size());
+                composite.addComponents(true, buffers);
+                return composite;
             }
+        } catch (IOException ex) {
+            throw new AssertionError("no IO happens here", ex);
         }
     }
 
     static BytesReference toBytesReference(final ByteBuf buffer) {
         final int readableBytes = buffer.readableBytes();
-        return readableBytes == 0 ? BytesArray.EMPTY : new ByteBufBytesReference(buffer, readableBytes);
-    }
-
-    private static class ByteBufBytesReference extends AbstractBytesReference {
-
-        private final ByteBuf buffer;
-        private final int length;
-        private final int offset;
-
-        ByteBufBytesReference(ByteBuf buffer, int length) {
-            this.buffer = buffer;
-            this.length = length;
-            this.offset = buffer.readerIndex();
-            assert length <= buffer.readableBytes() : "length[" + length +"] > " + buffer.readableBytes();
-        }
-
-        @Override
-        public byte get(int index) {
-            return buffer.getByte(offset + index);
-        }
-
-        @Override
-        public int getInt(int index) {
-            return buffer.getInt(offset + index);
-        }
-
-        @Override
-        public int indexOf(byte marker, int from) {
-            final int start = offset + from;
-            return buffer.forEachByte(start, length - start, value -> value != marker);
-        }
-
-        @Override
-        public int length() {
-            return length;
-        }
-
-        @Override
-        public BytesReference slice(int from, int length) {
-            return new ByteBufBytesReference(buffer.slice(offset + from, length), length);
-        }
-
-        @Override
-        public StreamInput streamInput() {
-            return new ByteBufStreamInput(buffer.duplicate(), length);
-        }
-
-        @Override
-        public void writeTo(OutputStream os) throws IOException {
-            buffer.getBytes(offset, os, length);
-        }
-
-        ByteBuf toByteBuf() {
-            return buffer.duplicate();
-        }
-
-        @Override
-        public String utf8ToString() {
-            return buffer.toString(offset, length, StandardCharsets.UTF_8);
-        }
-
-        @Override
-        public BytesRef toBytesRef() {
-            if (buffer.hasArray()) {
-                return new BytesRef(buffer.array(), buffer.arrayOffset() + offset, length);
-            }
-            final byte[] copy = new byte[length];
-            buffer.getBytes(offset, copy);
-            return new BytesRef(copy);
-        }
-
-        @Override
-        public long ramBytesUsed() {
-            return buffer.capacity();
-        }
-
-    }
-
-    private static class ByteBufStreamInput extends StreamInput {
-
-        private final ByteBuf buffer;
-        private final int endIndex;
-
-        ByteBufStreamInput(ByteBuf buffer, int length) {
-            if (length > buffer.readableBytes()) {
-                throw new IndexOutOfBoundsException();
-            }
-            this.buffer = buffer;
-            int startIndex = buffer.readerIndex();
-            endIndex = startIndex + length;
-            buffer.markReaderIndex();
-        }
-
-        @Override
-        public BytesReference readBytesReference(int length) throws IOException {
-            // NOTE: It is unsafe to share a reference of the internal structure, so we
-            // use the default implementation which will copy the bytes. It is unsafe because
-            // a netty ByteBuf might be pooled which requires a manual release to prevent
-            // memory leaks.
-            return super.readBytesReference(length);
-        }
-
-        @Override
-        public BytesRef readBytesRef(int length) throws IOException {
-            // NOTE: It is unsafe to share a reference of the internal structure, so we
-            // use the default implementation which will copy the bytes. It is unsafe because
-            // a netty ByteBuf might be pooled which requires a manual release to prevent
-            // memory leaks.
-            return super.readBytesRef(length);
-        }
-
-        @Override
-        public int available() throws IOException {
-            return endIndex - buffer.readerIndex();
-        }
-
-        @Override
-        protected void ensureCanReadBytes(int length) throws EOFException {
-            int bytesAvailable = endIndex - buffer.readerIndex();
-            if (bytesAvailable < length) {
-                throw new EOFException("tried to read: " + length + " bytes but only " + bytesAvailable + " remaining");
-            }
-        }
-
-        @Override
-        public void mark(int readlimit) {
-            buffer.markReaderIndex();
-        }
-
-        @Override
-        public boolean markSupported() {
-            return true;
-        }
-
-        @Override
-        public int read() throws IOException {
-            if (available() == 0) {
-                return -1;
-            }
-            return buffer.readByte() & 0xff;
-        }
-
-        @Override
-        public int read(byte[] b, int off, int len) throws IOException {
-            if (len == 0) {
-                return 0;
-            }
-            int available = available();
-            if (available == 0) {
-                return -1;
-            }
-
-            len = Math.min(available, len);
-            buffer.readBytes(b, off, len);
-            return len;
-        }
-
-        @Override
-        public short readShort() throws IOException {
-            try {
-                return buffer.readShort();
-            } catch (IndexOutOfBoundsException ex) {
-                EOFException eofException = new EOFException();
-                eofException.initCause(ex);
-                throw eofException;
-            }
-        }
-
-        @Override
-        public int readInt() throws IOException {
-            try {
-                return buffer.readInt();
-            } catch (IndexOutOfBoundsException ex) {
-                EOFException eofException = new EOFException();
-                eofException.initCause(ex);
-                throw eofException;
-            }
-        }
-
-        @Override
-        public long readLong() throws IOException {
-            try {
-                return buffer.readLong();
-            } catch (IndexOutOfBoundsException ex) {
-                EOFException eofException = new EOFException();
-                eofException.initCause(ex);
-                throw eofException;
-            }
-        }
-
-        @Override
-        public void reset() throws IOException {
-            buffer.resetReaderIndex();
-        }
-
-        @Override
-        public long skip(long n) throws IOException {
-            if (n > Integer.MAX_VALUE) {
-                return skipBytes(Integer.MAX_VALUE);
-            } else {
-                return skipBytes((int) n);
-            }
-        }
-
-        public int skipBytes(int n) throws IOException {
-            int nBytes = Math.min(available(), n);
-            buffer.skipBytes(nBytes);
-            return nBytes;
-        }
-
-
-        @Override
-        public byte readByte() throws IOException {
-            try {
-                return buffer.readByte();
-            } catch (IndexOutOfBoundsException ex) {
-                EOFException eofException = new EOFException();
-                eofException.initCause(ex);
-                throw eofException;
-            }
-        }
-
-        @Override
-        public void readBytes(byte[] b, int offset, int len) throws IOException {
-            int read = read(b, offset, len);
-            if (read < len) {
-                throw new IndexOutOfBoundsException();
-            }
-        }
-
-        @Override
-        public void close() throws IOException {
-            // nothing to do here
+        if (readableBytes == 0) {
+            return BytesArray.EMPTY;
+        } else {
+            final ByteBuffer[] byteBuffers = buffer.nioBuffers();
+            return BytesReference.fromByteBuffers(byteBuffers);
         }
     }
 }
diff --git a/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java
index d19cf3ebd974b..7feec6868e7b7 100644
--- a/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java
+++ b/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java
@@ -117,7 +117,7 @@ private static MockTransportService startTransport(
                     InternalSearchResponse response = new InternalSearchResponse(new SearchHits(new SearchHit[0],
                         new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), InternalAggregations.EMPTY, null, null, false, null, 1);
                     SearchResponse searchResponse = new SearchResponse(response, null, 1, 1, 0, 100, ShardSearchFailure.EMPTY_ARRAY,
-                        SearchResponse.Clusters.EMPTY);
+                        SearchResponse.Clusters.EMPTY, null);
                     channel.sendResponse(searchResponse);
                 });
             newService.registerRequestHandler(ClusterStateAction.NAME, ThreadPool.Names.SAME, ClusterStateRequest::new,
diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle
index a0908e24b41fc..125a341de6c11 100644
--- a/qa/multi-cluster-search/build.gradle
+++ b/qa/multi-cluster-search/build.gradle
@@ -35,7 +35,7 @@ task 'remote-cluster'(type: RestIntegTestTask) {
 
 testClusters.'remote-cluster' {
   numberOfNodes = 2
-  setting 'cluster.remote.connect', 'false'
+  setting 'node.remote_cluster_client', 'false'
 }
 
 task mixedClusterTest(type: RestIntegTestTask) {
@@ -50,7 +50,7 @@ testClusters.mixedClusterTest {
   setting 'cluster.remote.my_remote_cluster.seeds',
     { "\"${testClusters.'remote-cluster'.getAllTransportPortURI().get(0)}\"" }
   setting 'cluster.remote.connections_per_cluster', '1'
-  setting 'cluster.remote.connect', 'true'
+  setting 'node.remote_cluster_client', 'true'
 }
 
 
diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
index a4e76fc7ec436..d2f876cd17848 100644
--- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
+++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
@@ -124,7 +124,7 @@ public class CCSDuelIT extends ESRestTestCase {
 
     private static final String INDEX_NAME = "ccs_duel_index";
     private static final String REMOTE_INDEX_NAME = "my_remote_cluster:" + INDEX_NAME;
-    private static final String[] TAGS = new String[]{"java", "xml", "sql", "html", "php", "ruby", "python", "perl"};
+    private static final String[] TAGS = new String[] {"java", "xml", "sql", "html", "php", "ruby", "python", "perl"};
 
     private static RestHighLevelClient restHighLevelClient;
 
@@ -435,6 +435,8 @@ public void testSortByField() throws Exception  {
     public void testSortByFieldOneClusterHasNoResults() throws Exception {
         assumeMultiClusterSetup();
         SearchRequest searchRequest = initSearchRequest();
+        // set to a value greater than the number of shards to avoid differences due to the skipping of shards
+        searchRequest.setPreFilterShardSize(128);
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
         boolean onlyRemote = randomBoolean();
         sourceBuilder.query(new TermQueryBuilder("_index", onlyRemote ? REMOTE_INDEX_NAME : INDEX_NAME));
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml
index b4dd748f8286e..77dafaf76ca91 100644
--- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml
@@ -15,15 +15,15 @@
         refresh: true
         body:
             - '{"index": {"_index": "test_index"}}'
-            - '{"f1": "local_cluster", "filter_field": 0}'
+            - '{"f1": "local_cluster", "animal": "dog", "filter_field": 0}'
             - '{"index": {"_index": "test_index"}}'
-            - '{"f1": "local_cluster", "filter_field": 1}'
+            - '{"f1": "local_cluster", "animal": "dog", "filter_field": 1}'
             - '{"index": {"_index": "test_index"}}'
-            - '{"f1": "local_cluster", "filter_field": 0}'
+            - '{"f1": "local_cluster", "animal": "dog", "filter_field": 0}'
             - '{"index": {"_index": "test_index"}}'
-            - '{"f1": "local_cluster", "filter_field": 1}'
+            - '{"f1": "local_cluster", "animal": "dog", "filter_field": 1}'
             - '{"index": {"_index": "test_index"}}'
-            - '{"f1": "local_cluster", "filter_field": 0}'
+            - '{"f1": "local_cluster", "animal": "pig", "filter_field": 0}'
 
   - do:
       search:
@@ -115,6 +115,87 @@
   - match: { aggregations.cluster.buckets.0.key: "local_cluster" }
   - match: { aggregations.cluster.buckets.0.doc_count: 5 }
 
+  # once more, this time with a top level pipeline agg
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test_index,my_remote_cluster:test_index
+        body:
+          seq_no_primary_term: true
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
+              aggs:
+                s:
+                  sum:
+                    field: filter_field
+            average_sum:
+              avg_bucket:
+                buckets_path: cluster.s
+
+  - match: { num_reduce_phases: 3 }
+  - match: {_clusters.total: 2}
+  - match: {_clusters.successful: 2}
+  - match: {_clusters.skipped: 0}
+  - match: { _shards.total: 5 }
+  - match: { hits.total: 11 }
+  - length: { aggregations.cluster.buckets: 2 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+  - match: { aggregations.cluster.buckets.0.s.value: 2 }
+  - match: { aggregations.cluster.buckets.1.key: "local_cluster" }
+  - match: { aggregations.cluster.buckets.1.s.value: 2 }
+  - match: { aggregations.average_sum.value: 2 }
+
+  # and now a non-top-level pipeline agg!
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test_index,my_remote_cluster:test_index
+        body:
+          seq_no_primary_term: true
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
+              aggs:
+                animal:
+                  terms:
+                    field: animal.keyword
+                  aggs:
+                    s:
+                      sum:
+                        field: filter_field
+                average_sum:
+                  avg_bucket:
+                    buckets_path: animal.s
+
+  - match: { num_reduce_phases: 3 }
+  - match: {_clusters.total: 2}
+  - match: {_clusters.successful: 2}
+  - match: {_clusters.skipped: 0}
+  - match: { _shards.total: 5 }
+  - match: { hits.total: 11 }
+  - length: { aggregations.cluster.buckets: 2 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+  - match: { aggregations.cluster.buckets.0.animal.buckets.0.key: "chicken" }
+  - match: { aggregations.cluster.buckets.0.animal.buckets.0.doc_count: 4 }
+  - match: { aggregations.cluster.buckets.0.animal.buckets.0.s.value: 1 }
+  - match: { aggregations.cluster.buckets.0.animal.buckets.1.key: "pig" }
+  - match: { aggregations.cluster.buckets.0.animal.buckets.1.doc_count: 2 }
+  - match: { aggregations.cluster.buckets.0.animal.buckets.1.s.value: 1 }
+  - match: { aggregations.cluster.buckets.0.average_sum.value: 1 }
+  - match: { aggregations.cluster.buckets.1.key: "local_cluster" }
+  - match: { aggregations.cluster.buckets.1.animal.buckets.0.key: "dog" }
+  - match: { aggregations.cluster.buckets.1.animal.buckets.0.doc_count: 4 }
+  - match: { aggregations.cluster.buckets.1.animal.buckets.0.s.value: 2 }
+  - match: { aggregations.cluster.buckets.1.animal.buckets.1.key: "pig" }
+  - match: { aggregations.cluster.buckets.1.animal.buckets.1.doc_count: 1 }
+  - match: { aggregations.cluster.buckets.1.animal.buckets.1.s.value: 0 }
+  - match: { aggregations.cluster.buckets.1.average_sum.value: 1 }
+
 ---
 "Add transient remote cluster based on the preset cluster":
   - do:
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml
index 19b3771acf877..95109a0a59370 100644
--- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml
@@ -91,17 +91,17 @@
          refresh: true
          body:
            - '{"index": {"_index": "test_index"}}'
-           - '{"f1": "remote_cluster", "filter_field": 0}'
+           - '{"f1": "remote_cluster", "animal": "pig", "filter_field": 0}'
            - '{"index": {"_index": "test_index"}}'
-           - '{"f1": "remote_cluster", "filter_field": 1}'
+           - '{"f1": "remote_cluster", "animal": "pig", "filter_field": 1}'
            - '{"index": {"_index": "test_index"}}'
-           - '{"f1": "remote_cluster", "filter_field": 0}'
+           - '{"f1": "remote_cluster", "animal": "chicken", "filter_field": 0}'
            - '{"index": {"_index": "test_index"}}'
-           - '{"f1": "remote_cluster", "filter_field": 1}'
+           - '{"f1": "remote_cluster", "animal": "chicken", "filter_field": 1}'
            - '{"index": {"_index": "test_index"}}'
-           - '{"f1": "remote_cluster", "filter_field": 0}'
+           - '{"f1": "remote_cluster", "animal": "chicken", "filter_field": 0}'
            - '{"index": {"_index": "test_index"}}'
-           - '{"f1": "remote_cluster", "filter_field": 0}'
+           - '{"f1": "remote_cluster", "animal": "chicken", "filter_field": 0}'
 
   - do:
       search:
diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
index 88be6d93b5d56..d45ae29e3d15e 100644
--- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
+++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.packaging.util.ServerUtils;
 import org.elasticsearch.packaging.util.Shell.Result;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
@@ -45,6 +46,7 @@
 import static org.elasticsearch.packaging.util.Docker.getImageLabels;
 import static org.elasticsearch.packaging.util.Docker.getJson;
 import static org.elasticsearch.packaging.util.Docker.mkDirWithPrivilegeEscalation;
+import static org.elasticsearch.packaging.util.Docker.removeContainer;
 import static org.elasticsearch.packaging.util.Docker.rmDirWithPrivilegeEscalation;
 import static org.elasticsearch.packaging.util.Docker.runContainer;
 import static org.elasticsearch.packaging.util.Docker.runContainerExpectingFailure;
@@ -77,6 +79,11 @@ public static void filterDistros() {
         assumeTrue("only Docker", distribution().isDocker());
     }
 
+    @AfterClass
+    public static void teardownSuite() {
+        removeContainer();
+    }
+
     @Before
     public void setupTest() throws IOException {
         installation = runContainer(distribution());
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json b/rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json
new file mode 100644
index 0000000000000..353d82a0ff9d5
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json
@@ -0,0 +1,23 @@
+{
+  "clear_reader":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-body.html#_clear_scroll_api",
+      "description":"Explicitly clears the point in time readers"
+    },
+    "stability":"beta",
+    "url":{
+      "paths":[
+        {
+          "path":"/_search/reader",
+          "methods":[
+            "DELETE"
+          ]
+        }
+      ]
+    },
+    "params":{},
+    "body":{
+      "description": "A point in time reader to clear"
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
index 8dd9da844520a..968ccfd8e718f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
@@ -50,8 +50,7 @@
       },
       "pre_filter_shard_size":{
         "type":"number",
-        "description":"A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.",
-        "default":128
+        "description":"A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint."
       },
       "max_concurrent_shard_requests":{
         "type":"number",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index 1e0c232efa055..ac321acf8907b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -219,8 +219,7 @@
       },
       "pre_filter_shard_size":{
         "type":"number",
-        "description":"A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.",
-        "default":128
+        "description":"A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint."
       },
       "rest_total_hits_as_int":{
         "type":"boolean",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
index 920fedd3e7414..1c9ce4e7640d8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
@@ -133,6 +133,109 @@ by default, thus the related skip sections can be removed from the tests.
 The `features` field can either be a string or an array of strings.
 The skip section requires to specify either a `version` or a `features` list.
 
+=== Available Features
+
+=== `xpack`
+Requires x-pack to be enabled on the `Elasticsearch` instance the rest test is running against
+
+=== `no_xpack`
+Requires the test to run against an oss distribution of `Elasticsearch`
+
+=== `catch_unauthorized`
+
+Runner supports `catch: unauthorized` on a `do` operator.
+
+=== `default_shards`
+
+This test can only run if the cluster is running with the distributions default number of shards.
+
+The Java test runner introduces randomness and sometimes overrides the default number of shards to `2`.
+If the default number of shards is changed, test marked with this feature should *not* run
+
+=== `headers`
+
+The runner is able to set per request headers on the `do` operation
+
+=== `node_selector`
+
+Indicates the runner can parse `node_selector` under the `do` operator and use its metadata to select the node to
+perform the `do` operation on.
+
+=== `stash_in_key`
+
+Allows you to use a stashed value in any key of an object during a `match` assertion
+
+....
+- set: {nodes.$master.http.publish_address: host}
+- match:
+    $body:
+      {
+        "nodes": {
+          $host: {
+            ... stuff in here ...
+          }
+        }
+     }
+....
+
+=== `stash_in_path`
+
+Allows a stashed value to be referenced in path lookups as a single token. E.g:
+
+....
+path.$stash.value
+....
+
+=== `embedded_stash_key`
+
+Allows a stashed key to appear anywhere in the path (note the placeholder needs to be within curly brackets too in this case):
+
+....
+field1.e${placeholder}ments.element1
+....
+
+=== `stash_path_replace`
+Used only in the doc snippet tests. Allow you to do ease replacements using a special `$_path` marker.
+
+....
+// TESTRESPONSEs/somevalue/$body.${_path}/ to mean "replace
+somevalue with whatever is the response in the same position."
+....
+
+=== `warnings`
+
+The runner can assert the warnings headers returned by Elasticsearch through the `warning:` assertations
+under `do:`  operations.
+
+=== `yaml`
+
+The runner is able to send and receive `application/yaml` and perform all assertions on the returned data.
+
+=== `contains`
+
+Asserts an array of object contains an object with a property set to a certain value. e.g:
+
+...
+contains:  { nodes.$master.plugins: { name: painless-whitelist } }
+...
+
+Asserts the plugins array contains an object with a `name` property with the value `painless-whitelist`
+
+=== `transform_and_set`
+
+Supports the `transform_and_set` operator as described in this document.
+
+=== `arbitrary_key`
+
+Allows you to stash an arbitrary key from a returned map e.g:
+
+....
+- set:
+    nodes._arbitrary_key_: node_id
+....
+
+This means: Stash any of the keys returned under `nodes` as `$node_id`
+
 Required operators:
 -------------------
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml
index dbe18fbd83829..4c28f0bec5dfb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml
@@ -7,7 +7,7 @@
   - match:
       $body: |
                /  #ip                          heap.percent        ram.percent     cpu         load_1m                load_5m                load_15m               node.role           master          name
-               ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
+               ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilmrt]{1,6}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
@@ -16,7 +16,7 @@
   - match:
       $body: |
                /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role      \s+  master   \s+   name  \n
-                  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
+                  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilmrt]{1,6}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
index de5e632975752..aa4abc7a11eae 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
@@ -1,8 +1,8 @@
 ---
 "Help":
   - skip:
-      version: " - 7.1.99"
-      reason:  external refresh stats were added in 7.2.0
+      version: " - 7.99.99"
+      reason:  shard path stats were added in 8.0.0
   - do:
       cat.shards:
         help: true
@@ -78,6 +78,8 @@
                     warmer.current                   .+   \n
                     warmer.total                     .+   \n
                     warmer.total_time                .+   \n
+                    path.data                        .+   \n
+                    path.state                       .+   \n
                $/
 ---
 "Test cat shards output":
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
index 035c20c1600bf..2861c443bfa8e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
@@ -1,26 +1,33 @@
 ---
-"Test stubs":
+"Create data stream":
   - skip:
       version: " - 7.6.99"
-      reason: only available in 7.7+
+      reason: available only in 7.7+
 
   - do:
       indices.create_data_stream:
-        name: data-stream2
+        name: simple-data-stream1
         body:
           timestamp_field: "@timestamp"
   - is_true: acknowledged
 
+  - do:
+      indices.create_data_stream:
+        name: simple-data-stream2
+        body:
+          timestamp_field: "@timestamp2"
+  - is_true: acknowledged
+
   - do:
       indices.get_data_streams: {}
-  - match: { 0.name: my_data_stream1 }
+  - match: { 0.name: simple-data-stream1 }
   - match: { 0.timestamp_field: '@timestamp' }
-  - match: { 0.indices: ['my_data_stream1-000000'] }
-  - match: { 1.name: my_data_stream2 }
-  - match: { 1.timestamp_field: '@timestamp' }
+  - match: { 0.indices: [] }
+  - match: { 1.name: simple-data-stream2 }
+  - match: { 1.timestamp_field: '@timestamp2' }
   - match: { 1.indices: [] }
 
   - do:
       indices.delete_data_stream:
-        name: data-stream2
+        name: simple-data-stream2
   - is_true: acknowledged
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml
index e3b4dcc46230b..ab117eebbf607 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml
@@ -34,7 +34,7 @@
             search.max_keep_alive: "1m"
 
   - do:
-      catch: /.*Keep alive for scroll.*is too large.*/
+      catch: /.*Keep alive for.*is too large.*/
       search:
         rest_total_hits_as_int: true
         index: test_scroll
@@ -61,7 +61,7 @@
   - length: {hits.hits:      1    }
 
   - do:
-      catch: /.*Keep alive for scroll.*is too large.*/
+      catch: /.*Keep alive for.*is too large.*/
       scroll:
         rest_total_hits_as_int: true
         scroll_id: $scroll_id
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml
new file mode 100644
index 0000000000000..a0834565df258
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml
@@ -0,0 +1,122 @@
+setup:
+  - do:
+      indices.create:
+          index:  test
+  - do:
+      index:
+          index:  test
+          id:     1
+          body:   { id: 1, foo: bar, age: 18 }
+
+  - do:
+      index:
+          index:  test
+          id:     42
+          body:   { id: 42, foo: bar, age: 18 }
+
+  - do:
+        index:
+            index:  test
+            id:     172
+            body:   { id: 172, foo: bar, age: 24 }
+
+  - do:
+      indices.refresh:
+        index: test
+
+---
+"search with reader context":
+  - skip:
+      version: " - 7.9.99"
+      reason: "reader context is introduced in 8.0"
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          size: 1
+          query:
+            match:
+              foo: bar
+          sort: [{ age: desc }, { id: desc }]
+          reader:
+            keep_alive: 1m
+
+  - match: {hits.total: 3 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0._index: test }
+  - match: {hits.hits.0._id: "172" }
+  - match: {hits.hits.0.sort: [24, 172] }
+  - set: {reader_id: reader_id}
+
+  - do:
+      index:
+        index:  test
+        id:     100
+        body:   { id: 100, foo: bar, age: 23 }
+  - do:
+      indices.refresh:
+        index: test
+
+  # search with the retaining readers
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          size: 1
+          query:
+            match:
+              foo: bar
+          sort: [{ age: desc }, { id: desc }]
+          search_after: [24, 172]
+          reader:
+            id: "$reader_id"
+            keep_alive: 1m
+
+  - match: {hits.total: 3 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0._index: test }
+  - match: {hits.hits.0._id: "42" }
+  - match: {hits.hits.0.sort: [18, 42] }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          size: 1
+          query:
+            match:
+              foo: bar
+          sort: [ { age: desc }, { id: desc } ]
+          search_after: [18, 42]
+          reader:
+            id: "$reader_id"
+            keep_alive: 1m
+
+  - match: {hits.total: 3}
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0._index: test }
+  - match: {hits.hits.0._id: "1" }
+  - match: {hits.hits.0.sort: [18, 1] }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          size: 1
+          query:
+            match:
+              foo: bar
+          sort: [{ age: desc }, { id: desc } ]
+          search_after: [18, 1]
+          reader:
+            id: "$reader_id"
+            keep_alive: 1m
+
+  - match: {hits.total: 3}
+  - length: {hits.hits: 0 }
+
+  - do:
+      clear_reader:
+        body:
+          id: "$reader_id"
diff --git a/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 242bbfc9bd604..0000000000000
--- a/server/licenses/lucene-analyzers-common-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0365c37a03123ee8e30f75e44a1cb7d5ddd2fc52
\ No newline at end of file
diff --git a/server/licenses/lucene-analyzers-common-8.5.0.jar.sha1 b/server/licenses/lucene-analyzers-common-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..398344daaa992
--- /dev/null
+++ b/server/licenses/lucene-analyzers-common-8.5.0.jar.sha1
@@ -0,0 +1 @@
+7156f2e545fd6e7faaee4781d15eb60cf5f07646
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 056fe4a00cbb5..0000000000000
--- a/server/licenses/lucene-backward-codecs-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d56b30f75b2df92da8c6c0965ce72e7abb86347b
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-8.5.0.jar.sha1 b/server/licenses/lucene-backward-codecs-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..1f926add1ffa2
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-8.5.0.jar.sha1
@@ -0,0 +1 @@
+5837d9ec231b998d9eb75a99f3bf1dc9748c8f46
\ No newline at end of file
diff --git a/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index fe30b9975cab4..0000000000000
--- a/server/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-39933692162e28c2719b60f499204b28236a2858
\ No newline at end of file
diff --git a/server/licenses/lucene-core-8.5.0.jar.sha1 b/server/licenses/lucene-core-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..588b450154d09
--- /dev/null
+++ b/server/licenses/lucene-core-8.5.0.jar.sha1
@@ -0,0 +1 @@
+3f9ea85fff4fc3f7c83869dddb9b0ef7818c0cae
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index c3ef488e826f8..0000000000000
--- a/server/licenses/lucene-grouping-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2e56cc12d2f77d82946299b66f3416f9e621b2f3
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-8.5.0.jar.sha1 b/server/licenses/lucene-grouping-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..89c60926a4a82
--- /dev/null
+++ b/server/licenses/lucene-grouping-8.5.0.jar.sha1
@@ -0,0 +1 @@
+08d26d94f32b38d15eaf68b17bef52158e4bbc87
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index cfa7fa961780b..0000000000000
--- a/server/licenses/lucene-highlighter-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5cddb5b65e7ead641483dcc2ffb0e50ad8d26eb7
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-8.5.0.jar.sha1 b/server/licenses/lucene-highlighter-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..9ec3d4b53fef0
--- /dev/null
+++ b/server/licenses/lucene-highlighter-8.5.0.jar.sha1
@@ -0,0 +1 @@
+8c653f47ea042dec2920bab83b039774b567eb9f
\ No newline at end of file
diff --git a/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 94a18fd3c3913..0000000000000
--- a/server/licenses/lucene-join-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-cfabaedd80fe600cc7fda5ee12d90927fa96d87c
\ No newline at end of file
diff --git a/server/licenses/lucene-join-8.5.0.jar.sha1 b/server/licenses/lucene-join-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..fa153682841ab
--- /dev/null
+++ b/server/licenses/lucene-join-8.5.0.jar.sha1
@@ -0,0 +1 @@
+98713495a3f48558a5acd7aaa22bfa7da394e78a
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index ae7da5c259f69..0000000000000
--- a/server/licenses/lucene-memory-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e90fbcc53531978fc03ef847ba396d4cdd89c7e4
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-8.5.0.jar.sha1 b/server/licenses/lucene-memory-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..402e037eb1a20
--- /dev/null
+++ b/server/licenses/lucene-memory-8.5.0.jar.sha1
@@ -0,0 +1 @@
+18e3ed87c7f29bb0fe4b5db244a4f31018a9e518
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 8f6c2fe6d63c1..0000000000000
--- a/server/licenses/lucene-misc-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd0b4cef132a50b3fa919f214a5316fcc78c46ea
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-8.5.0.jar.sha1 b/server/licenses/lucene-misc-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..fa96c7ab7e70c
--- /dev/null
+++ b/server/licenses/lucene-misc-8.5.0.jar.sha1
@@ -0,0 +1 @@
+9786e13764f55dbf28e71fe7e0a90d1e94bea0bc
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 4b2a346309f3f..0000000000000
--- a/server/licenses/lucene-queries-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c90cc35089afc3f7802668c3969b5e7391b6d15a
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-8.5.0.jar.sha1 b/server/licenses/lucene-queries-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..ec92e3918b4d5
--- /dev/null
+++ b/server/licenses/lucene-queries-8.5.0.jar.sha1
@@ -0,0 +1 @@
+708c1f850ed70c506822b021a722e42f29c397a1
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 3b5ec91e4f938..0000000000000
--- a/server/licenses/lucene-queryparser-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e324233cb8f069e4f6abcbab47368a83c3696f36
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-8.5.0.jar.sha1 b/server/licenses/lucene-queryparser-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..3fe3e5fc193b0
--- /dev/null
+++ b/server/licenses/lucene-queryparser-8.5.0.jar.sha1
@@ -0,0 +1 @@
+13c38f39b1a7d10c4749ba789fa95da5868d4885
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 58d7518e7aefc..0000000000000
--- a/server/licenses/lucene-sandbox-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-985a451e5f564c84271419a446e044ab589d6f22
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-8.5.0.jar.sha1 b/server/licenses/lucene-sandbox-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..f36b8b978bede
--- /dev/null
+++ b/server/licenses/lucene-sandbox-8.5.0.jar.sha1
@@ -0,0 +1 @@
+2b275921f2fd92b15b4f1a2a565467c3fa221ef9
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index f51b876ad5fdf..0000000000000
--- a/server/licenses/lucene-spatial-extras-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-beff7cafe0fa5330b9b915825b69321faf0fcaa9
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-8.5.0.jar.sha1 b/server/licenses/lucene-spatial-extras-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..02c7de2cd466f
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-8.5.0.jar.sha1
@@ -0,0 +1 @@
+a8603576227b03fa94c2cde81b877f711c8b4c3f
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 736c206ba1b48..0000000000000
--- a/server/licenses/lucene-spatial3d-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b9256d3a2a64d79435a4c726af8a3c28c2b77d7f
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-8.5.0.jar.sha1 b/server/licenses/lucene-spatial3d-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..df28d9295c4a0
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-8.5.0.jar.sha1
@@ -0,0 +1 @@
+6f05de532ca9497e4d8364addf123441b01372a8
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1 b/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index 586728768c5b4..0000000000000
--- a/server/licenses/lucene-suggest-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f38949db273a910e94a57229db2d8f3e4aef5e1f
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-8.5.0.jar.sha1 b/server/licenses/lucene-suggest-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..b0abc5dc57319
--- /dev/null
+++ b/server/licenses/lucene-suggest-8.5.0.jar.sha1
@@ -0,0 +1 @@
+4ae6bea433acecbbaf7ae8fa3d56941ec2ad004d
\ No newline at end of file
diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java
index efbf8d25c9a2c..8b3b14d87b1e9 100644
--- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -276,7 +276,7 @@ public Throwable getRootCause() {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(this.getMessage());
         out.writeException(this.getCause());
-        writeStackTraces(this, out);
+        writeStackTraces(this, out, StreamOutput::writeException);
         out.writeMapOfLists(headers, StreamOutput::writeString, StreamOutput::writeString);
         out.writeMapOfLists(metadata, StreamOutput::writeString, StreamOutput::writeString);
     }
@@ -715,7 +715,8 @@ public static <T extends Throwable> T readStackTrace(T throwable, StreamInput in
     /**
      * Serializes the given exceptions stacktrace elements as well as it's suppressed exceptions to the given output stream.
      */
-    public static <T extends Throwable> T writeStackTraces(T throwable, StreamOutput out) throws IOException {
+    public static <T extends Throwable> T writeStackTraces(T throwable, StreamOutput out,
+                                                           Writer<Throwable> exceptionWriter) throws IOException {
         StackTraceElement[] stackTrace = throwable.getStackTrace();
         out.writeVInt(stackTrace.length);
         for (StackTraceElement element : stackTrace) {
@@ -727,7 +728,7 @@ public static <T extends Throwable> T writeStackTraces(T throwable, StreamOutput
         Throwable[] suppressed = throwable.getSuppressed();
         out.writeVInt(suppressed.length);
         for (Throwable t : suppressed) {
-            out.writeException(t);
+            exceptionWriter.write(out, t);
         }
         return throwable;
     }
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index a006a265916a3..120adc9faca0e 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -76,6 +76,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_7_6_1 = new Version(7060199, org.apache.lucene.util.Version.LUCENE_8_4_0);
     public static final Version V_7_6_2 = new Version(7060299, org.apache.lucene.util.Version.LUCENE_8_4_0);
     public static final Version V_7_7_0 = new Version(7070099, org.apache.lucene.util.Version.LUCENE_8_5_0);
+    public static final Version V_7_8_0 = new Version(7080099, org.apache.lucene.util.Version.LUCENE_8_5_0);
     public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_5_0);
     public static final Version CURRENT = V_8_0_0;
 
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 09814a4059b71..23d1820d76ecb 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -193,10 +193,12 @@
 import org.elasticsearch.action.ingest.SimulatePipelineTransportAction;
 import org.elasticsearch.action.main.MainAction;
 import org.elasticsearch.action.main.TransportMainAction;
+import org.elasticsearch.action.search.ClearReaderAction;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.MultiSearchAction;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchScrollAction;
+import org.elasticsearch.action.search.TransportClearReaderAction;
 import org.elasticsearch.action.search.TransportClearScrollAction;
 import org.elasticsearch.action.search.TransportMultiSearchAction;
 import org.elasticsearch.action.search.TransportSearchAction;
@@ -215,6 +217,7 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.NamedRegistry;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.TypeLiteral;
@@ -347,6 +350,7 @@
 import org.elasticsearch.rest.action.ingest.RestGetPipelineAction;
 import org.elasticsearch.rest.action.ingest.RestPutPipelineAction;
 import org.elasticsearch.rest.action.ingest.RestSimulatePipelineAction;
+import org.elasticsearch.rest.action.search.RestClearReaderAction;
 import org.elasticsearch.rest.action.search.RestClearScrollAction;
 import org.elasticsearch.rest.action.search.RestCountAction;
 import org.elasticsearch.rest.action.search.RestExplainAction;
@@ -419,17 +423,19 @@ public class ActionModule extends AbstractModule {
     private final RestController restController;
     private final RequestValidators<PutMappingRequest> mappingRequestValidators;
     private final RequestValidators<IndicesAliasesRequest> indicesAliasesRequestRequestValidators;
+    private final ClusterService clusterService;
 
     public ActionModule(Settings settings, IndexNameExpressionResolver indexNameExpressionResolver,
                         IndexScopedSettings indexScopedSettings, ClusterSettings clusterSettings, SettingsFilter settingsFilter,
                         ThreadPool threadPool, List<ActionPlugin> actionPlugins, NodeClient nodeClient,
-                        CircuitBreakerService circuitBreakerService, UsageService usageService) {
+                        CircuitBreakerService circuitBreakerService, UsageService usageService, ClusterService clusterService) {
         this.settings = settings;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.indexScopedSettings = indexScopedSettings;
         this.clusterSettings = clusterSettings;
         this.settingsFilter = settingsFilter;
         this.actionPlugins = actionPlugins;
+        this.clusterService = clusterService;
         actions = setupActions(actionPlugins);
         actionFilters = setupActionFilters(actionPlugins);
         autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver);
@@ -454,10 +460,10 @@ public ActionModule(Settings settings, IndexNameExpressionResolver indexNameExpr
         indicesAliasesRequestRequestValidators = new RequestValidators<>(
                 actionPlugins.stream().flatMap(p -> p.indicesAliasesRequestValidators().stream()).collect(Collectors.toList()));
 
-        final boolean restrictSystemIndices = RestController.RESTRICT_SYSTEM_INDICES.get(settings);
-        restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService, restrictSystemIndices);
+        restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService);
     }
 
+
     public Map<String, ActionHandler<?, ?>> getActions() {
         return actions;
     }
@@ -563,6 +569,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(MultiSearchAction.INSTANCE, TransportMultiSearchAction.class);
         actions.register(ExplainAction.INSTANCE, TransportExplainAction.class);
         actions.register(ClearScrollAction.INSTANCE, TransportClearScrollAction.class);
+        actions.register(ClearReaderAction.INSTANCE, TransportClearReaderAction.class);
         actions.register(RecoveryAction.INSTANCE, TransportRecoveryAction.class);
         actions.register(NodesReloadSecureSettingsAction.INSTANCE, TransportNodesReloadSecureSettingsAction.class);
 
@@ -698,7 +705,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
 
         registerHandler.accept(new RestIndexAction());
         registerHandler.accept(new CreateHandler());
-        registerHandler.accept(new AutoIdHandler(nodesInCluster));
+        registerHandler.accept(new AutoIdHandler(clusterService));
         registerHandler.accept(new RestGetAction());
         registerHandler.accept(new RestGetSourceAction());
         registerHandler.accept(new RestMultiGetAction(settings));
@@ -712,6 +719,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestSearchAction());
         registerHandler.accept(new RestSearchScrollAction());
         registerHandler.accept(new RestClearScrollAction());
+        registerHandler.accept(new RestClearReaderAction());
         registerHandler.accept(new RestMultiSearchAction(settings));
 
         registerHandler.accept(new RestValidateQueryAction());
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
index 6f33f75692294..9ed72f2fb95be 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
@@ -27,6 +27,8 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.stream.Collectors;
 
 /**
@@ -34,7 +36,7 @@
  */
 public class NodesInfoRequest extends BaseNodesRequest<NodesInfoRequest> {
 
-    private Set<String> requestedMetrics = Metrics.allMetrics();
+    private Set<String> requestedMetrics = Metric.allMetrics();
 
     /**
      * Create a new NodeInfoRequest from a {@link StreamInput} object.
@@ -48,16 +50,16 @@ public NodesInfoRequest(StreamInput in) throws IOException {
         if (in.getVersion().before(Version.V_7_7_0)){
             // prior to version 8.x, a NodesInfoRequest was serialized as a list
             // of booleans in a fixed order
-            addOrRemoveMetric(in.readBoolean(), Metrics.SETTINGS.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metrics.OS.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metrics.PROCESS.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metrics.JVM.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metrics.THREAD_POOL.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metrics.TRANSPORT.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metrics.HTTP.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metrics.PLUGINS.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metrics.INGEST.metricName());
-            addOrRemoveMetric(in.readBoolean(), Metrics.INDICES.metricName());
+            optionallyAddMetric(in.readBoolean(), Metric.SETTINGS.metricName());
+            optionallyAddMetric(in.readBoolean(), Metric.OS.metricName());
+            optionallyAddMetric(in.readBoolean(), Metric.PROCESS.metricName());
+            optionallyAddMetric(in.readBoolean(), Metric.JVM.metricName());
+            optionallyAddMetric(in.readBoolean(), Metric.THREAD_POOL.metricName());
+            optionallyAddMetric(in.readBoolean(), Metric.TRANSPORT.metricName());
+            optionallyAddMetric(in.readBoolean(), Metric.HTTP.metricName());
+            optionallyAddMetric(in.readBoolean(), Metric.PLUGINS.metricName());
+            optionallyAddMetric(in.readBoolean(), Metric.INGEST.metricName());
+            optionallyAddMetric(in.readBoolean(), Metric.INDICES.metricName());
         } else {
             requestedMetrics.addAll(Arrays.asList(in.readStringArray()));
         }
@@ -84,174 +86,63 @@ public NodesInfoRequest clear() {
      * Sets to return all the data.
      */
     public NodesInfoRequest all() {
-        requestedMetrics.addAll(Metrics.allMetrics());
+        requestedMetrics.addAll(Metric.allMetrics());
         return this;
     }
 
     /**
-     * Should the node settings be returned.
+     * Get the names of requested metrics
      */
-    public boolean settings() {
-        return Metrics.SETTINGS.containedIn(requestedMetrics);
+    public Set<String> requestedMetrics() {
+        return Set.copyOf(requestedMetrics);
     }
 
     /**
-     * Should the node settings be returned.
+     * Add metric
      */
-    public NodesInfoRequest settings(boolean settings) {
-        addOrRemoveMetric(settings, Metrics.SETTINGS.metricName());
-        return this;
-    }
-
-    /**
-     * Should the node OS be returned.
-     */
-    public boolean os() {
-        return Metrics.OS.containedIn(requestedMetrics);
-    }
-
-    /**
-     * Should the node OS be returned.
-     */
-    public NodesInfoRequest os(boolean os) {
-        addOrRemoveMetric(os, Metrics.OS.metricName());
-        return this;
-    }
-
-    /**
-     * Should the node Process be returned.
-     */
-    public boolean process() {
-        return Metrics.PROCESS.containedIn(requestedMetrics);
-    }
-
-    /**
-     * Should the node Process be returned.
-     */
-    public NodesInfoRequest process(boolean process) {
-        addOrRemoveMetric(process, Metrics.PROCESS.metricName());
-        return this;
-    }
-
-    /**
-     * Should the node JVM be returned.
-     */
-    public boolean jvm() {
-        return Metrics.JVM.containedIn(requestedMetrics);
-    }
-
-    /**
-     * Should the node JVM be returned.
-     */
-    public NodesInfoRequest jvm(boolean jvm) {
-        addOrRemoveMetric(jvm, Metrics.JVM.metricName());
-        return this;
-    }
-
-    /**
-     * Should the node Thread Pool info be returned.
-     */
-    public boolean threadPool() {
-        return Metrics.THREAD_POOL.containedIn(requestedMetrics);
-    }
-
-    /**
-     * Should the node Thread Pool info be returned.
-     */
-    public NodesInfoRequest threadPool(boolean threadPool) {
-        addOrRemoveMetric(threadPool, Metrics.THREAD_POOL.metricName());
-        return this;
-    }
-
-    /**
-     * Should the node Transport be returned.
-     */
-    public boolean transport() {
-        return Metrics.TRANSPORT.containedIn(requestedMetrics);
-    }
-
-    /**
-     * Should the node Transport be returned.
-     */
-    public NodesInfoRequest transport(boolean transport) {
-        addOrRemoveMetric(transport, Metrics.TRANSPORT.metricName());
-        return this;
-    }
-
-    /**
-     * Should the node HTTP be returned.
-     */
-    public boolean http() {
-        return Metrics.HTTP.containedIn(requestedMetrics);
-    }
-
-    /**
-     * Should the node HTTP be returned.
-     */
-    public NodesInfoRequest http(boolean http) {
-        addOrRemoveMetric(http, Metrics.HTTP.metricName());
-        return this;
-    }
-
-    /**
-     * Should information about plugins be returned
-     * @param plugins true if you want info
-     * @return The request
-     */
-    public NodesInfoRequest plugins(boolean plugins) {
-        addOrRemoveMetric(plugins, Metrics.PLUGINS.metricName());
+    public NodesInfoRequest addMetric(String metric) {
+        if (Metric.allMetrics().contains(metric) == false) {
+            throw new IllegalStateException("Used an illegal metric: " + metric);
+        }
+        requestedMetrics.add(metric);
         return this;
     }
 
     /**
-     * @return true if information about plugins is requested
-     */
-    public boolean plugins() {
-        return Metrics.PLUGINS.containedIn(requestedMetrics);
-    }
-
-    /**
-     * Should information about ingest be returned
-     * @param ingest true if you want info
+     * Add multiple metrics
      */
-    public NodesInfoRequest ingest(boolean ingest) {
-        addOrRemoveMetric(ingest, Metrics.INGEST.metricName());
+    public NodesInfoRequest addMetrics(String... metrics) {
+        SortedSet<String> metricsSet = new TreeSet<>(Set.of(metrics));
+        if (Metric.allMetrics().containsAll(metricsSet) == false) {
+            metricsSet.removeAll(Metric.allMetrics());
+            String plural = metricsSet.size() == 1 ? "" : "s";
+            throw new IllegalStateException("Used illegal metric" + plural + ": " + metricsSet);
+        }
+        requestedMetrics.addAll(metricsSet);
         return this;
     }
 
     /**
-     * @return true if information about ingest is requested
+     * Remove metric
      */
-    public boolean ingest() {
-        return Metrics.INGEST.containedIn(requestedMetrics);
-    }
-
-    /**
-     * Should information about indices (currently just indexing buffers) be returned
-     * @param indices true if you want info
-     */
-    public NodesInfoRequest indices(boolean indices) {
-        addOrRemoveMetric(indices, Metrics.INDICES.metricName());
+    public NodesInfoRequest removeMetric(String metric) {
+        if (Metric.allMetrics().contains(metric) == false) {
+            throw new IllegalStateException("Used an illegal metric: " + metric);
+        }
+        requestedMetrics.remove(metric);
         return this;
     }
 
     /**
-     * @return true if information about indices (currently just indexing buffers)
-     */
-    public boolean indices() {
-        return Metrics.INDICES.containedIn(requestedMetrics);
-    }
-
-    /**
-     * Helper method for adding and removing metrics.
-     * @param includeMetric Whether or not to include a metric.
+     * Helper method for adding and removing metrics. Used when deserializing
+     * a NodesInfoRequest from an ordered list of booleans.
+     *
+     * @param addMetric Whether or not to include a metric.
      * @param metricName Name of the metric to include or remove.
      */
-    private void addOrRemoveMetric(boolean includeMetric, String metricName) {
-        if (includeMetric) {
+    private void optionallyAddMetric(boolean addMetric, String metricName) {
+        if (addMetric) {
             requestedMetrics.add(metricName);
-        } else {
-            requestedMetrics.remove(metricName);
         }
     }
 
@@ -261,16 +152,16 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().before(Version.V_7_7_0)){
             // prior to version 8.x, a NodesInfoRequest was serialized as a list
             // of booleans in a fixed order
-            out.writeBoolean(Metrics.SETTINGS.containedIn(requestedMetrics));
-            out.writeBoolean(Metrics.OS.containedIn(requestedMetrics));
-            out.writeBoolean(Metrics.PROCESS.containedIn(requestedMetrics));
-            out.writeBoolean(Metrics.JVM.containedIn(requestedMetrics));
-            out.writeBoolean(Metrics.THREAD_POOL.containedIn(requestedMetrics));
-            out.writeBoolean(Metrics.TRANSPORT.containedIn(requestedMetrics));
-            out.writeBoolean(Metrics.HTTP.containedIn(requestedMetrics));
-            out.writeBoolean(Metrics.PLUGINS.containedIn(requestedMetrics));
-            out.writeBoolean(Metrics.INGEST.containedIn(requestedMetrics));
-            out.writeBoolean(Metrics.INDICES.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.SETTINGS.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.OS.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.PROCESS.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.JVM.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.THREAD_POOL.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.TRANSPORT.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.HTTP.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.PLUGINS.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.INGEST.containedIn(requestedMetrics));
+            out.writeBoolean(Metric.INDICES.containedIn(requestedMetrics));
         } else {
             out.writeStringArray(requestedMetrics.toArray(String[]::new));
         }
@@ -281,7 +172,7 @@ public void writeTo(StreamOutput out) throws IOException {
      * from the nodes information endpoint. Eventually this list list will be
      * pluggable.
      */
-    enum Metrics {
+    public enum Metric {
         SETTINGS("settings"),
         OS("os"),
         PROCESS("process"),
@@ -295,11 +186,11 @@ enum Metrics {
 
         private String metricName;
 
-        Metrics(String name) {
+        Metric(String name) {
             this.metricName = name;
         }
 
-        String metricName() {
+        public String metricName() {
             return this.metricName;
         }
 
@@ -307,8 +198,8 @@ boolean containedIn(Set<String> metricNames) {
             return metricNames.contains(this.metricName());
         }
 
-        static Set<String> allMetrics() {
-            return Arrays.stream(values()).map(Metrics::metricName).collect(Collectors.toSet());
+        public static Set<String> allMetrics() {
+            return Arrays.stream(values()).map(Metric::metricName).collect(Collectors.toSet());
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java
index 1222101616069..97b55b7fd496a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 
+// TODO: This class's interface should match that of NodesInfoRequest
 public class NodesInfoRequestBuilder extends NodesOperationRequestBuilder<NodesInfoRequest, NodesInfoResponse, NodesInfoRequestBuilder> {
 
     public NodesInfoRequestBuilder(ElasticsearchClient client, NodesInfoAction action) {
@@ -48,7 +49,7 @@ public NodesInfoRequestBuilder all() {
      * Should the node settings be returned.
      */
     public NodesInfoRequestBuilder setSettings(boolean settings) {
-        request.settings(settings);
+        addOrRemoveMetric(settings, NodesInfoRequest.Metric.SETTINGS);
         return this;
     }
 
@@ -56,7 +57,7 @@ public NodesInfoRequestBuilder setSettings(boolean settings) {
      * Should the node OS info be returned.
      */
     public NodesInfoRequestBuilder setOs(boolean os) {
-        request.os(os);
+        addOrRemoveMetric(os, NodesInfoRequest.Metric.OS);
         return this;
     }
 
@@ -64,7 +65,7 @@ public NodesInfoRequestBuilder setOs(boolean os) {
      * Should the node OS process be returned.
      */
     public NodesInfoRequestBuilder setProcess(boolean process) {
-        request.process(process);
+        addOrRemoveMetric(process, NodesInfoRequest.Metric.PROCESS);
         return this;
     }
 
@@ -72,7 +73,7 @@ public NodesInfoRequestBuilder setProcess(boolean process) {
      * Should the node JVM info be returned.
      */
     public NodesInfoRequestBuilder setJvm(boolean jvm) {
-        request.jvm(jvm);
+        addOrRemoveMetric(jvm, NodesInfoRequest.Metric.JVM);
         return this;
     }
 
@@ -80,7 +81,7 @@ public NodesInfoRequestBuilder setJvm(boolean jvm) {
      * Should the node thread pool info be returned.
      */
     public NodesInfoRequestBuilder setThreadPool(boolean threadPool) {
-        request.threadPool(threadPool);
+        addOrRemoveMetric(threadPool, NodesInfoRequest.Metric.THREAD_POOL);
         return this;
     }
 
@@ -88,7 +89,7 @@ public NodesInfoRequestBuilder setThreadPool(boolean threadPool) {
      * Should the node Transport info be returned.
      */
     public NodesInfoRequestBuilder setTransport(boolean transport) {
-        request.transport(transport);
+        addOrRemoveMetric(transport, NodesInfoRequest.Metric.TRANSPORT);
         return this;
     }
 
@@ -96,7 +97,7 @@ public NodesInfoRequestBuilder setTransport(boolean transport) {
      * Should the node HTTP info be returned.
      */
     public NodesInfoRequestBuilder setHttp(boolean http) {
-        request.http(http);
+        addOrRemoveMetric(http, NodesInfoRequest.Metric.HTTP);
         return this;
     }
 
@@ -104,7 +105,7 @@ public NodesInfoRequestBuilder setHttp(boolean http) {
      * Should the node plugins info be returned.
      */
     public NodesInfoRequestBuilder setPlugins(boolean plugins) {
-        request().plugins(plugins);
+        addOrRemoveMetric(plugins, NodesInfoRequest.Metric.PLUGINS);
         return this;
     }
 
@@ -112,7 +113,7 @@ public NodesInfoRequestBuilder setPlugins(boolean plugins) {
      * Should the node ingest info be returned.
      */
     public NodesInfoRequestBuilder setIngest(boolean ingest) {
-        request().ingest(ingest);
+        addOrRemoveMetric(ingest, NodesInfoRequest.Metric.INGEST);
         return this;
     }
 
@@ -120,7 +121,15 @@ public NodesInfoRequestBuilder setIngest(boolean ingest) {
      * Should the node indices info be returned.
      */
     public NodesInfoRequestBuilder setIndices(boolean indices) {
-        request().indices(indices);
+        addOrRemoveMetric(indices, NodesInfoRequest.Metric.INDICES);
         return this;
     }
+
+    private void addOrRemoveMetric(boolean includeMetric, NodesInfoRequest.Metric metric) {
+        if (includeMetric) {
+            request.addMetric(metric.metricName());
+        } else {
+            request.removeMetric(metric.metricName());
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
index cd960d75da3a4..dcfdfedda125f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
@@ -34,6 +34,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Set;
 
 public class TransportNodesInfoAction extends TransportNodesAction<NodesInfoRequest,
                                                                    NodesInfoResponse,
@@ -69,8 +70,18 @@ protected NodeInfo newNodeResponse(StreamInput in) throws IOException {
     @Override
     protected NodeInfo nodeOperation(NodeInfoRequest nodeRequest, Task task) {
         NodesInfoRequest request = nodeRequest.request;
-        return nodeService.info(request.settings(), request.os(), request.process(), request.jvm(), request.threadPool(),
-                request.transport(), request.http(), request.plugins(), request.ingest(), request.indices());
+        Set<String> metrics = request.requestedMetrics();
+        return nodeService.info(
+            metrics.contains(NodesInfoRequest.Metric.SETTINGS.metricName()),
+            metrics.contains(NodesInfoRequest.Metric.OS.metricName()),
+            metrics.contains(NodesInfoRequest.Metric.PROCESS.metricName()),
+            metrics.contains(NodesInfoRequest.Metric.JVM.metricName()),
+            metrics.contains(NodesInfoRequest.Metric.THREAD_POOL.metricName()),
+            metrics.contains(NodesInfoRequest.Metric.TRANSPORT.metricName()),
+            metrics.contains(NodesInfoRequest.Metric.HTTP.metricName()),
+            metrics.contains(NodesInfoRequest.Metric.PLUGINS.metricName()),
+            metrics.contains(NodesInfoRequest.Metric.INGEST.metricName()),
+            metrics.contains(NodesInfoRequest.Metric.INDICES.metricName()));
     }
 
     public static class NodeInfoRequest extends BaseNodeRequest {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java
index df6e829a28af4..37f190691f33a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java
@@ -18,30 +18,42 @@
  */
 package org.elasticsearch.action.admin.indices.datastream;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Objects;
 
 public class CreateDataStreamAction extends ActionType<AcknowledgedResponse> {
 
+    private static final Logger logger = LogManager.getLogger(CreateDataStreamAction.class);
+
     public static final CreateDataStreamAction INSTANCE = new CreateDataStreamAction();
     public static final String NAME = "indices:admin/data_stream/create";
 
@@ -64,7 +76,14 @@ public void setTimestampFieldName(String timestampFieldName) {
 
         @Override
         public ActionRequestValidationException validate() {
-            return null;
+            ActionRequestValidationException validationException = null;
+            if (Strings.hasText(name) == false) {
+                validationException = ValidateActions.addValidationError("name is missing", validationException);
+            }
+            if (Strings.hasText(timestampFieldName) == false) {
+                validationException = ValidateActions.addValidationError("timestamp field name is missing", validationException);
+            }
+            return validationException;
         }
 
         public Request(StreamInput in) throws IOException {
@@ -116,7 +135,41 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException {
         @Override
         protected void masterOperation(Task task, Request request, ClusterState state,
                                        ActionListener<AcknowledgedResponse> listener) throws Exception {
-            listener.onResponse(new AcknowledgedResponse(true));
+            clusterService.submitStateUpdateTask("create-data-stream [" + request.name + "]",
+                new ClusterStateUpdateTask(Priority.HIGH) {
+
+                    @Override
+                    public TimeValue timeout() {
+                        return request.masterNodeTimeout();
+                    }
+
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        listener.onFailure(e);
+                    }
+
+                    @Override
+                    public ClusterState execute(ClusterState currentState) throws Exception {
+                        return createDataStream(currentState, request);
+                    }
+
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        listener.onResponse(new AcknowledgedResponse(true));
+                    }
+                });
+        }
+
+        static ClusterState createDataStream(ClusterState currentState, Request request) {
+            if (currentState.metaData().dataStreams().containsKey(request.name)) {
+                throw new IllegalArgumentException("data_stream [" + request.name + "] already exists");
+            }
+
+            MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(
+                new DataStream(request.name, request.timestampFieldName, Collections.emptyList()));
+
+            logger.info("adding data stream [{}]", request.name);
+            return ClusterState.builder(currentState).metaData(builder).build();
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java
index 20a2ba4aa2cd6..91444ef64320b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java
@@ -18,30 +18,44 @@
  */
 package org.elasticsearch.action.admin.indices.datastream;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.Objects;
+import java.util.Set;
 
 public class DeleteDataStreamAction extends ActionType<AcknowledgedResponse> {
 
+    private static final Logger logger = LogManager.getLogger(DeleteDataStreamAction.class);
+
     public static final DeleteDataStreamAction INSTANCE = new DeleteDataStreamAction();
     public static final String NAME = "indices:admin/data_stream/delete";
 
@@ -59,7 +73,11 @@ public Request(String name) {
 
         @Override
         public ActionRequestValidationException validate() {
-            return null;
+            ActionRequestValidationException validationException = null;
+            if (Strings.hasText(name) == false) {
+                validationException = ValidateActions.addValidationError("name is missing", validationException);
+            }
+            return validationException;
         }
 
         public Request(StreamInput in) throws IOException {
@@ -108,7 +126,51 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException {
         @Override
         protected void masterOperation(Task task, Request request, ClusterState state,
                                        ActionListener<AcknowledgedResponse> listener) throws Exception {
-            listener.onResponse(new AcknowledgedResponse(true));
+            clusterService.submitStateUpdateTask("remove-data-stream [" + request.name + "]", new ClusterStateUpdateTask(Priority.HIGH) {
+
+                @Override
+                public TimeValue timeout() {
+                    return request.masterNodeTimeout();
+                }
+
+                @Override
+                public void onFailure(String source, Exception e) {
+                    listener.onFailure(e);
+                }
+
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    return removeDataStream(currentState, request);
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    listener.onResponse(new AcknowledgedResponse(true));
+                }
+            });
+        }
+
+        static ClusterState removeDataStream(ClusterState currentState, Request request) {
+            Set<String> dataStreams = new HashSet<>();
+            for (String dataStreamName : currentState.metaData().dataStreams().keySet()) {
+                if (Regex.simpleMatch(request.name, dataStreamName)) {
+                    dataStreams.add(dataStreamName);
+                }
+            }
+            if (dataStreams.isEmpty()) {
+                // if a match-all pattern was specified and no data streams were found because none exist, do not
+                // fail with data stream missing exception
+                if (Regex.isMatchAllPattern(request.name)) {
+                    return currentState;
+                }
+                throw new ResourceNotFoundException("data_streams matching [" + request.name + "] not found");
+            }
+            MetaData.Builder metaData = MetaData.builder(currentState.metaData());
+            for (String dataStreamName : dataStreams) {
+                logger.info("removing data stream [{}]", dataStreamName);
+                metaData.removeDataStream(dataStreamName);
+            }
+            return ClusterState.builder(currentState).metaData(metaData).build();
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java
index 1549f056e811f..ed2411401aeb6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.Task;
@@ -41,8 +42,10 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 public class GetDataStreamsAction extends ActionType<GetDataStreamsAction.Response> {
@@ -154,11 +157,30 @@ protected Response read(StreamInput in) throws IOException {
         @Override
         protected void masterOperation(Task task, Request request, ClusterState state,
                                        ActionListener<Response> listener) throws Exception {
-            List<DataStream> dataStreams = List.of(
-                new DataStream("my_data_stream1", "@timestamp", List.of("my_data_stream1-000000")),
-                new DataStream("my_data_stream2", "@timestamp", List.of())
-            );
-            listener.onResponse(new Response(dataStreams));
+            listener.onResponse(new Response(getDataStreams(state, request)));
+        }
+
+        static List<DataStream> getDataStreams(ClusterState clusterState, Request request) {
+            Map<String, DataStream> dataStreams = clusterState.metaData().dataStreams();
+
+            // return all data streams if no name was specified
+            if (request.names.length == 0) {
+                return new ArrayList<>(dataStreams.values());
+            }
+
+            final List<DataStream> results = new ArrayList<>();
+            for (String name : request.names) {
+                if (Regex.isSimpleMatchPattern(name)) {
+                    for (Map.Entry<String, DataStream> entry : dataStreams.entrySet()) {
+                        if (Regex.simpleMatch(name, entry.getKey())) {
+                            results.add(entry.getValue());
+                        }
+                    }
+                } else if (dataStreams.containsKey(name)) {
+                    results.add(dataStreams.get(name));
+                }
+            }
+            return results;
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
index c915a56231472..bc2816c5614b5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -45,14 +46,17 @@ public class TransportPutComponentTemplateAction
     extends TransportMasterNodeAction<PutComponentTemplateAction.Request, AcknowledgedResponse> {
 
     private final MetaDataIndexTemplateService indexTemplateService;
+    private final IndexScopedSettings indexScopedSettings;
 
     @Inject
     public TransportPutComponentTemplateAction(TransportService transportService, ClusterService clusterService,
                                                ThreadPool threadPool, MetaDataIndexTemplateService indexTemplateService,
-                                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+                                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                               IndexScopedSettings indexScopedSettings) {
         super(PutComponentTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters,
             PutComponentTemplateAction.Request::new, indexNameExpressionResolver);
         this.indexTemplateService = indexTemplateService;
+        this.indexScopedSettings = indexScopedSettings;
     }
 
     @Override
@@ -78,8 +82,10 @@ protected void masterOperation(Task task, final PutComponentTemplateAction.Reque
         Template template = componentTemplate.template();
         // Normalize the index settings if necessary
         if (template.settings() != null) {
-            Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
-            template = new Template(settings.build(), template.mappings(), template.aliases());
+            Settings.Builder builder = Settings.builder().put(template.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
+            Settings settings = builder.build();
+            indexScopedSettings.validate(settings, true);
+            template = new Template(settings, template.mappings(), template.aliases());
             componentTemplate = new ComponentTemplate(template, componentTemplate.version(), componentTemplate.metadata());
         }
         indexTemplateService.putComponentTemplate(request.cause(), request.create(), request.name(), request.masterNodeTimeout(),
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
index 4a651b70a28bd..4817d76f1806d 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
@@ -420,8 +420,7 @@ public boolean isCancelled() {
                 }
             };
         }
-        final Runnable flushRunnable = scheduler.preserveContext(new Flush());
-        return scheduler.scheduleWithFixedDelay(flushRunnable, flushInterval, ThreadPool.Names.GENERIC);
+        return scheduler.scheduleWithFixedDelay(new Flush(), flushInterval, ThreadPool.Names.GENERIC);
     }
 
     // needs to be executed under a lock
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/Retry.java b/server/src/main/java/org/elasticsearch/action/bulk/Retry.java
index e2f90049759ec..727a2318556af 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/Retry.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/Retry.java
@@ -136,8 +136,7 @@ private void retry(BulkRequest bulkRequestForRetry) {
             assert backoff.hasNext();
             TimeValue next = backoff.next();
             logger.trace("Retry of bulk request scheduled in {} ms.", next.millis());
-            Runnable command = scheduler.preserveContext(() -> this.execute(bulkRequestForRetry));
-            retryCancellable = scheduler.schedule(command, next, ThreadPool.Names.SAME);
+            retryCancellable = scheduler.schedule(() -> this.execute(bulkRequestForRetry), next, ThreadPool.Names.SAME);
         }
 
         private BulkRequest createBulkRequestForRetry(BulkResponse bulkItemResponses) {
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index b0a88c512a75e..acc063e85fb82 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -96,6 +96,8 @@
  */
 public class TransportBulkAction extends HandledTransportAction<BulkRequest, BulkResponse> {
 
+    private static final Logger logger = LogManager.getLogger(TransportBulkAction.class);
+
     private final ThreadPool threadPool;
     private final AutoCreateIndex autoCreateIndex;
     private final ClusterService clusterService;
@@ -233,7 +235,7 @@ protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener<Bulk
             } else {
                 final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size());
                 for (String index : autoCreateIndices) {
-                    createIndex(index, bulkRequest.timeout(), new ActionListener<CreateIndexResponse>() {
+                    createIndex(index, bulkRequest.timeout(), new ActionListener<>() {
                         @Override
                         public void onResponse(CreateIndexResponse result) {
                             if (counter.decrementAndGet() == 0) {
@@ -634,7 +636,7 @@ private long relativeTime() {
         return relativeTimeProvider.getAsLong();
     }
 
-    void processBulkIndexIngestRequest(Task task, BulkRequest original, ActionListener<BulkResponse> listener) {
+    private void processBulkIndexIngestRequest(Task task, BulkRequest original, ActionListener<BulkResponse> listener) {
         final long ingestStartTimeInNanos = System.nanoTime();
         final BulkRequestModifier bulkRequestModifier = new BulkRequestModifier(original);
         ingestService.executeBulkRequest(
@@ -643,7 +645,7 @@ void processBulkIndexIngestRequest(Task task, BulkRequest original, ActionListen
             bulkRequestModifier::markItemAsFailed,
             (originalThread, exception) -> {
                 if (exception != null) {
-                    logger.error("failed to execute pipeline for a bulk request", exception);
+                    logger.debug("failed to execute pipeline for a bulk request", exception);
                     listener.onFailure(exception);
                 } else {
                     long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos);
@@ -692,7 +694,7 @@ public boolean isForceExecution() {
 
     static final class BulkRequestModifier implements Iterator<DocWriteRequest<?>> {
 
-        private static final Logger LOGGER = LogManager.getLogger(BulkRequestModifier.class);
+        private static final Logger logger = LogManager.getLogger(BulkRequestModifier.class);
 
         final BulkRequest bulkRequest;
         final SparseFixedBitSet failedSlots;
@@ -774,7 +776,7 @@ synchronized void markItemAsDropped(int slot) {
 
         synchronized void markItemAsFailed(int slot, Exception e) {
             IndexRequest indexRequest = getIndexWriteRequest(bulkRequest.requests().get(slot));
-            LOGGER.debug(() -> new ParameterizedMessage("failed to execute pipeline [{}] for document [{}/{}]",
+            logger.debug(() -> new ParameterizedMessage("failed to execute pipeline [{}] for document [{}/{}]",
                 indexRequest.getPipeline(), indexRequest.index(), indexRequest.id()), e);
 
             // We hit a error during preprocessing a request, so we:
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
index 7250369fae4be..c51f09c2da0dc 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
@@ -74,8 +74,8 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException {
     protected void masterOperation(Task task, PutPipelineRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener)
             throws Exception {
         NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
-        nodesInfoRequest.clear();
-        nodesInfoRequest.ingest(true);
+        nodesInfoRequest.clear()
+            .addMetric(NodesInfoRequest.Metric.INGEST.metricName());
         client.admin().cluster().nodesInfo(nodesInfoRequest, ActionListener.wrap(nodeInfos -> {
             Map<DiscoveryNode, IngestInfo> ingestInfos = new HashMap<>();
             for (NodeInfo nodeInfo : nodeInfos.getNodes()) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index f6f90cc7ccbb3..fef296de2c9c9 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.ShardOperationFailedException;
@@ -163,7 +164,7 @@ public final void start() {
             // total hits is null in the response if the tracking of total hits is disabled
             boolean withTotalHits = trackTotalHitsUpTo != SearchContext.TRACK_TOTAL_HITS_DISABLED;
             listener.onResponse(new SearchResponse(InternalSearchResponse.empty(withTotalHits), null, 0, 0, 0, buildTookInMillis(),
-                ShardSearchFailure.EMPTY_ARRAY, clusters));
+                ShardSearchFailure.EMPTY_ARRAY, clusters, null));
             return;
         }
         executePhase(this);
@@ -528,21 +529,24 @@ public final SearchRequest getRequest() {
     }
 
     protected final SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse,
-                                                       String scrollId,
+                                                       AtomicArray<SearchPhaseResult> queryResults,
                                                        ShardSearchFailure[] failures) {
+        final Version minNodeVersion = clusterState.nodes().getMinNodeVersion();
+        String scrollId = request.scroll() != null ? TransportSearchHelper.buildScrollId(queryResults, minNodeVersion) : null;
+        String readerId = request.reader() != null ? TransportSearchHelper.encodeReaderIds(queryResults, minNodeVersion) : null;
         return new SearchResponse(internalSearchResponse, scrollId, getNumShards(), successfulOps.get(),
-            skippedOps.get(), buildTookInMillis(), failures, clusters);
+            skippedOps.get(), buildTookInMillis(), failures, clusters, readerId);
     }
 
     @Override
-    public void sendSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId) {
+    public void sendSearchResponse(InternalSearchResponse internalSearchResponse, AtomicArray<SearchPhaseResult> queryResults) {
         ShardSearchFailure[] failures = buildShardFailures();
         Boolean allowPartialResults = request.allowPartialSearchResults();
         assert allowPartialResults != null : "SearchRequest missing setting for allowPartialSearchResults";
-        if (allowPartialResults == false && failures.length > 0){
+        if ((request.reader() == null || request.reader().getId() == null) && allowPartialResults == false && failures.length > 0) {
             raisePhaseFailure(new SearchPhaseExecutionException("", "Shard failures", null, failures));
         } else {
-            listener.onResponse(buildSearchResponse(internalSearchResponse, scrollId, failures));
+            listener.onResponse(buildSearchResponse(internalSearchResponse, queryResults, failures));
         }
     }
 
@@ -609,12 +613,13 @@ public final ShardSearchRequest buildShardSearchRequest(SearchShardIterator shar
         final String[] routings = indexRoutings.getOrDefault(indexName, Collections.emptySet())
             .toArray(new String[0]);
         ShardSearchRequest shardRequest = new ShardSearchRequest(shardIt.getOriginalIndices(), request, shardIt.shardId(), getNumShards(),
-            filter, indexBoost, timeProvider.getAbsoluteStartMillis(), shardIt.getClusterAlias(), routings);
+            filter, indexBoost, timeProvider.getAbsoluteStartMillis(), shardIt.getClusterAlias(), routings,
+            shardIt.getReaderId(), shardIt.getReaderKeepAlive());
         // if we already received a search result we can inform the shard that it
         // can return a null response if the request rewrites to match none rather
         // than creating an empty response in the search thread pool.
-        // Note that, we have to disable this shortcut for scroll queries.
-        shardRequest.canReturnNullResponseIfMatchNoDocs(hasShardResponse.get() && request.scroll() == null);
+        // Note that, we have to disable this shortcut for queries that create a context (scroll and reader_id).
+        shardRequest.canReturnNullResponseIfMatchNoDocs(hasShardResponse.get() && shardRequest.shouldCreatePersistentReader() == false);
         return shardRequest;
     }
 
@@ -684,8 +689,4 @@ private synchronized Runnable tryQueue(Runnable runnable) {
             return toExecute;
         }
     }
-
-    protected ClusterState clusterState() {
-        return clusterState;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java b/server/src/main/java/org/elasticsearch/action/search/ClearContextController.java
similarity index 71%
rename from server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
rename to server/src/main/java/org/elasticsearch/action/search/ClearContextController.java
index d0abf798501b3..a9c0da94a8bc8 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearContextController.java
@@ -28,51 +28,56 @@
 import org.elasticsearch.transport.TransportResponse;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.elasticsearch.action.search.TransportSearchHelper.parseScrollId;
 
-final class ClearScrollController implements Runnable {
+final class ClearContextController implements Runnable {
     private final DiscoveryNodes nodes;
     private final SearchTransportService searchTransportService;
     private final CountDown expectedOps;
-    private final ActionListener<ClearScrollResponse> listener;
+    private final ActionListener<? super ClearReaderResponse> listener;
     private final AtomicBoolean hasFailed = new AtomicBoolean(false);
     private final AtomicInteger freedSearchContexts = new AtomicInteger(0);
     private final Logger logger;
     private final Runnable runner;
 
-    ClearScrollController(ClearScrollRequest request, ActionListener<ClearScrollResponse> listener, DiscoveryNodes nodes, Logger logger,
-                          SearchTransportService searchTransportService) {
+    ClearContextController(ClearScrollRequest clearScrollRequest, ActionListener<ClearScrollResponse> listener,
+                           DiscoveryNodes nodes, Logger logger, SearchTransportService searchTransportService) {
         this.nodes = nodes;
         this.logger = logger;
         this.searchTransportService = searchTransportService;
         this.listener = listener;
-        List<String> scrollIds = request.getScrollIds();
         final int expectedOps;
+        final List<String> scrollIds = clearScrollRequest.getScrollIds();
         if (scrollIds.size() == 1 && "_all".equals(scrollIds.get(0))) {
             expectedOps = nodes.getSize();
             runner = this::cleanAllScrolls;
         } else {
-            List<ScrollIdForNode> parsedScrollIds = new ArrayList<>();
-            for (String parsedScrollId : request.getScrollIds()) {
-                ScrollIdForNode[] context = parseScrollId(parsedScrollId).getContext();
-                for (ScrollIdForNode id : context) {
-                    parsedScrollIds.add(id);
-                }
-            }
-            if (parsedScrollIds.isEmpty()) {
-                expectedOps = 0;
-                runner = () -> listener.onResponse(new ClearScrollResponse(true, 0));
-            } else {
-                expectedOps = parsedScrollIds.size();
-                runner = () -> cleanScrollIds(parsedScrollIds);
+            List<ReaderIdForNode> contexts = new ArrayList<>();
+            for (String scrollId : scrollIds) {
+                contexts.addAll(Arrays.asList(parseScrollId(scrollId).getContext()));
             }
+            expectedOps = contexts.size();
+            runner = () -> cleanReaderIds(contexts);
         }
         this.expectedOps = new CountDown(expectedOps);
+    }
 
+    ClearContextController(ClearReaderRequest clearReaderRequest, ActionListener<ClearReaderResponse> listener,
+                           DiscoveryNodes nodes, Logger logger, SearchTransportService searchTransportService) {
+        this.nodes = nodes;
+        this.logger = logger;
+        this.searchTransportService = searchTransportService;
+        this.listener = listener;
+        final Collection<ReaderIdForNode> contexts =
+            TransportSearchHelper.decodeReaderIds(clearReaderRequest.getId()).values();
+        expectedOps = new CountDown(contexts.size());
+        runner = () -> cleanReaderIds(contexts);
     }
 
     @Override
@@ -101,10 +106,14 @@ public void onFailure(Exception e) {
         }
     }
 
-    void cleanScrollIds(List<ScrollIdForNode> parsedScrollIds) {
-        SearchScrollAsyncAction.collectNodesAndRun(parsedScrollIds, nodes, searchTransportService, ActionListener.wrap(
+    void cleanReaderIds(Collection<ReaderIdForNode> readerIds) {
+        if (readerIds.isEmpty()) {
+            listener.onResponse(new ClearReaderResponse(true, 0));
+            return;
+        }
+        SearchScrollAsyncAction.collectNodesAndRun(readerIds, nodes, searchTransportService, ActionListener.wrap(
             lookup -> {
-                for (ScrollIdForNode target : parsedScrollIds) {
+                for (ReaderIdForNode target : readerIds) {
                     final DiscoveryNode node = lookup.apply(target.getClusterAlias(), target.getNode());
                     if (node == null) {
                         onFreedContext(false);
@@ -127,7 +136,7 @@ private void onFreedContext(boolean freed) {
         }
         if (expectedOps.countDown()) {
             boolean succeeded = hasFailed.get() == false;
-            listener.onResponse(new ClearScrollResponse(succeeded, freedSearchContexts.get()));
+            listener.onResponse(new ClearReaderResponse(succeeded, freedSearchContexts.get()));
         }
     }
 
@@ -139,7 +148,7 @@ private void onFailedFreedContext(Throwable e, DiscoveryNode node) {
          */
         hasFailed.set(true);
         if (expectedOps.countDown()) {
-            listener.onResponse(new ClearScrollResponse(false, freedSearchContexts.get()));
+            listener.onResponse(new ClearReaderResponse(false, freedSearchContexts.get()));
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearReaderAction.java b/server/src/main/java/org/elasticsearch/action/search/ClearReaderAction.java
new file mode 100644
index 0000000000000..d6215365e4aa9
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearReaderAction.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.ActionType;
+
+public class ClearReaderAction extends ActionType<ClearReaderResponse> {
+
+    public static final ClearReaderAction INSTANCE = new ClearReaderAction();
+    public static final String NAME = "indices:data/read/reader/clear";
+
+    private ClearReaderAction() {
+        super(NAME, ClearReaderResponse::new);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearReaderRequest.java b/server/src/main/java/org/elasticsearch/action/search/ClearReaderRequest.java
new file mode 100644
index 0000000000000..75f85c3024d13
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearReaderRequest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class ClearReaderRequest extends ActionRequest implements ToXContentObject {
+    private static final ParseField ID = new ParseField("id");
+
+    private final String id;
+
+    public ClearReaderRequest(StreamInput in) throws IOException {
+        super(in);
+        this.id = in.readString();
+    }
+
+    public ClearReaderRequest(String id) {
+        this.id = id;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        if (Strings.isEmpty(id)) {
+            throw new IllegalArgumentException("reader id must be specified");
+        }
+        return null;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(id);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ID.getPreferredName(), id);
+        builder.endObject();
+        return builder;
+    }
+
+    public static ClearReaderRequest fromXContent(XContentParser parser) throws IOException {
+        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
+            throw new IllegalArgumentException("Malformed content, must start with an object");
+        } else {
+            XContentParser.Token token;
+            String id = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME && parser.currentName().equals(ID.getPreferredName())) {
+                    token = parser.nextToken();
+                    if (token.isValue() == false) {
+                        throw new IllegalArgumentException("the request must contain only [" + ID.getPreferredName() + " field");
+                    }
+                    id = parser.text();
+                } else {
+                    throw new IllegalArgumentException("Unknown parameter [" + parser.currentName() +
+                        "] in request body or parameter is of the wrong type[" + token + "] ");
+                }
+            }
+            if (Strings.isNullOrEmpty(id)) {
+                throw new IllegalArgumentException("reader context is is not provided");
+            }
+            return new ClearReaderRequest(id);
+        }
+    }
+}
diff --git a/modules/kibana/build.gradle b/server/src/main/java/org/elasticsearch/action/search/ClearReaderResponse.java
similarity index 67%
rename from modules/kibana/build.gradle
rename to server/src/main/java/org/elasticsearch/action/search/ClearReaderResponse.java
index f9d11e5a6c58b..6efdc11bdb0d2 100644
--- a/modules/kibana/build.gradle
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearReaderResponse.java
@@ -17,15 +17,18 @@
  * under the License.
  */
 
-esplugin {
-  description 'Plugin exposing APIs for Kibana system indices'
-  classname 'org.elasticsearch.kibana.KibanaPlugin'
-}
+package org.elasticsearch.action.search;
 
-dependencies {
-  compile project(path: ':modules:reindex', configuration: 'runtime')
-}
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+
+public class ClearReaderResponse extends ClearScrollResponse {
+    public ClearReaderResponse(boolean succeeded, int numFreed) {
+        super(succeeded, numFreed);
+    }
 
-testClusters.integTest {
-  module file(project(':modules:reindex').tasks.bundlePlugin.archiveFile)
+    public ClearReaderResponse(StreamInput in) throws IOException {
+        super(in);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
index f667606917b07..cffbf7ea0a072 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
@@ -20,12 +20,14 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.internal.InternalSearchResponse;
@@ -42,13 +44,13 @@
 final class ExpandSearchPhase extends SearchPhase {
     private final SearchPhaseContext context;
     private final InternalSearchResponse searchResponse;
-    private final String scrollId;
+    private final AtomicArray<SearchPhaseResult> queryResults;
 
-    ExpandSearchPhase(SearchPhaseContext context, InternalSearchResponse searchResponse, String scrollId) {
+    ExpandSearchPhase(SearchPhaseContext context, InternalSearchResponse searchResponse, AtomicArray<SearchPhaseResult> queryResults) {
         super("expand");
         this.context = context;
         this.searchResponse = searchResponse;
-        this.scrollId = scrollId;
+        this.queryResults = queryResults;
     }
 
     /**
@@ -110,11 +112,11 @@ public void run() {
                             hit.getInnerHits().put(innerHitBuilder.getName(), innerHits);
                         }
                     }
-                    context.sendSearchResponse(searchResponse, scrollId);
+                    context.sendSearchResponse(searchResponse, queryResults);
                 }, context::onFailure)
             );
         } else {
-            context.sendSearchResponse(searchResponse, scrollId);
+            context.sendSearchResponse(searchResponse, queryResults);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 18433cd2590f8..38fd70798fcc3 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -22,9 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.search.ScoreDoc;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.RescoreDocIds;
@@ -39,7 +37,6 @@
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.Transport;
 
-import java.io.IOException;
 import java.util.List;
 import java.util.function.BiFunction;
 
@@ -51,29 +48,26 @@ final class FetchSearchPhase extends SearchPhase {
     private final AtomicArray<FetchSearchResult> fetchResults;
     private final SearchPhaseController searchPhaseController;
     private final AtomicArray<SearchPhaseResult> queryResults;
-    private final BiFunction<InternalSearchResponse, String, SearchPhase> nextPhaseFactory;
+    private final BiFunction<InternalSearchResponse, AtomicArray<SearchPhaseResult>, SearchPhase> nextPhaseFactory;
     private final SearchPhaseContext context;
     private final Logger logger;
     private final SearchPhaseResults<SearchPhaseResult> resultConsumer;
     private final SearchProgressListener progressListener;
     private final AggregatedDfs aggregatedDfs;
-    private final ClusterState clusterState;
 
     FetchSearchPhase(SearchPhaseResults<SearchPhaseResult> resultConsumer,
                      SearchPhaseController searchPhaseController,
                      AggregatedDfs aggregatedDfs,
-                     SearchPhaseContext context,
-                     ClusterState clusterState) {
-        this(resultConsumer, searchPhaseController, aggregatedDfs, context, clusterState,
-            (response, scrollId) -> new ExpandSearchPhase(context, response, scrollId));
+                     SearchPhaseContext context) {
+        this(resultConsumer, searchPhaseController, aggregatedDfs, context,
+            (response, queryPhaseResults) -> new ExpandSearchPhase(context, response, queryPhaseResults));
     }
 
     FetchSearchPhase(SearchPhaseResults<SearchPhaseResult> resultConsumer,
                      SearchPhaseController searchPhaseController,
                      AggregatedDfs aggregatedDfs,
                      SearchPhaseContext context,
-                     ClusterState clusterState,
-                     BiFunction<InternalSearchResponse, String, SearchPhase> nextPhaseFactory) {
+                     BiFunction<InternalSearchResponse, AtomicArray<SearchPhaseResult>, SearchPhase> nextPhaseFactory) {
         super("fetch");
         if (context.getNumShards() != resultConsumer.getNumShards()) {
             throw new IllegalStateException("number of shards must match the length of the query results but doesn't:"
@@ -88,7 +82,6 @@ final class FetchSearchPhase extends SearchPhase {
         this.logger = context.getLogger();
         this.resultConsumer = resultConsumer;
         this.progressListener = context.getTask().getProgressListener();
-        this.clusterState = clusterState;
     }
 
     @Override
@@ -109,21 +102,14 @@ public void onFailure(Exception e) {
         });
     }
 
-    private void innerRun() throws IOException {
+    private void innerRun() {
         final int numShards = context.getNumShards();
         final boolean isScrollSearch = context.getRequest().scroll() != null;
         final List<SearchPhaseResult> phaseResults = queryResults.asList();
-        final String scrollId;
-        if (isScrollSearch) {
-            final boolean includeContextUUID = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0);
-            scrollId = TransportSearchHelper.buildScrollId(queryResults, includeContextUUID);
-        } else {
-            scrollId = null;
-        }
         final SearchPhaseController.ReducedQueryPhase reducedQueryPhase = resultConsumer.reduce();
         final boolean queryAndFetchOptimization = queryResults.length() == 1;
         final Runnable finishPhase = ()
-            -> moveToNextPhase(searchPhaseController, scrollId, reducedQueryPhase, queryAndFetchOptimization ?
+            -> moveToNextPhase(searchPhaseController, queryResults, reducedQueryPhase, queryAndFetchOptimization ?
             queryResults : fetchResults);
         if (queryAndFetchOptimization) {
             assert phaseResults.isEmpty() || phaseResults.get(0).fetchResult() != null : "phaseResults empty [" + phaseResults.isEmpty()
@@ -222,7 +208,7 @@ public void onFailure(Exception e) {
     private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) {
         // we only release search context that we did not fetch from if we are not scrolling
         // and if it has at lease one hit that didn't make it to the global topDocs
-        if (context.getRequest().scroll() == null && queryResult.hasSearchContext()) {
+        if (context.getRequest().scroll() == null &&  context.getRequest().reader() == null && queryResult.hasSearchContext()) {
             try {
                 SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget();
                 Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
@@ -234,10 +220,11 @@ private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) {
     }
 
     private void moveToNextPhase(SearchPhaseController searchPhaseController,
-                                 String scrollId, SearchPhaseController.ReducedQueryPhase reducedQueryPhase,
+                                 AtomicArray<SearchPhaseResult> queryPhaseResults,
+                                 SearchPhaseController.ReducedQueryPhase reducedQueryPhase,
                                  AtomicArray<? extends SearchPhaseResult> fetchResultsArr) {
         final InternalSearchResponse internalResponse = searchPhaseController.merge(context.getRequest().scroll() != null,
             reducedQueryPhase, fetchResultsArr.asList(), fetchResultsArr::get);
-        context.executeNextPhase(this, nextPhaseFactory.apply(internalResponse, scrollId));
+        context.executeNextPhase(this, nextPhaseFactory.apply(internalResponse, queryPhaseResults));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java b/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
index b588827867fbb..a638e6e3cf1e0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
@@ -29,9 +29,9 @@ class ParsedScrollId {
 
     private final String type;
 
-    private final ScrollIdForNode[] context;
+    private final ReaderIdForNode[] context;
 
-    ParsedScrollId(String source, String type, ScrollIdForNode[] context) {
+    ParsedScrollId(String source, String type, ReaderIdForNode[] context) {
         this.source = source;
         this.type = type;
         this.context = context;
@@ -45,7 +45,7 @@ public String getType() {
         return type;
     }
 
-    public ScrollIdForNode[] getContext() {
+    public ReaderIdForNode[] getContext() {
         return context;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/ReaderIdForNode.java
similarity index 92%
rename from server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
rename to server/src/main/java/org/elasticsearch/action/search/ReaderIdForNode.java
index d69a10334bd78..74cec46416fe8 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ReaderIdForNode.java
@@ -22,12 +22,12 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.search.internal.SearchContextId;
 
-class ScrollIdForNode {
+final class ReaderIdForNode {
     private final String node;
     private final SearchContextId contextId;
     private final String clusterAlias;
 
-    ScrollIdForNode(@Nullable String clusterAlias, String node, SearchContextId contextId) {
+    ReaderIdForNode(@Nullable String clusterAlias, String node, SearchContextId contextId) {
         this.node = node;
         this.clusterAlias = clusterAlias;
         this.contextId = contextId;
@@ -48,7 +48,7 @@ public SearchContextId getContextId() {
 
     @Override
     public String toString() {
-        return "ScrollIdForNode{" +
+        return "ReaderIdForNode{" +
             "node='" + node + '\'' +
             ", scrollId=" + contextId +
             ", clusterAlias='" + clusterAlias + '\'' +
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index 0be8b0e799210..01a82fc0cde9d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -73,6 +73,6 @@ protected SearchPhase getNextPhase(final SearchPhaseResults<DfsSearchResult> res
         final AggregatedDfs aggregatedDfs = searchPhaseController.aggregateDfs(dfsSearchResults);
 
         return new DfsQueryPhase(dfsSearchResults, aggregatedDfs, searchPhaseController, (queryResults) ->
-            new FetchSearchPhase(queryResults, searchPhaseController, aggregatedDfs, context, clusterState()), context);
+            new FetchSearchPhase(queryResults, searchPhaseController, aggregatedDfs, context), context);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
index e22104b8f70af..601407cc1ca0f 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
@@ -21,6 +21,8 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.SearchContextId;
@@ -57,10 +59,11 @@ interface SearchPhaseContext extends Executor {
 
     /**
      * Builds and sends the final search response back to the user.
+     *
      * @param internalSearchResponse the internal search response
-     * @param scrollId an optional scroll ID if this search is a scroll search
+     * @param queryResults           the results of the query phase
      */
-    void sendSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId);
+    void sendSearchResponse(InternalSearchResponse internalSearchResponse, AtomicArray<SearchPhaseResult> queryResults);
 
     /**
      * Notifies the top-level listener of the provided exception
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index f2cf6b199dcbe..9648f74e96b27 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -19,8 +19,17 @@
 
 package org.elasticsearch.action.search;
 
-import com.carrotsearch.hppc.IntArrayList;
-import com.carrotsearch.hppc.ObjectObjectHashMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.IntFunction;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.CollectionStatistics;
@@ -58,16 +67,8 @@
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.function.IntFunction;
-import java.util.stream.Collectors;
+import com.carrotsearch.hppc.IntArrayList;
+import com.carrotsearch.hppc.ObjectObjectHashMap;
 
 public final class SearchPhaseController {
     private static final ScoreDoc[] EMPTY_DOCS = new ScoreDoc[0];
@@ -429,7 +430,7 @@ public ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResul
      * @see QuerySearchResult#consumeProfileResult()
      */
     private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResult> queryResults,
-                                                List<InternalAggregations> bufferedAggs, List<TopDocs> bufferedTopDocs,
+                                                List<Supplier<InternalAggregations>> bufferedAggs, List<TopDocs> bufferedTopDocs,
                                                 TopDocsStats topDocsStats, int numReducePhases, boolean isScrollRequest,
                                                 InternalAggregation.ReduceContextBuilder aggReduceContextBuilder,
                                                 boolean performFinalReduce) {
@@ -453,7 +454,7 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
         final boolean hasSuggest = firstResult.suggest() != null;
         final boolean hasProfileResults = firstResult.hasProfileResults();
         final boolean consumeAggs;
-        final List<InternalAggregations> aggregationsList;
+        final List<Supplier<InternalAggregations>> aggregationsList;
         if (bufferedAggs != null) {
             consumeAggs = false;
             // we already have results from intermediate reduces and just need to perform the final reduce
@@ -492,7 +493,7 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
                 }
             }
             if (consumeAggs) {
-                aggregationsList.add((InternalAggregations) result.consumeAggs());
+                aggregationsList.add(result.consumeAggs());
             }
             if (hasProfileResults) {
                 String key = result.getSearchShardTarget().toString();
@@ -508,8 +509,7 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
             reducedSuggest = new Suggest(Suggest.reduce(groupedSuggestions));
             reducedCompletionSuggestions = reducedSuggest.filter(CompletionSuggestion.class);
         }
-        final InternalAggregations aggregations = aggregationsList.isEmpty() ? null : InternalAggregations.topLevelReduce(aggregationsList,
-                    performFinalReduce ? aggReduceContextBuilder.forFinalReduction() : aggReduceContextBuilder.forPartialReduction());
+        final InternalAggregations aggregations = reduceAggs(aggReduceContextBuilder, performFinalReduce, aggregationsList);
         final SearchProfileShardResults shardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults);
         final SortedTopDocs sortedTopDocs = sortDocs(isScrollRequest, queryResults, bufferedTopDocs, topDocsStats, from, size,
             reducedCompletionSuggestions);
@@ -519,6 +519,24 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
             firstResult.sortValueFormats(), numReducePhases, size, from, false);
     }
 
+    private InternalAggregations reduceAggs(
+        InternalAggregation.ReduceContextBuilder aggReduceContextBuilder,
+        boolean performFinalReduce,
+        List<Supplier<InternalAggregations>> aggregationsList
+    ) {
+        /*
+         * Parse the aggregations, clearing the list as we go so bits backing
+         * the DelayedWriteable can be collected immediately.
+         */
+        List<InternalAggregations> toReduce = new ArrayList<>(aggregationsList.size());
+        for (int i = 0; i < aggregationsList.size(); i++) {
+            toReduce.add(aggregationsList.get(i).get());
+            aggregationsList.set(i, null);
+        }
+        return aggregationsList.isEmpty() ? null : InternalAggregations.topLevelReduce(toReduce,
+            performFinalReduce ? aggReduceContextBuilder.forFinalReduction() : aggReduceContextBuilder.forPartialReduction());
+    }
+
     /*
      * Returns the size of the requested top documents (from + size)
      */
@@ -600,7 +618,7 @@ public InternalSearchResponse buildResponse(SearchHits hits) {
      */
     static final class QueryPhaseResultConsumer extends ArraySearchPhaseResults<SearchPhaseResult> {
         private final SearchShardTarget[] processedShards;
-        private final InternalAggregations[] aggsBuffer;
+        private final Supplier<InternalAggregations>[] aggsBuffer;
         private final TopDocs[] topDocsBuffer;
         private final boolean hasAggs;
         private final boolean hasTopDocs;
@@ -642,7 +660,9 @@ private QueryPhaseResultConsumer(SearchProgressListener progressListener, Search
             this.progressListener = progressListener;
             this.processedShards = new SearchShardTarget[expectedResultSize];
             // no need to buffer anything if we have less expected results. in this case we don't consume any results ahead of time.
-            this.aggsBuffer = new InternalAggregations[hasAggs ? bufferSize : 0];
+            @SuppressWarnings("unchecked")
+            Supplier<InternalAggregations>[] aggsBuffer = new Supplier[hasAggs ? bufferSize : 0];
+            this.aggsBuffer = aggsBuffer;
             this.topDocsBuffer = new TopDocs[hasTopDocs ? bufferSize : 0];
             this.hasTopDocs = hasTopDocs;
             this.hasAggs = hasAggs;
@@ -665,10 +685,14 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) {
             if (querySearchResult.isNull() == false) {
                 if (index == bufferSize) {
                     if (hasAggs) {
-                        ReduceContext reduceContext = aggReduceContextBuilder.forPartialReduction();
-                        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(Arrays.asList(aggsBuffer), reduceContext);
-                        Arrays.fill(aggsBuffer, null);
-                        aggsBuffer[0] = reducedAggs;
+                        List<InternalAggregations> aggs = new ArrayList<>(aggsBuffer.length);
+                        for (int i = 0; i < aggsBuffer.length; i++) {
+                            aggs.add(aggsBuffer[i].get());
+                            aggsBuffer[i] = null; // null the buffer so it can be GCed now.
+                        }
+                        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(
+                                aggs, aggReduceContextBuilder.forPartialReduction());
+                        aggsBuffer[0] = () -> reducedAggs;
                     }
                     if (hasTopDocs) {
                         TopDocs reducedTopDocs = mergeTopDocs(Arrays.asList(topDocsBuffer),
@@ -681,12 +705,12 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) {
                     index = 1;
                     if (hasAggs || hasTopDocs) {
                         progressListener.notifyPartialReduce(SearchProgressListener.buildSearchShards(processedShards),
-                            topDocsStats.getTotalHits(), hasAggs ? aggsBuffer[0] : null, numReducePhases);
+                            topDocsStats.getTotalHits(), hasAggs ? aggsBuffer[0].get() : null, numReducePhases);
                     }
                 }
                 final int i = index++;
                 if (hasAggs) {
-                    aggsBuffer[i] = (InternalAggregations) querySearchResult.consumeAggs();
+                    aggsBuffer[i] = querySearchResult.consumeAggs();
                 }
                 if (hasTopDocs) {
                     final TopDocsAndMaxScore topDocs = querySearchResult.consumeTopDocs(); // can't be null
@@ -698,7 +722,7 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) {
             processedShards[querySearchResult.getShardIndex()] = querySearchResult.getSearchShardTarget();
         }
 
-        private synchronized List<InternalAggregations> getRemainingAggs() {
+        private synchronized List<Supplier<InternalAggregations>> getRemainingAggs() {
             return hasAggs ? Arrays.asList(aggsBuffer).subList(0, index) : null;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index f49243874f7da..d0677f61330ae 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -107,7 +107,7 @@ && getRequest().scroll() == null
 
     @Override
     protected SearchPhase getNextPhase(final SearchPhaseResults<SearchPhaseResult> results, final SearchPhaseContext context) {
-        return new FetchSearchPhase(results, searchPhaseController, null, context, clusterState());
+        return new FetchSearchPhase(results, searchPhaseController, null, context);
     }
 
     private ShardSearchRequest rewriteShardSearchRequest(ShardSearchRequest request) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 761a50bfa4168..16aa4c32b3ef0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -89,7 +89,7 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla
 
     private int maxConcurrentShardRequests = 0;
 
-    private int preFilterShardSize = DEFAULT_PRE_FILTER_SHARD_SIZE;
+    private Integer preFilterShardSize;
 
     private boolean ccsMinimizeRoundtrips = true;
 
@@ -201,7 +201,11 @@ public SearchRequest(StreamInput in) throws IOException {
         requestCache = in.readOptionalBoolean();
         batchedReduceSize = in.readVInt();
         maxConcurrentShardRequests = in.readVInt();
-        preFilterShardSize = in.readVInt();
+        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+            preFilterShardSize = in.readOptionalVInt();
+        } else {
+            preFilterShardSize = in.readVInt();
+        }
         allowPartialSearchResults = in.readOptionalBoolean();
         localClusterAlias = in.readOptionalString();
         if (localClusterAlias != null) {
@@ -231,7 +235,11 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalBoolean(requestCache);
         out.writeVInt(batchedReduceSize);
         out.writeVInt(maxConcurrentShardRequests);
-        out.writeVInt(preFilterShardSize);
+        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+            out.writeOptionalVInt(preFilterShardSize);
+        } else {
+            out.writeVInt(preFilterShardSize == null ? DEFAULT_BATCHED_REDUCE_SIZE : preFilterShardSize);
+        }
         out.writeOptionalBoolean(allowPartialSearchResults);
         out.writeOptionalString(localClusterAlias);
         if (localClusterAlias != null) {
@@ -269,6 +277,16 @@ public ActionRequestValidationException validate() {
                     addValidationError("[request_cache] cannot be used in a scroll context", validationException);
             }
         }
+        if (source != null) {
+            if (source.aggregations() != null) {
+                validationException = source.aggregations().validate(validationException);
+            }
+        }
+        if (reader() != null && reader().getId() != null) {
+            if (indices.length > 0) {
+                validationException = addValidationError("[indices] cannot be used with reader contexts", validationException);
+            }
+        }
         return validationException;
     }
 
@@ -418,6 +436,13 @@ public SearchSourceBuilder source() {
         return source;
     }
 
+    public SearchSourceBuilder.ReaderBuilder reader() {
+        if (source != null) {
+            return source.reader();
+        }
+        return null;
+    }
+
     /**
      * The tye of search to execute.
      */
@@ -531,8 +556,15 @@ public void setMaxConcurrentShardRequests(int maxConcurrentShardRequests) {
     /**
      * Sets a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
      * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
-     * instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard
-     * bounds and the query are disjoint. The default is {@code 128}
+     * instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard
+     * bounds and the query are disjoint.
+     *
+     * When unspecified, the pre-filter phase is executed if any of these conditions is met:
+     * <ul>
+     * <li>The request targets more than 128 shards</li>
+     * <li>The request targets one or more read-only index</li>
+     * <li>The primary sort of the query targets an indexed field</li>
+     * </ul>
      */
     public void setPreFilterShardSize(int preFilterShardSize) {
         if (preFilterShardSize < 1) {
@@ -543,11 +575,20 @@ public void setPreFilterShardSize(int preFilterShardSize) {
 
     /**
      * Returns a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
-     * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
-     * instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard
-     * bounds and the query are disjoint. The default is {@code 128}
+     * the search request expands to exceeds the threshold, or <code>null</code> if the threshold is unspecified.
+     * This filter roundtrip can limit the number of shards significantly if for
+     * instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard
+     * bounds and the query are disjoint.
+     *
+     * When unspecified, the pre-filter phase is executed if any of these conditions is met:
+     * <ul>
+     * <li>The request targets more than 128 shards</li>
+     * <li>The request targets one or more read-only index</li>
+     * <li>The primary sort of the query targets an indexed field</li>
+     * </ul>
      */
-    public int getPreFilterShardSize() {
+    @Nullable
+    public Integer getPreFilterShardSize() {
         return preFilterShardSize;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index b8b791360d30c..6a9f1e0c1e389 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -521,6 +521,17 @@ public SearchRequestBuilder setCollapse(CollapseBuilder collapse) {
         return this;
     }
 
+    /**
+     * Specifies the point in time reader contexts that this request should execute against
+     *
+     * @param readerId  the base64 string of the set of reader ids.
+     * @param keepAlive the extended keep alive for the readers used in this request
+     */
+    public SearchRequestBuilder setReader(@Nullable String readerId, @Nullable TimeValue keepAlive) {
+        sourceBuilder().reader(new SearchSourceBuilder.ReaderBuilder(readerId, keepAlive));
+        return this;
+    }
+
     @Override
     public String toString() {
         if (request.source() != null) {
@@ -558,8 +569,15 @@ public SearchRequestBuilder setMaxConcurrentShardRequests(int maxConcurrentShard
     /**
      * Sets a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
      * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
-     * instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard
-     * bounds and the query are disjoint. The default is {@code 128}
+     * instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard
+     * bounds and the query are disjoint.
+     *
+     * When unspecified, the pre-filter phase is executed if any of these conditions is met:
+     * <ul>
+     * <li>The request targets more than 128 shards</li>
+     * <li>The request targets one or more read-only index</li>
+     * <li>The primary sort of the query targets an indexed field</li>
+     * </ul>
      */
     public SearchRequestBuilder setPreFilterShardSize(int preFilterShardSize) {
         this.request.setPreFilterShardSize(preFilterShardSize);
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 80487710729c9..8db0abf3fa574 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.search;
 
 import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -61,6 +62,7 @@
 public class SearchResponse extends ActionResponse implements StatusToXContentObject {
 
     private static final ParseField SCROLL_ID = new ParseField("_scroll_id");
+    private static final ParseField READER_ID = new ParseField("reader_id");
     private static final ParseField TOOK = new ParseField("took");
     private static final ParseField TIMED_OUT = new ParseField("timed_out");
     private static final ParseField TERMINATED_EARLY = new ParseField("terminated_early");
@@ -68,6 +70,7 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
 
     private final SearchResponseSections internalResponse;
     private final String scrollId;
+    private final String readerId;
     private final int totalShards;
     private final int successfulShards;
     private final int skippedShards;
@@ -93,12 +96,18 @@ public SearchResponse(StreamInput in) throws IOException {
         scrollId = in.readOptionalString();
         tookInMillis = in.readVLong();
         skippedShards = in.readVInt();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            readerId = in.readOptionalString();
+        } else {
+            readerId = null;
+        }
     }
 
     public SearchResponse(SearchResponseSections internalResponse, String scrollId, int totalShards, int successfulShards,
-                          int skippedShards, long tookInMillis, ShardSearchFailure[] shardFailures, Clusters clusters) {
+                          int skippedShards, long tookInMillis, ShardSearchFailure[] shardFailures, Clusters clusters, String readerId) {
         this.internalResponse = internalResponse;
         this.scrollId = scrollId;
+        this.readerId = readerId;
         this.clusters = clusters;
         this.totalShards = totalShards;
         this.successfulShards = successfulShards;
@@ -106,6 +115,8 @@ public SearchResponse(SearchResponseSections internalResponse, String scrollId,
         this.tookInMillis = tookInMillis;
         this.shardFailures = shardFailures;
         assert skippedShards <= totalShards : "skipped: " + skippedShards + " total: " + totalShards;
+        assert scrollId == null || readerId == null :
+            "SearchResponse can't have both scrollId [" + scrollId + "] and readerId [" + readerId + "]";
     }
 
     @Override
@@ -207,6 +218,13 @@ public String getScrollId() {
         return scrollId;
     }
 
+    /**
+     * Returns the encoded string representing reader ids that ared used to executed the request.
+     */
+    public String getReaderId() {
+        return readerId;
+    }
+
     /**
      * If profiling was enabled, this returns an object containing the profile results from
      * each shard.  If profiling was not enabled, this will return null
@@ -239,6 +257,9 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         if (scrollId != null) {
             builder.field(SCROLL_ID.getPreferredName(), scrollId);
         }
+        if (readerId != null) {
+            builder.field(READER_ID.getPreferredName(), readerId);
+        }
         builder.field(TOOK.getPreferredName(), tookInMillis);
         builder.field(TIMED_OUT.getPreferredName(), isTimedOut());
         if (isTerminatedEarly() != null) {
@@ -275,6 +296,7 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE
         int totalShards = -1;
         int skippedShards = 0; // 0 for BWC
         String scrollId = null;
+        String readerContextId = null;
         List<ShardSearchFailure> failures = new ArrayList<>();
         Clusters clusters = Clusters.EMPTY;
         for (Token token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) {
@@ -283,6 +305,8 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE
             } else if (token.isValue()) {
                 if (SCROLL_ID.match(currentFieldName, parser.getDeprecationHandler())) {
                     scrollId = parser.text();
+                } else if (READER_ID.match(currentFieldName, parser.getDeprecationHandler())) {
+                    readerContextId = parser.text();
                 } else if (TOOK.match(currentFieldName, parser.getDeprecationHandler())) {
                     tookInMillis = parser.longValue();
                 } else if (TIMED_OUT.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -361,7 +385,7 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE
         SearchResponseSections searchResponseSections = new SearchResponseSections(hits, aggs, suggest, timedOut, terminatedEarly,
                 profile, numReducePhases);
         return new SearchResponse(searchResponseSections, scrollId, totalShards, successfulShards, skippedShards, tookInMillis,
-                failures.toArray(ShardSearchFailure.EMPTY_ARRAY), clusters);
+                failures.toArray(ShardSearchFailure.EMPTY_ARRAY), clusters, readerContextId);
     }
 
     @Override
@@ -378,6 +402,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(scrollId);
         out.writeVLong(tookInMillis);
         out.writeVInt(skippedShards);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalString(readerId);
+        }
     }
 
     @Override
@@ -486,6 +513,6 @@ static SearchResponse empty(Supplier<Long> tookInMillisSupplier, Clusters cluste
         InternalSearchResponse internalSearchResponse = new InternalSearchResponse(searchHits,
             InternalAggregations.EMPTY, null, null, false, null, 0);
         return new SearchResponse(internalSearchResponse, null, 0, 0, 0, tookInMillisSupplier.get(),
-            ShardSearchFailure.EMPTY_ARRAY, clusters);
+            ShardSearchFailure.EMPTY_ARRAY, clusters, null);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java
index 5c1cacab559fd..e28226b77ac8d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java
@@ -203,7 +203,8 @@ SearchResponse getMergedResponse(Clusters clusters) {
         InternalSearchResponse response = new InternalSearchResponse(mergedSearchHits, reducedAggs, suggest, profileShardResults,
             topDocsStats.timedOut, topDocsStats.terminatedEarly, numReducePhases);
         long tookInMillis = searchTimeProvider.buildTookInMillis();
-        return new SearchResponse(response, null, totalShards, successfulShards, skippedShards, tookInMillis, shardFailures, clusters);
+        return new SearchResponse(response, null, totalShards, successfulShards, skippedShards, tookInMillis, shardFailures,
+            clusters, null);
     }
 
     private static final Comparator<ShardSearchFailure> FAILURES_COMPARATOR = new Comparator<ShardSearchFailure>() {
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
index edc3f1b96a143..58f5a9ff0afbe 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
@@ -104,7 +104,7 @@ private long buildTookInMillis() {
     }
 
     public final void run() {
-        final ScrollIdForNode[] context = scrollId.getContext();
+        final ReaderIdForNode[] context = scrollId.getContext();
         if (context.length == 0) {
             listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", ShardSearchFailure.EMPTY_ARRAY));
         } else {
@@ -117,11 +117,11 @@ public final void run() {
      * This method collects nodes from the remote clusters asynchronously if any of the scroll IDs references a remote cluster.
      * Otherwise the action listener will be invoked immediately with a function based on the given discovery nodes.
      */
-    static void collectNodesAndRun(final Iterable<ScrollIdForNode> scrollIds, DiscoveryNodes nodes,
+    static void collectNodesAndRun(final Iterable<ReaderIdForNode> scrollIds, DiscoveryNodes nodes,
                                    SearchTransportService searchTransportService,
                                    ActionListener<BiFunction<String, String, DiscoveryNode>> listener) {
         Set<String> clusters = new HashSet<>();
-        for (ScrollIdForNode target : scrollIds) {
+        for (ReaderIdForNode target : scrollIds) {
             if (target.getClusterAlias() != null) {
                 clusters.add(target.getClusterAlias());
             }
@@ -135,10 +135,10 @@ static void collectNodesAndRun(final Iterable<ScrollIdForNode> scrollIds, Discov
         }
     }
 
-    private void run(BiFunction<String, String, DiscoveryNode> clusterNodeLookup, final ScrollIdForNode[] context) {
+    private void run(BiFunction<String, String, DiscoveryNode> clusterNodeLookup, final ReaderIdForNode[] context) {
         final CountDown counter = new CountDown(scrollId.getContext().length);
         for (int i = 0; i < context.length; i++) {
-            ScrollIdForNode target = context[i];
+            ReaderIdForNode target = context[i];
             final int shardIndex = i;
             final Transport.Connection connection;
             try {
@@ -242,7 +242,7 @@ protected final void sendResponse(SearchPhaseController.ReducedQueryPhase queryP
                 scrollId = request.scrollId();
             }
             listener.onResponse(new SearchResponse(internalResponse, scrollId, this.scrollId.getContext().length, successfulOps.get(),
-                0, buildTookInMillis(), buildShardFailures(), SearchResponse.Clusters.EMPTY));
+                0, buildTookInMillis(), buildShardFailures(), SearchResponse.Clusters.EMPTY, null));
         } catch (Exception e) {
             listener.onFailure(new ReduceSearchPhaseException("fetch", "inner finish failed", e, buildShardFailures()));
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
index ec27af0970545..974b062e0cc1d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
@@ -24,8 +24,10 @@
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.SearchContextId;
 
 import java.util.List;
 import java.util.Objects;
@@ -42,6 +44,9 @@ public final class SearchShardIterator extends PlainShardIterator {
     private final String clusterAlias;
     private boolean skip = false;
 
+    private final SearchContextId readerId;
+    private final TimeValue readerKeepAlive;
+
     /**
      * Creates a {@link PlainShardIterator} instance that iterates over a subset of the given shards
      * this the a given <code>shardId</code>.
@@ -52,9 +57,17 @@ public final class SearchShardIterator extends PlainShardIterator {
      * @param originalIndices the indices that the search request originally related to (before any rewriting happened)
      */
     public SearchShardIterator(@Nullable String clusterAlias, ShardId shardId, List<ShardRouting> shards, OriginalIndices originalIndices) {
+        this(clusterAlias, shardId, shards, originalIndices, null, null);
+    }
+
+    public SearchShardIterator(@Nullable String clusterAlias, ShardId shardId,
+                               List<ShardRouting> shards, OriginalIndices originalIndices,
+                               SearchContextId readerId, TimeValue readerKeepAlive) {
         super(shardId, shards);
         this.originalIndices = originalIndices;
         this.clusterAlias = clusterAlias;
+        this.readerId = readerId;
+        this.readerKeepAlive = readerKeepAlive;
     }
 
     /**
@@ -80,6 +93,21 @@ SearchShardTarget newSearchShardTarget(String nodeId) {
         return new SearchShardTarget(nodeId, shardId(), clusterAlias, originalIndices);
     }
 
+    /**
+     * Returns a non-null value if this request should execute using a specific point-in-time reader;
+     * otherwise, using the most up to date point-in-time reader.
+     */
+    SearchContextId getReaderId() {
+        return readerId;
+    }
+
+    /**
+     * Returns a non-null to specify the time to live of the point-in-time reader that is used to execute this request.
+     */
+    TimeValue getReaderKeepAlive() {
+        return readerKeepAlive;
+    }
+
     /**
      * Reset the iterator and mark it as skippable
      * @see #skip()
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportClearReaderAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportClearReaderAction.java
new file mode 100644
index 0000000000000..eb0e586a1c774
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportClearReaderAction.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+
+public class TransportClearReaderAction extends HandledTransportAction<ClearReaderRequest, ClearReaderResponse> {
+
+    private final ClusterService clusterService;
+    private final SearchTransportService searchTransportService;
+
+    @Inject
+    public TransportClearReaderAction(TransportService transportService, ClusterService clusterService,
+                                      ActionFilters actionFilters, SearchTransportService searchTransportService) {
+        super(ClearReaderAction.NAME, transportService, actionFilters, ClearReaderRequest::new);
+        this.clusterService = clusterService;
+        this.searchTransportService = searchTransportService;
+    }
+
+    @Override
+    protected void doExecute(Task task, ClearReaderRequest request, ActionListener<ClearReaderResponse> listener) {
+        Runnable runnable = new ClearContextController(
+            request, listener, clusterService.state().nodes(), logger, searchTransportService);
+        runnable.run();
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
index f61d268e551b4..1a5580977e488 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
@@ -42,7 +42,8 @@ public TransportClearScrollAction(TransportService transportService, ClusterServ
 
     @Override
     protected void doExecute(Task task, ClearScrollRequest request, final ActionListener<ClearScrollResponse> listener) {
-        Runnable runnable = new ClearScrollController(request, listener, clusterService.state().nodes(), logger, searchTransportService);
+        Runnable runnable = new ClearContextController(
+            request, listener, clusterService.state().nodes(), logger, searchTransportService);
         runnable.run();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 61bbaa0a5097c..20f249160a72f 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -29,18 +29,21 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.Rewriteable;
@@ -55,7 +58,6 @@
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.profile.SearchProfileShardResults;
-import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteClusterAware;
@@ -80,9 +82,12 @@
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH;
 import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH;
+import static org.elasticsearch.search.sort.FieldSortBuilder.hasPrimaryFieldSort;
 
 public class TransportSearchAction extends HandledTransportAction<SearchRequest, SearchResponse> {
 
@@ -204,18 +209,26 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<
                 // situations when source is rewritten to null due to a bug
                 searchRequest.source(source);
             }
-            final ClusterState clusterState = clusterService.state();
-            final Map<String, OriginalIndices> remoteClusterIndices = remoteClusterService.groupIndices(searchRequest.indicesOptions(),
-                searchRequest.indices());
+            final Map<ShardId, ReaderIdForNode> readerContexts;
+            final Map<String, OriginalIndices> remoteClusterIndices;
+            if (searchRequest.reader() != null && searchRequest.reader().getId() != null) {
+                readerContexts = TransportSearchHelper.decodeReaderIds(searchRequest.reader().getId());
+                remoteClusterIndices = indicesFromReaderContexts(readerContexts, searchRequest.indicesOptions());
+            } else {
+                readerContexts = null;
+                remoteClusterIndices = remoteClusterService.groupIndices(searchRequest.indicesOptions(),
+                    searchRequest.indices());
+            }
             OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+            final ClusterState clusterState = clusterService.state();
             if (remoteClusterIndices.isEmpty()) {
-                executeLocalSearch(task, timeProvider, searchRequest, localIndices, clusterState, listener);
+                executeLocalSearch(task, timeProvider, searchRequest, localIndices, clusterState, listener, readerContexts);
             } else {
                 if (shouldMinimizeRoundtrips(searchRequest)) {
                     ccsRemoteReduce(searchRequest, localIndices, remoteClusterIndices, timeProvider,
                             searchService.aggReduceContextBuilder(searchRequest),
                             remoteClusterService, threadPool, listener,
-                            (r, l) -> executeLocalSearch(task, timeProvider, r, localIndices, clusterState, l));
+                            (r, l) -> executeLocalSearch(task, timeProvider, r, localIndices, clusterState, l, readerContexts));
                 } else {
                     AtomicInteger skippedClusters = new AtomicInteger(0);
                     collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(),
@@ -231,7 +244,7 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<
                                 int successfulClusters = searchShardsResponses.size() + localClusters;
                                 executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices,
                                     remoteShardIterators, clusterNodeLookup, clusterState, remoteAliasFilters, listener,
-                                    new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters.get()));
+                                    new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters.get()), readerContexts);
                             },
                             listener::onFailure));
                 }
@@ -252,6 +265,10 @@ static boolean shouldMinimizeRoundtrips(SearchRequest searchRequest) {
         if (searchRequest.scroll() != null) {
             return false;
         }
+        if (searchRequest.reader() != null) {
+            // TODO: support reader contexts
+            return false;
+        }
         if (searchRequest.searchType() == DFS_QUERY_THEN_FETCH) {
             return false;
         }
@@ -286,7 +303,8 @@ public void onResponse(SearchResponse searchResponse) {
                         searchResponse.isTimedOut(), searchResponse.isTerminatedEarly(), searchResponse.getNumReducePhases());
                     listener.onResponse(new SearchResponse(internalSearchResponse, searchResponse.getScrollId(),
                         searchResponse.getTotalShards(), searchResponse.getSuccessfulShards(), searchResponse.getSkippedShards(),
-                        timeProvider.buildTookInMillis(), searchResponse.getShardFailures(), new SearchResponse.Clusters(1, 1, 0)));
+                        timeProvider.buildTookInMillis(), searchResponse.getShardFailures(), new SearchResponse.Clusters(1, 1, 0),
+                        searchResponse.getReaderId()));
                 }
 
                 @Override
@@ -398,9 +416,10 @@ SearchResponse createFinalResponse() {
     }
 
     private void executeLocalSearch(Task task, SearchTimeProvider timeProvider, SearchRequest searchRequest, OriginalIndices localIndices,
-                                    ClusterState clusterState, ActionListener<SearchResponse> listener) {
+                                    ClusterState clusterState, ActionListener<SearchResponse> listener,
+                                    Map<ShardId, ReaderIdForNode> readerContexts) {
         executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, Collections.emptyList(),
-            (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener, SearchResponse.Clusters.EMPTY);
+            (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener, SearchResponse.Clusters.EMPTY, readerContexts);
     }
 
     static BiFunction<String, String, DiscoveryNode> processRemoteShards(Map<String, ClusterSearchShardsResponse> searchShardsResponses,
@@ -464,26 +483,54 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
                                OriginalIndices localIndices, List<SearchShardIterator> remoteShardIterators,
                                BiFunction<String, String, DiscoveryNode> remoteConnections, ClusterState clusterState,
                                Map<String, AliasFilter> remoteAliasMap, ActionListener<SearchResponse> listener,
-                               SearchResponse.Clusters clusters) {
+                               SearchResponse.Clusters clusters, Map<ShardId, ReaderIdForNode> readerContexts) {
 
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
+        if (searchRequest.allowPartialSearchResults() == null) {
+            // No user preference defined in search request - apply cluster service default
+            searchRequest.allowPartialSearchResults(searchService.defaultAllowPartialSearchResults());
+        }
+
         // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
         // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
         // of just for the _search api
-        final Index[] indices = resolveLocalIndices(localIndices, searchRequest.indicesOptions(), clusterState, timeProvider);
-        Map<String, AliasFilter> aliasFilter = buildPerIndexAliasFilter(searchRequest, clusterState, indices, remoteAliasMap);
-        Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(),
-            searchRequest.indices());
-        routingMap = routingMap == null ? Collections.emptyMap() : Collections.unmodifiableMap(routingMap);
-        String[] concreteIndices = new String[indices.length];
-        for (int i = 0; i < indices.length; i++) {
-            concreteIndices[i] = indices[i].getName();
-        }
-        Map<String, Long> nodeSearchCounts = searchTransportService.getPendingSearchRequests();
-        GroupShardsIterator<ShardIterator> localShardsIterator = clusterService.operationRouting().searchShards(clusterState,
+        final TimeValue keepAlive = searchRequest.reader() != null ? searchRequest.reader().getKeepAlive() : null;
+        final List<SearchShardIterator> localShardIterators;
+        final Map<String, AliasFilter> aliasFilter;
+        final Map<String, Set<String>> indexRoutings;
+
+        boolean preFilterSearchShards;
+        if (readerContexts != null) {
+            aliasFilter = new HashMap<>();
+            for (ShardId shardId : readerContexts.keySet()) {
+                aliasFilter.put(shardId.getIndex().getUUID(), AliasFilter.EMPTY);
+            }
+            indexRoutings = Map.of();
+            localShardIterators = searchShardsFromReaderContexts(
+                clusterState, localIndices, searchRequest.getLocalClusterAlias(), readerContexts, keepAlive);
+            preFilterSearchShards = false;
+        } else {
+            final Index[] indices = resolveLocalIndices(localIndices, searchRequest.indicesOptions(), clusterState, timeProvider);
+            Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(),
+                searchRequest.indices());
+            routingMap = routingMap == null ? Collections.emptyMap() : Collections.unmodifiableMap(routingMap);
+            final String[] concreteIndices = new String[indices.length];
+            for (int i = 0; i < indices.length; i++) {
+                concreteIndices[i] = indices[i].getName();
+            }
+            Map<String, Long> nodeSearchCounts = searchTransportService.getPendingSearchRequests();
+            GroupShardsIterator<ShardIterator> localShardRoutings = clusterService.operationRouting().searchShards(clusterState,
                 concreteIndices, routingMap, searchRequest.preference(), searchService.getResponseCollectorService(), nodeSearchCounts);
-        GroupShardsIterator<SearchShardIterator> shardIterators = mergeShardsIterators(localShardsIterator, localIndices,
-            searchRequest.getLocalClusterAlias(), remoteShardIterators);
+            localShardIterators = StreamSupport.stream(localShardRoutings.spliterator(), false)
+                .map(it -> new SearchShardIterator(
+                    searchRequest.getLocalClusterAlias(), it.shardId(), it.getShardRoutings(), localIndices, null, keepAlive))
+                .collect(Collectors.toList());
+            aliasFilter = buildPerIndexAliasFilter(searchRequest, clusterState, indices, remoteAliasMap);
+            indexRoutings = routingMap;
+            preFilterSearchShards = shouldPreFilterSearchShards(clusterState, searchRequest, concreteIndices,
+                localShardIterators.size() + remoteShardIterators.size());
+        }
+        final GroupShardsIterator<SearchShardIterator> shardIterators = mergeShardsIterators(localShardIterators, remoteShardIterators);
 
         failIfOverShardCountLimit(clusterService, shardIterators.size());
 
@@ -495,7 +542,7 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
             searchRequest.searchType(QUERY_THEN_FETCH);
         }
         if (searchRequest.allowPartialSearchResults() == null) {
-           // No user preference defined in search request - apply cluster service default
+            // No user preference defined in search request - apply cluster service default
             searchRequest.allowPartialSearchResults(searchService.defaultAllowPartialSearchResults());
         }
         if (searchRequest.isSuggestOnly()) {
@@ -508,13 +555,12 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
                     break;
             }
         }
-
         final DiscoveryNodes nodes = clusterState.nodes();
         BiFunction<String, String, Transport.Connection> connectionLookup = buildConnectionLookup(searchRequest.getLocalClusterAlias(),
             nodes::get, remoteConnections, searchTransportService::getConnection);
-        boolean preFilterSearchShards = shouldPreFilterSearchShards(searchRequest, shardIterators);
         searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState,
-            Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, routingMap, listener, preFilterSearchShards, clusters).start();
+            Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, indexRoutings, listener, preFilterSearchShards, clusters)
+            .start();
     }
 
     static BiFunction<String, String, Transport.Connection> buildConnectionLookup(String requestClusterAlias,
@@ -539,22 +585,37 @@ static BiFunction<String, String, Transport.Connection> buildConnectionLookup(St
         };
     }
 
-    private static boolean shouldPreFilterSearchShards(SearchRequest searchRequest,
-                                                       GroupShardsIterator<SearchShardIterator> shardIterators) {
+    static boolean shouldPreFilterSearchShards(ClusterState clusterState,
+                                               SearchRequest searchRequest,
+                                               String[] indices,
+                                               int numShards) {
         SearchSourceBuilder source = searchRequest.source();
+        Integer preFilterShardSize = searchRequest.getPreFilterShardSize();
+        if (preFilterShardSize == null
+                && (hasReadOnlyIndices(indices, clusterState) || hasPrimaryFieldSort(source))) {
+            preFilterShardSize = 1;
+        } else if (preFilterShardSize == null) {
+            preFilterShardSize = SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE;
+        }
         return searchRequest.searchType() == QUERY_THEN_FETCH // we can't do this for DFS it needs to fan out to all shards all the time
-                    && (SearchService.canRewriteToMatchNone(source) || FieldSortBuilder.hasPrimaryFieldSort(source))
-                    && searchRequest.getPreFilterShardSize() < shardIterators.size();
+                    && (SearchService.canRewriteToMatchNone(source) || hasPrimaryFieldSort(source))
+                    && preFilterShardSize < numShards;
     }
 
-    static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(GroupShardsIterator<ShardIterator> localShardsIterator,
-                                                             OriginalIndices localIndices,
-                                                             @Nullable String localClusterAlias,
-                                                             List<SearchShardIterator> remoteShardIterators) {
-        List<SearchShardIterator> shards = new ArrayList<>(remoteShardIterators);
-        for (ShardIterator shardIterator : localShardsIterator) {
-            shards.add(new SearchShardIterator(localClusterAlias, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices));
+    private static boolean hasReadOnlyIndices(String[] indices, ClusterState clusterState) {
+        for (String index : indices) {
+            ClusterBlockException writeBlock = clusterState.blocks().indexBlockedException(ClusterBlockLevel.WRITE, index);
+            if (writeBlock != null) {
+                return true;
+            }
         }
+        return false;
+    }
+
+    static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(List<SearchShardIterator> localShardIterators,
+                                                                         List<SearchShardIterator> remoteShardIterators) {
+        List<SearchShardIterator> shards = new ArrayList<>(remoteShardIterators);
+        shards.addAll(localShardIterators);
         return GroupShardsIterator.sortAndCreate(shards);
     }
 
@@ -694,4 +755,39 @@ private void maybeFinish() {
     private static RemoteTransportException wrapRemoteClusterFailure(String clusterAlias, Exception e) {
         return new RemoteTransportException("error while communicating with remote cluster [" + clusterAlias + "]", e);
     }
+
+    static Map<String, OriginalIndices> indicesFromReaderContexts(Map<ShardId, ReaderIdForNode> readerContexts,
+                                                                  IndicesOptions indicesOptions) {
+        final Map<String, List<String>> indices = new HashMap<>();
+        for (Map.Entry<ShardId, ReaderIdForNode> entry : readerContexts.entrySet()) {
+            String clusterAlias = entry.getValue().getClusterAlias() == null ?
+                RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY : entry.getValue().getClusterAlias();
+            indices.computeIfAbsent(clusterAlias, k -> new ArrayList<>()).add(entry.getKey().getIndexName());
+        }
+        return indices.entrySet().stream()
+            .collect(Collectors.toMap(Map.Entry::getKey, e -> new OriginalIndices(e.getValue().toArray(String[]::new), indicesOptions)));
+    }
+
+    static List<SearchShardIterator> searchShardsFromReaderContexts(ClusterState clusterState,
+                                                                    OriginalIndices originalIndices,
+                                                                    String localClusterAlias,
+                                                                    Map<ShardId, ReaderIdForNode> readerContexts,
+                                                                    TimeValue keepAlive) {
+        final List<SearchShardIterator> iterators = new ArrayList<>(readerContexts.size());
+        for (Map.Entry<ShardId, ReaderIdForNode> entry : readerContexts.entrySet()) {
+            final ShardId shardId = entry.getKey();
+            final ShardIterator shards = OperationRouting.getShards(clusterState, shardId);
+            final List<ShardRouting> matchingNodeFirstRoutings = new ArrayList<>();
+            for (ShardRouting shard : shards) {
+                if (shard.currentNodeId().equals(entry.getValue().getNode())) {
+                    matchingNodeFirstRoutings.add(0, shard);
+                } else {
+                    matchingNodeFirstRoutings.add(shard);
+                }
+            }
+            iterators.add(new SearchShardIterator(localClusterAlias, shardId, matchingNodeFirstRoutings, originalIndices,
+                entry.getValue().getContextId(), keepAlive));
+        }
+        return iterators;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
index f3755180b1e62..ab1051873a4f8 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
@@ -21,7 +21,10 @@
 
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.RAMOutputStream;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
@@ -29,7 +32,10 @@
 import org.elasticsearch.transport.RemoteClusterAware;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Base64;
+import java.util.HashMap;
+import java.util.Map;
 
 final class TransportSearchHelper {
 
@@ -39,8 +45,8 @@ static InternalScrollSearchRequest internalScrollSearchRequest(SearchContextId i
         return new InternalScrollSearchRequest(request, id);
     }
 
-    static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults,
-                                boolean includeContextUUID) throws IOException {
+    static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults, Version version) {
+        boolean includeContextUUID = version.onOrAfter(Version.V_7_7_0);
         try (RAMOutputStream out = new RAMOutputStream()) {
             if (includeContextUUID) {
                 out.writeString(INCLUDE_CONTEXT_UUID);
@@ -63,6 +69,8 @@ static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhase
             byte[] bytes = new byte[(int) out.getFilePointer()];
             out.writeTo(bytes, 0);
             return Base64.getUrlEncoder().encodeToString(bytes);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
         }
     }
 
@@ -80,7 +88,7 @@ static ParsedScrollId parseScrollId(String scrollId) {
                 includeContextUUID = false;
                 type = firstChunk;
             }
-            ScrollIdForNode[] context = new ScrollIdForNode[in.readVInt()];
+            ReaderIdForNode[] context = new ReaderIdForNode[in.readVInt()];
             for (int i = 0; i < context.length; ++i) {
                 final String contextUUID = includeContextUUID ? in.readString() : "";
                 long id = in.readLong();
@@ -93,7 +101,7 @@ static ParsedScrollId parseScrollId(String scrollId) {
                     clusterAlias = target.substring(0, index);
                     target = target.substring(index+1);
                 }
-                context[i] = new ScrollIdForNode(clusterAlias, target, new SearchContextId(contextUUID, id));
+                context[i] = new ReaderIdForNode(clusterAlias, target, new SearchContextId(contextUUID, id));
             }
             if (in.getPosition() != bytes.length) {
                 throw new IllegalArgumentException("Not all bytes were read");
@@ -104,6 +112,65 @@ static ParsedScrollId parseScrollId(String scrollId) {
         }
     }
 
+    static String encodeReaderIds(AtomicArray<? extends SearchPhaseResult> searchPhaseResults, Version version) {
+        try (RAMOutputStream out = new RAMOutputStream()) {
+            out.writeVInt(version.id);
+            out.writeVInt(searchPhaseResults.asList().size());
+            for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) {
+                SearchShardTarget target = searchPhaseResult.getSearchShardTarget();
+                ShardId shardId = target.getShardId();
+                // shardId
+                out.writeString(shardId.getIndex().getName());
+                out.writeString(shardId.getIndex().getUUID());
+                out.writeVInt(shardId.getId());
+                // nodeId
+                if (target.getClusterAlias() != null) {
+                    out.writeByte((byte) 1);
+                    out.writeString(target.getClusterAlias());
+                } else {
+                    out.writeByte((byte) 0);
+                }
+                out.writeString(target.getNodeId());
+                // readerId
+                out.writeString(searchPhaseResult.getContextId().getReaderId());
+                out.writeLong(searchPhaseResult.getContextId().getId());
+            }
+            byte[] bytes = new byte[(int) out.getFilePointer()];
+            out.writeTo(bytes, 0);
+            return Base64.getUrlEncoder().encodeToString(bytes);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    static Map<ShardId, ReaderIdForNode> decodeReaderIds(String readerId) {
+        try {
+            final byte[] bytes = Base64.getUrlDecoder().decode(readerId);
+            final ByteArrayDataInput in = new ByteArrayDataInput(bytes);
+            Version.fromId(in.readVInt());
+            final int numOfContexts = in.readVInt();
+            final Map<ShardId, ReaderIdForNode> contexts = new HashMap<>();
+            for (int i = 0; i < numOfContexts; i++) {
+                final ShardId shardId = new ShardId(new Index(in.readString(), in.readString()), in.readVInt());
+                final String clusterAlias;
+                if (in.readByte() == 1) {
+                    clusterAlias = in.readString();
+                } else {
+                    clusterAlias = null;
+                }
+                final String nodeId = in.readString();
+                final SearchContextId contextId = new SearchContextId(in.readString(), in.readLong());
+                contexts.put(shardId, new ReaderIdForNode(clusterAlias, nodeId, contextId));
+            }
+            if (in.getPosition() != bytes.length) {
+                throw new IllegalArgumentException("Not all bytes were read");
+            }
+            return contexts;
+        } catch (Exception e) {
+            throw new IllegalArgumentException("Can't parse reader context ids", e);
+        }
+    }
+
     private TransportSearchHelper() {
 
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index f52b32f3ccf67..6dfe07e5c20aa 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata;
+import org.elasticsearch.cluster.metadata.DataStreamMetadata;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateV2Metadata;
@@ -133,6 +134,7 @@ public static List<Entry> getNamedWriteables() {
             ComponentTemplateMetadata::readDiffFrom);
         registerMetaDataCustom(entries, IndexTemplateV2Metadata.TYPE, IndexTemplateV2Metadata::new,
             IndexTemplateV2Metadata::readDiffFrom);
+        registerMetaDataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);
         // Task Status (not Diffable)
         entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
         return entries;
@@ -155,6 +157,8 @@ public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
             ComponentTemplateMetadata::fromXContent));
         entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(IndexTemplateV2Metadata.TYPE),
             IndexTemplateV2Metadata::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(DataStreamMetadata.TYPE),
+            DataStreamMetadata::fromXContent));
         return entries;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
index f7ea8a68283bb..ff95d739bfa32 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.cluster;
 
-import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.Version;
@@ -29,8 +28,6 @@
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfigExclusion;
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -43,7 +40,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -55,13 +51,11 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.discovery.Discovery;
 
 import java.io.IOException;
 import java.util.EnumSet;
 import java.util.HashMap;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -419,79 +413,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         // meta data
         if (metrics.contains(Metric.METADATA)) {
-            builder.startObject("metadata");
-            builder.field("cluster_uuid", metaData().clusterUUID());
-
-            builder.startObject("cluster_coordination");
-            coordinationMetaData().toXContent(builder, params);
-            builder.endObject();
-
-            builder.startObject("templates");
-            for (ObjectCursor<IndexTemplateMetaData> cursor : metaData().templates().values()) {
-                IndexTemplateMetaData templateMetaData = cursor.value;
-                IndexTemplateMetaData.Builder.toXContentWithTypes(templateMetaData, builder, params);
-            }
-            builder.endObject();
-
-            builder.startObject("indices");
-            for (IndexMetaData indexMetaData : metaData()) {
-                builder.startObject(indexMetaData.getIndex().getName());
-
-                builder.field("state", indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));
-
-                builder.startObject("settings");
-                Settings settings = indexMetaData.getSettings();
-                settings.toXContent(builder, params);
-                builder.endObject();
-
-                builder.startObject("mappings");
-                MappingMetaData mmd = indexMetaData.mapping();
-                if (mmd != null) {
-                    Map<String, Object> mapping = XContentHelper
-                            .convertToMap(new BytesArray(mmd.source().uncompressed()), false).v2();
-                    if (mapping.size() == 1 && mapping.containsKey(mmd.type())) {
-                        // the type name is the root value, reduce it
-                        mapping = (Map<String, Object>) mapping.get(mmd.type());
-                    }
-                    builder.field(mmd.type());
-                    builder.map(mapping);
-                }
-                builder.endObject();
-
-                builder.startArray("aliases");
-                for (ObjectCursor<String> cursor : indexMetaData.getAliases().keys()) {
-                    builder.value(cursor.value);
-                }
-                builder.endArray();
-
-                builder.startObject(IndexMetaData.KEY_PRIMARY_TERMS);
-                for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) {
-                    builder.field(Integer.toString(shard), indexMetaData.primaryTerm(shard));
-                }
-                builder.endObject();
-
-                builder.startObject(IndexMetaData.KEY_IN_SYNC_ALLOCATIONS);
-                for (IntObjectCursor<Set<String>> cursor : indexMetaData.getInSyncAllocationIds()) {
-                    builder.startArray(String.valueOf(cursor.key));
-                    for (String allocationId : cursor.value) {
-                        builder.value(allocationId);
-                    }
-                    builder.endArray();
-                }
-                builder.endObject();
-
-                // index metadata
-                builder.endObject();
-            }
-            builder.endObject();
-
-            for (ObjectObjectCursor<String, MetaData.Custom> cursor : metaData.customs()) {
-                builder.startObject(cursor.key);
-                cursor.value.toXContent(builder, params);
-                builder.endObject();
-            }
-
-            builder.endObject();
+            metaData.toXContent(builder, params);
         }
 
         // routing table
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
index 754dacab0c003..2c02a07ecc158 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
@@ -24,31 +24,36 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.rollover.Condition;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
-import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.NodeMetaData;
 import org.elasticsearch.gateway.PersistedClusterStateService;
-import org.elasticsearch.indices.IndicesModule;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Map;
 import java.util.Objects;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
     private static final Logger logger = LogManager.getLogger(ElasticsearchNodeCommand.class);
@@ -65,10 +70,34 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
     protected static final String CS_MISSING_MSG =
         "cluster state is empty, cluster has never been bootstrapped?";
 
-    protected static final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(
-        Stream.of(ClusterModule.getNamedXWriteables().stream(), IndicesModule.getNamedXContents().stream())
-            .flatMap(Function.identity())
-            .collect(Collectors.toList()));
+    // fake the registry here, as command-line tools are not loading plugins, and ensure that it preserves the parsed XContent
+    public static final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()) {
+
+        @SuppressWarnings("unchecked")
+        @Override
+        public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentParser parser, C context) throws IOException {
+            // Currently, two unknown top-level objects are present
+            if (MetaData.Custom.class.isAssignableFrom(categoryClass)) {
+                return (T) new UnknownMetaDataCustom(name, parser.mapOrdered());
+            }
+            if (Condition.class.isAssignableFrom(categoryClass)) {
+                // The parsing for conditions is a bit weird as these represent JSON primitives (strings or numbers)
+                // TODO: Make Condition non-pluggable
+                assert parser.currentToken() == XContentParser.Token.FIELD_NAME : parser.currentToken();
+                if (parser.currentToken() != XContentParser.Token.FIELD_NAME) {
+                    throw new UnsupportedOperationException("Unexpected token for Condition: " + parser.currentToken());
+                }
+                parser.nextToken();
+                assert parser.currentToken().isValue() : parser.currentToken();
+                if (parser.currentToken().isValue() == false) {
+                    throw new UnsupportedOperationException("Unexpected token for Condition: " + parser.currentToken());
+                }
+                return (T) new UnknownCondition(name, parser.objectText());
+            }
+            assert false : "Unexpected category class " + categoryClass + " for name " + name;
+            throw new UnsupportedOperationException("Unexpected category class " + categoryClass + " for name " + name);
+        }
+    };
 
     public ElasticsearchNodeCommand(String description) {
         super(description);
@@ -82,7 +111,7 @@ public static PersistedClusterStateService createPersistedClusterStateService(Se
 
         String nodeId = nodeMetaData.nodeId();
         return new PersistedClusterStateService(dataPaths, nodeId, namedXContentRegistry, BigArrays.NON_RECYCLING_INSTANCE,
-            new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L, true);
+            new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L);
     }
 
     public static ClusterState clusterState(Environment environment, PersistedClusterStateService.OnDiskState onDiskState) {
@@ -168,4 +197,78 @@ private static NodeEnvironment.NodePath createNodePath(Path path) {
     OptionParser getParser() {
         return parser;
     }
+
+    public static class UnknownMetaDataCustom implements MetaData.Custom {
+
+        private final String name;
+        private final Map<String, Object> contents;
+
+        public UnknownMetaDataCustom(String name, Map<String, Object> contents) {
+            this.name = name;
+            this.contents = contents;
+        }
+
+        @Override
+        public EnumSet<MetaData.XContentContext> context() {
+            return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.GATEWAY);
+        }
+
+        @Override
+        public Diff<MetaData.Custom> diff(MetaData.Custom previousState) {
+            assert false;
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getWriteableName() {
+            return name;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            assert false;
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            assert false;
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.mapContents(contents);
+        }
+    }
+
+    public static class UnknownCondition extends Condition<Object> {
+
+        public UnknownCondition(String name, Object value) {
+            super(name);
+            this.value = value;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return name;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            assert false;
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.field(name, value);
+        }
+
+        @Override
+        public Result evaluate(Stats stats) {
+            assert false;
+            throw new UnsupportedOperationException();
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
index 1a6db35a8091e..f5c325d3915d4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
@@ -321,8 +321,8 @@ private void sendClusterStateDiff(ClusterState clusterState,
 
     public static BytesReference serializeFullClusterState(ClusterState clusterState, Version nodeVersion) throws IOException {
         final BytesStreamOutput bStream = new BytesStreamOutput();
-        bStream.setVersion(nodeVersion);
         try (StreamOutput stream = CompressorFactory.COMPRESSOR.streamOutput(bStream)) {
+            stream.setVersion(nodeVersion);
             stream.writeBoolean(true);
             clusterState.writeTo(stream);
         }
@@ -331,8 +331,8 @@ public static BytesReference serializeFullClusterState(ClusterState clusterState
 
     public static BytesReference serializeDiffClusterState(Diff diff, Version nodeVersion) throws IOException {
         final BytesStreamOutput bStream = new BytesStreamOutput();
-        bStream.setVersion(nodeVersion);
         try (StreamOutput stream = CompressorFactory.COMPRESSOR.streamOutput(bStream)) {
+            stream.setVersion(nodeVersion);
             stream.writeBoolean(false);
             diff.writeTo(stream);
         }
@@ -342,12 +342,12 @@ public static BytesReference serializeDiffClusterState(Diff diff, Version nodeVe
     private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportRequest request) throws IOException {
         final Compressor compressor = CompressorFactory.compressor(request.bytes());
         StreamInput in = request.bytes().streamInput();
-        in.setVersion(request.version());
         try {
             if (compressor != null) {
                 in = compressor.streamInput(in);
             }
             in = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry);
+            in.setVersion(request.version());
             // If true we received full cluster state - otherwise diffs
             if (in.readBoolean()) {
                 final ClusterState incomingState;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
index 99ba5a703f6a2..f248f6884ea01 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
@@ -387,6 +387,8 @@ public static AliasMetaData fromXContent(XContentParser parser) throws IOExcepti
                         builder.indexRouting(parser.text());
                     } else if ("search_routing".equals(currentFieldName) || "searchRouting".equals(currentFieldName)) {
                         builder.searchRouting(parser.text());
+                    } else if ("filter".equals(currentFieldName)) {
+                        builder.filter(new CompressedXContent(parser.binaryValue()));
                     }
                 } else if (token == XContentParser.Token.START_ARRAY) {
                     parser.skipChildren();
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
index f6bba191a173b..9742c5c67901e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.Index;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,9 +37,9 @@ public final class DataStream extends AbstractDiffable<DataStream> implements To
 
     private final String name;
     private final String timeStampField;
-    private final List<String> indices;
+    private final List<Index> indices;
 
-    public DataStream(String name, String timeStampField, List<String> indices) {
+    public DataStream(String name, String timeStampField, List<Index> indices) {
         this.name = name;
         this.timeStampField = timeStampField;
         this.indices = indices;
@@ -52,12 +53,12 @@ public String getTimeStampField() {
         return timeStampField;
     }
 
-    public List<String> getIndices() {
+    public List<Index> getIndices() {
         return indices;
     }
 
     public DataStream(StreamInput in) throws IOException {
-        this(in.readString(), in.readString(), in.readStringList());
+        this(in.readString(), in.readString(), in.readList(Index::new));
     }
 
     public static Diff<DataStream> readDiffFrom(StreamInput in) throws IOException {
@@ -68,7 +69,7 @@ public static Diff<DataStream> readDiffFrom(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeString(timeStampField);
-        out.writeStringCollection(indices);
+        out.writeList(indices);
     }
 
     public static final ParseField NAME_FIELD = new ParseField("name");
@@ -77,12 +78,12 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>("data_stream",
-        args -> new DataStream((String) args[0], (String) args[1], (List<String>) args[2]));
+        args -> new DataStream((String) args[0], (String) args[1], (List<Index>) args[2]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD);
         PARSER.declareString(ConstructingObjectParser.constructorArg(), TIMESTAMP_FIELD_FIELD);
-        PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD);
+        PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Index.fromXContent(p), INDICES_FIELD);
     }
 
     public static DataStream fromXContent(XContentParser parser) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
new file mode 100644
index 0000000000000..5731c1353f989
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Custom {@link MetaData} implementation for storing a map of {@link DataStream}s and their names.
+ */
+public class DataStreamMetadata implements MetaData.Custom {
+
+    public static final String TYPE = "data_stream";
+    private static final ParseField DATA_STREAM = new ParseField("data_stream");
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<DataStreamMetadata, Void> PARSER = new ConstructingObjectParser<>(TYPE, false,
+        a -> new DataStreamMetadata((Map<String, DataStream>) a[0]));
+
+    static {
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
+            Map<String, DataStream> dataStreams = new HashMap<>();
+            while (p.nextToken() != XContentParser.Token.END_OBJECT) {
+                String name = p.currentName();
+                dataStreams.put(name, DataStream.fromXContent(p));
+            }
+            return dataStreams;
+        }, DATA_STREAM);
+    }
+
+    private final Map<String, DataStream> dataStreams;
+
+    public DataStreamMetadata(Map<String, DataStream> dataStreams) {
+        this.dataStreams = dataStreams;
+    }
+
+    public DataStreamMetadata(StreamInput in) throws IOException {
+        this.dataStreams = in.readMap(StreamInput::readString, DataStream::new);
+    }
+
+    public Map<String, DataStream> dataStreams() {
+        return this.dataStreams;
+    }
+
+    @Override
+    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
+        return new DataStreamMetadata.DataStreamMetadataDiff((DataStreamMetadata) before, this);
+    }
+
+    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+        return new DataStreamMetadata.DataStreamMetadataDiff(in);
+    }
+
+    @Override
+    public EnumSet<MetaData.XContentContext> context() {
+        return MetaData.ALL_CONTEXTS;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return Version.V_8_0_0;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(this.dataStreams, StreamOutput::writeString, (stream, val) -> val.writeTo(stream));
+    }
+
+    public static DataStreamMetadata fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(DATA_STREAM.getPreferredName());
+        for (Map.Entry<String, DataStream> dataStream : dataStreams.entrySet()) {
+            builder.field(dataStream.getKey(), dataStream.getValue());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(this.dataStreams);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        DataStreamMetadata other = (DataStreamMetadata) obj;
+        return Objects.equals(this.dataStreams, other.dataStreams);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    public static class Builder {
+
+        private final Map<String, DataStream> dataStreams = new HashMap<>();
+
+        public Builder putDataStream(DataStream dataStream) {
+            dataStreams.put(dataStream.getName(), dataStream);
+            return this;
+        }
+
+        public DataStreamMetadata build() {
+            return new DataStreamMetadata(dataStreams);
+        }
+    }
+
+    static class DataStreamMetadataDiff implements NamedDiff<MetaData.Custom> {
+
+        final Diff<Map<String, DataStream>> dataStreamDiff;
+
+        DataStreamMetadataDiff(DataStreamMetadata before, DataStreamMetadata after) {
+            this.dataStreamDiff = DiffableUtils.diff(before.dataStreams, after.dataStreams,
+                DiffableUtils.getStringKeySerializer());
+        }
+
+        DataStreamMetadataDiff(StreamInput in) throws IOException {
+            this.dataStreamDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
+                DataStream::new, DataStream::readDiffFrom);
+        }
+
+        @Override
+        public MetaData.Custom apply(MetaData.Custom part) {
+            return new DataStreamMetadata(dataStreamDiff.apply(((DataStreamMetadata) part).dataStreams));
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            dataStreamDiff.writeTo(out);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index 82bcf2001b352..68c6ef2da88d4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -65,6 +65,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -73,6 +74,7 @@
 import java.util.Set;
 import java.util.function.Function;
 
+import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_PARAM;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.IP_VALIDATOR;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR;
@@ -1187,6 +1189,9 @@ public IndexMetaData build() {
         }
 
         public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
+            MetaData.XContentContext context = MetaData.XContentContext.valueOf(
+                params.param(CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_API));
+
             builder.startObject(indexMetaData.getIndex().getName());
 
             builder.field(KEY_VERSION, indexMetaData.getVersion());
@@ -1194,41 +1199,76 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build
             builder.field(KEY_SETTINGS_VERSION, indexMetaData.getSettingsVersion());
             builder.field(KEY_ALIASES_VERSION, indexMetaData.getAliasesVersion());
             builder.field(KEY_ROUTING_NUM_SHARDS, indexMetaData.getRoutingNumShards());
+
             builder.field(KEY_STATE, indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));
 
             boolean binary = params.paramAsBoolean("binary", false);
 
             builder.startObject(KEY_SETTINGS);
-            indexMetaData.getSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
+            if (context != MetaData.XContentContext.API) {
+                indexMetaData.getSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
+            } else {
+                indexMetaData.getSettings().toXContent(builder, params);
+            }
             builder.endObject();
 
-            builder.startArray(KEY_MAPPINGS);
-            MappingMetaData mmd = indexMetaData.mapping();
-            if (mmd != null) {
-                if (binary) {
-                    builder.value(mmd.source().compressed());
-                } else {
-                    builder.map(XContentHelper.convertToMap(new BytesArray(mmd.source().uncompressed()), true).v2());
+            if (context != MetaData.XContentContext.API) {
+                builder.startArray(KEY_MAPPINGS);
+                MappingMetaData mmd = indexMetaData.mapping();
+                if (mmd != null) {
+                    if (binary) {
+                        builder.value(mmd.source().compressed());
+                    } else {
+                        builder.map(XContentHelper.convertToMap(new BytesArray(mmd.source().uncompressed()), true).v2());
+                    }
+                }
+                builder.endArray();
+            } else {
+                builder.startObject(KEY_MAPPINGS);
+                MappingMetaData mmd = indexMetaData.mapping();
+                if (mmd != null) {
+                    Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(mmd.source().uncompressed()), false).v2();
+                    if (mapping.size() == 1 && mapping.containsKey(mmd.type())) {
+                        // the type name is the root value, reduce it
+                        mapping = (Map<String, Object>) mapping.get(mmd.type());
+                    }
+                    builder.field(mmd.type());
+                    builder.map(mapping);
                 }
+                builder.endObject();
             }
-            builder.endArray();
 
             for (ObjectObjectCursor<String, DiffableStringMap> cursor : indexMetaData.customData) {
                 builder.field(cursor.key);
                 builder.map(cursor.value);
             }
 
-            builder.startObject(KEY_ALIASES);
-            for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
-                AliasMetaData.Builder.toXContent(cursor.value, builder, params);
-            }
-            builder.endObject();
+            if (context != MetaData.XContentContext.API) {
+                builder.startObject(KEY_ALIASES);
+                for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
+                    AliasMetaData.Builder.toXContent(cursor.value, builder, params);
+                }
+                builder.endObject();
 
-            builder.startArray(KEY_PRIMARY_TERMS);
-            for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) {
-                builder.value(indexMetaData.primaryTerm(i));
+                builder.startArray(KEY_PRIMARY_TERMS);
+                for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) {
+                    builder.value(indexMetaData.primaryTerm(i));
+                }
+                builder.endArray();
+            } else {
+                builder.startArray(KEY_ALIASES);
+                for (ObjectCursor<String> cursor : indexMetaData.getAliases().keys()) {
+                    builder.value(cursor.value);
+                }
+                builder.endArray();
+
+                builder.startObject(IndexMetaData.KEY_PRIMARY_TERMS);
+                for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) {
+                    builder.field(Integer.toString(shard), indexMetaData.primaryTerm(shard));
+                }
+                builder.endObject();
             }
-            builder.endArray();
+
 
             builder.startObject(KEY_IN_SYNC_ALLOCATIONS);
             for (IntObjectCursor<Set<String>> cursor : indexMetaData.inSyncAllocationIds) {
@@ -1414,7 +1454,13 @@ public static Settings addHumanReadableSettings(Settings settings) {
         return builder.build();
     }
 
-    private static final ToXContent.Params FORMAT_PARAMS = new MapParams(Collections.singletonMap("binary", "true"));
+    private static final ToXContent.Params FORMAT_PARAMS;
+    static {
+        Map<String, String> params = new HashMap<>(2);
+        params.put("binary", "true");
+        params.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
+        FORMAT_PARAMS = new MapParams(params);
+    }
 
     /**
      * State format for {@link IndexMetaData} to write to and load from disk
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
index 4d2db27a4a840..0e59dded012e7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
@@ -378,7 +378,6 @@ private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
                                             XContentBuilder builder,
                                             ToXContent.Params params,
                                             boolean includeTypeName) throws IOException {
-
             builder.field("order", indexTemplateMetaData.order());
             if (indexTemplateMetaData.version() != null) {
                 builder.field("version", indexTemplateMetaData.version());
@@ -412,7 +411,7 @@ private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
 
         @SuppressWarnings("unchecked")
         private static Map<String, Object> reduceMapping(Map<String, Object> mapping) {
-            assert mapping.keySet().size() == 1;
+            assert mapping.keySet().size() == 1 : mapping.keySet();
             return (Map<String, Object>) mapping.values().iterator().next();
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 8471364b07dea..fb174690a6a43 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -51,10 +51,8 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.MetaDataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -156,6 +154,8 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment {
 
     public static final String CONTEXT_MODE_GATEWAY = XContentContext.GATEWAY.toString();
 
+    public static final String CONTEXT_MODE_API = XContentContext.API.toString();
+
     public static final String GLOBAL_STATE_FILE_PREFIX = "global-";
 
     private static final NamedDiffableValueSerializer<Custom> CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class);
@@ -674,6 +674,12 @@ public Map<String, IndexTemplateV2> templatesV2() {
             .orElse(Collections.emptyMap());
     }
 
+    public Map<String, DataStream> dataStreams() {
+        return Optional.ofNullable((DataStreamMetadata) this.custom(DataStreamMetadata.TYPE))
+            .map(DataStreamMetadata::dataStreams)
+            .orElse(Collections.emptyMap());
+    }
+
     public ImmutableOpenMap<String, Custom> customs() {
         return this.customs;
     }
@@ -802,7 +808,7 @@ public static Diff<MetaData> readDiffFrom(StreamInput in) throws IOException {
     }
 
     public static MetaData fromXContent(XContentParser parser) throws IOException {
-        return Builder.fromXContent(parser, false);
+        return Builder.fromXContent(parser);
     }
 
     @Override
@@ -1121,6 +1127,32 @@ public Builder removeIndexTemplate(String name) {
             return this;
         }
 
+        public Builder dataStreams(Map<String, DataStream> dataStreams) {
+            this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(dataStreams));
+            return this;
+        }
+
+        public Builder put(DataStream dataStream) {
+            Objects.requireNonNull(dataStream, "it is invalid to add a null data stream");
+            Map<String, DataStream> existingDataStreams =
+                Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE))
+                    .map(dsmd -> new HashMap<>(dsmd.dataStreams()))
+                    .orElse(new HashMap<>());
+            existingDataStreams.put(dataStream.getName(), dataStream);
+            this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams));
+            return this;
+        }
+
+        public Builder removeDataStream(String name) {
+            Map<String, DataStream> existingDataStreams =
+                Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE))
+                    .map(dsmd -> new HashMap<>(dsmd.dataStreams()))
+                    .orElse(new HashMap<>());
+            existingDataStreams.remove(name);
+            this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStreams));
+            return this;
+        }
+
         public Custom getCustom(String type) {
             return customs.get(type);
         }
@@ -1339,20 +1371,16 @@ private SortedMap<String, AliasOrIndex> buildAliasAndIndexLookup() {
             return aliasAndIndexLookup;
         }
 
-        public static String toXContent(MetaData metaData) throws IOException {
-            XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-            builder.startObject();
-            toXContent(metaData, builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-            return Strings.toString(builder);
-        }
-
         public static void toXContent(MetaData metaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
-            XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, "API"));
+            XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API));
 
-            builder.startObject("meta-data");
+            if (context == XContentContext.API) {
+                builder.startObject("metadata");
+            } else {
+                builder.startObject("meta-data");
+                builder.field("version", metaData.version());
+            }
 
-            builder.field("version", metaData.version());
             builder.field("cluster_uuid", metaData.clusterUUID);
             builder.field("cluster_uuid_committed", metaData.clusterUUIDCommitted);
 
@@ -1360,25 +1388,19 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
             metaData.coordinationMetaData().toXContent(builder, params);
             builder.endObject();
 
-            if (!metaData.persistentSettings().isEmpty()) {
+            if (context != XContentContext.API && !metaData.persistentSettings().isEmpty()) {
                 builder.startObject("settings");
                 metaData.persistentSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
                 builder.endObject();
             }
 
-            if (context == XContentContext.API && !metaData.transientSettings().isEmpty()) {
-                builder.startObject("transient_settings");
-                metaData.transientSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
-                builder.endObject();
-            }
-
             builder.startObject("templates");
             for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
                 IndexTemplateMetaData.Builder.toXContentWithTypes(cursor.value, builder, params);
             }
             builder.endObject();
 
-            if (context == XContentContext.API && !metaData.indices().isEmpty()) {
+            if (context == XContentContext.API) {
                 builder.startObject("indices");
                 for (IndexMetaData indexMetaData : metaData) {
                     IndexMetaData.Builder.toXContent(indexMetaData, builder, params);
@@ -1396,7 +1418,7 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
             builder.endObject();
         }
 
-        public static MetaData fromXContent(XContentParser parser, boolean preserveUnknownCustoms) throws IOException {
+        public static MetaData fromXContent(XContentParser parser) throws IOException {
             Builder builder = new Builder();
 
             // we might get here after the meta-data element, or on a fresh parser
@@ -1446,13 +1468,8 @@ public static MetaData fromXContent(XContentParser parser, boolean preserveUnkno
                             Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
                             builder.putCustom(custom.getWriteableName(), custom);
                         } catch (NamedObjectNotFoundException ex) {
-                            if (preserveUnknownCustoms) {
-                                logger.warn("Adding unknown custom object with type {}", currentFieldName);
-                                builder.putCustom(currentFieldName, new UnknownGatewayOnlyCustom(parser.mapOrdered()));
-                            } else {
-                                logger.warn("Skipping unknown custom object with type {}", currentFieldName);
-                                parser.skipChildren();
-                            }
+                            logger.warn("Skipping unknown custom object with type {}", currentFieldName);
+                            parser.skipChildren();
                         }
                     }
                 } else if (token.isValue()) {
@@ -1473,45 +1490,6 @@ public static MetaData fromXContent(XContentParser parser, boolean preserveUnkno
         }
     }
 
-    public static class UnknownGatewayOnlyCustom implements Custom {
-
-        private final Map<String, Object> contents;
-
-        UnknownGatewayOnlyCustom(Map<String, Object> contents) {
-            this.contents = contents;
-        }
-
-        @Override
-        public EnumSet<XContentContext> context() {
-            return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.GATEWAY);
-        }
-
-        @Override
-        public Diff<Custom> diff(Custom previousState) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public String getWriteableName() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public Version getMinimalSupportedVersion() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return builder.mapContents(contents);
-        }
-    }
-
     private static final ToXContent.Params FORMAT_PARAMS;
     static {
         Map<String, String> params = new HashMap<>(2);
@@ -1523,25 +1501,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * State format for {@link MetaData} to write to and load from disk
      */
-    public static final MetaDataStateFormat<MetaData> FORMAT = createMetaDataStateFormat(false);
-
-    /**
-     * Special state format for {@link MetaData} to write to and load from disk, preserving unknown customs
-     */
-    public static final MetaDataStateFormat<MetaData> FORMAT_PRESERVE_CUSTOMS = createMetaDataStateFormat(true);
+    public static final MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<>(GLOBAL_STATE_FILE_PREFIX) {
 
-    private static MetaDataStateFormat<MetaData> createMetaDataStateFormat(boolean preserveUnknownCustoms) {
-        return new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {
+        @Override
+        public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
+            Builder.toXContent(state, builder, FORMAT_PARAMS);
+        }
 
-            @Override
-            public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
-                Builder.toXContent(state, builder, FORMAT_PARAMS);
-            }
+        @Override
+        public MetaData fromXContent(XContentParser parser) throws IOException {
+            return Builder.fromXContent(parser);
+        }
+    };
 
-            @Override
-            public MetaData fromXContent(XContentParser parser) throws IOException {
-                return Builder.fromXContent(parser, preserveUnknownCustoms);
-            }
-        };
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index ecce58e5a7154..21b7333eeace7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -154,7 +155,7 @@ public void onFailure(String source, Exception e) {
                 }
 
                 @Override
-                public ClusterState execute(ClusterState currentState) {
+                public ClusterState execute(ClusterState currentState) throws Exception {
                     return addComponentTemplate(currentState, create, name, template);
                 }
 
@@ -166,14 +167,15 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
     }
 
     // Package visible for testing
-    static ClusterState addComponentTemplate(final ClusterState currentState, final boolean create,
-                                             final String name, final ComponentTemplate template) {
+    ClusterState addComponentTemplate(final ClusterState currentState, final boolean create,
+                                             final String name, final ComponentTemplate template) throws Exception {
         if (create && currentState.metaData().componentTemplates().containsKey(name)) {
             throw new IllegalArgumentException("component template [" + name + "] already exists");
         }
 
-        // TODO: validation of component template
-        // validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
+        CompressedXContent mappings = template.template().mappings();
+        String stringMappings = mappings == null ? null : mappings.string();
+        validateTemplate(template.template().settings(), stringMappings, indicesService, xContentRegistry);
 
         logger.info("adding component template [{}]", name);
         return ClusterState.builder(currentState)
@@ -274,7 +276,20 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                     throw new IllegalArgumentException("index_template [" + request.name + "] already exists");
                 }
 
-                validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
+                templateBuilder.order(request.order);
+                templateBuilder.version(request.version);
+                templateBuilder.patterns(request.indexPatterns);
+                templateBuilder.settings(request.settings);
+
+                if (request.mappings != null) {
+                    try {
+                        templateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, request.mappings);
+                    } catch (Exception e) {
+                        throw new MapperParsingException("Failed to parse mapping: {}", e, request.mappings);
+                    }
+                }
+
+                validateTemplate(request.settings, request.mappings, indicesService, xContentRegistry);
 
                 for (Alias alias : request.aliases) {
                     AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
@@ -355,20 +370,20 @@ public static List<IndexTemplateMetaData> findTemplates(MetaData metaData, Strin
         return matchedTemplates;
     }
 
-    private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder,
-            IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
+    private static void validateTemplate(Settings settings, String mappings,
+                                         IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
         Index createdIndex = null;
         final String temporaryIndexName = UUIDs.randomBase64UUID();
         try {
             // use the provided values, otherwise just pick valid dummy values
-            int dummyPartitionSize = IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(request.settings);
-            int dummyShards = request.settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS,
+            int dummyPartitionSize = IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(settings);
+            int dummyShards = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS,
                     dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1);
 
             //create index service for parsing and validating "mappings"
             Settings dummySettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put(request.settings)
+                .put(settings)
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, dummyShards)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
@@ -378,19 +393,9 @@ private static void validateAndAddTemplate(final PutRequest request, IndexTempla
             IndexService dummyIndexService = indicesService.createIndex(tmpIndexMetadata, Collections.emptyList(), false);
             createdIndex = dummyIndexService.index();
 
-            templateBuilder.order(request.order);
-            templateBuilder.version(request.version);
-            templateBuilder.patterns(request.indexPatterns);
-            templateBuilder.settings(request.settings);
-
-            if (request.mappings != null) {
-                try {
-                    templateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, request.mappings);
-                } catch (Exception e) {
-                    throw new MapperParsingException("Failed to parse mapping: {}", e, request.mappings);
-                }
+            if (mappings != null) {
                 dummyIndexService.mapperService().merge(MapperService.SINGLE_MAPPING_NAME,
-                    MapperService.parseMapping(xContentRegistry, request.mappings), MergeReason.MAPPING_UPDATE);
+                    MapperService.parseMapping(xContentRegistry, mappings), MergeReason.MAPPING_UPDATE);
             }
 
         } finally {
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
index 6e9ad85aec56c..f131d03b12908 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
@@ -73,6 +73,10 @@ public static boolean isIngestNode(Settings settings) {
         return Node.NODE_INGEST_SETTING.get(settings);
     }
 
+    public static boolean isRemoteClusterClient(final Settings settings) {
+        return Node.NODE_REMOTE_CLUSTER_CLIENT.get(settings);
+    }
+
     private final String nodeName;
     private final String nodeId;
     private final String ephemeralId;
@@ -281,7 +285,7 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             // an old node will only understand legacy roles since pluggable roles is a new concept
             final List<DiscoveryNodeRole> rolesToWrite =
-                    roles.stream().filter(DiscoveryNodeRole.BUILT_IN_ROLES::contains).collect(Collectors.toUnmodifiableList());
+                    roles.stream().filter(DiscoveryNodeRole.LEGACY_ROLES::contains).collect(Collectors.toUnmodifiableList());
             out.writeVInt(rolesToWrite.size());
             for (final DiscoveryNodeRole role : rolesToWrite) {
                 if (role == DiscoveryNodeRole.MASTER_ROLE) {
@@ -356,6 +360,15 @@ public boolean isIngestNode() {
         return roles.contains(DiscoveryNodeRole.INGEST_ROLE);
     }
 
+    /**
+     * Returns whether or not the node can be a remote cluster client.
+     *
+     * @return true if the node can be a remote cluster client, false otherwise
+     */
+    public boolean isRemoteClusterClient() {
+        return roles.contains(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE);
+    }
+
     /**
      * Returns a set of all the roles that the node fulfills.
      * If the node doesn't have any specific role, the set is returned empty, which means that the node is a coordinating only node.
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
index 4bdf27793f66d..8d293b9231ae5 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
@@ -134,10 +134,21 @@ protected Setting<Boolean> roleSetting() {
 
     };
 
+    public static final DiscoveryNodeRole REMOTE_CLUSTER_CLIENT_ROLE = new DiscoveryNodeRole("remote_cluster_client", "r") {
+
+        @Override
+        protected Setting<Boolean> roleSetting() {
+            return Node.NODE_REMOTE_CLUSTER_CLIENT;
+        }
+
+    };
+
     /**
      * The built-in node roles.
      */
-    public static Set<DiscoveryNodeRole> BUILT_IN_ROLES = Set.of(DATA_ROLE, INGEST_ROLE, MASTER_ROLE);
+    public static Set<DiscoveryNodeRole> BUILT_IN_ROLES = Set.of(DATA_ROLE, INGEST_ROLE, MASTER_ROLE, REMOTE_CLUSTER_CLIENT_ROLE);
+
+    static Set<DiscoveryNodeRole> LEGACY_ROLES = Set.of(DATA_ROLE, INGEST_ROLE, MASTER_ROLE);
 
     /**
      * Represents an unknown role. This can occur if a newer version adds a role that an older version does not know about, or a newer
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index bc5d51b918796..627ac98467f8a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -98,6 +98,11 @@ public GroupShardsIterator<ShardIterator> searchShards(ClusterState clusterState
         return GroupShardsIterator.sortAndCreate(new ArrayList<>(set));
     }
 
+    public static ShardIterator getShards(ClusterState clusterState, ShardId shardId) {
+        final IndexShardRoutingTable shard = clusterState.routingTable().shardRoutingTable(shardId);
+        return shard.activeInitializingShardsRandomIt();
+    }
+
     private static final Map<String, Set<String>> EMPTY_ROUTING = Collections.emptyMap();
 
     private Set<IndexShardRoutingTable> computeTargetedShards(ClusterState clusterState, String[] concreteIndices,
diff --git a/server/src/main/java/org/elasticsearch/common/bytes/ByteBufferReference.java b/server/src/main/java/org/elasticsearch/common/bytes/ByteBufferReference.java
index 07353ea67eced..d0156d581c4d5 100644
--- a/server/src/main/java/org/elasticsearch/common/bytes/ByteBufferReference.java
+++ b/server/src/main/java/org/elasticsearch/common/bytes/ByteBufferReference.java
@@ -51,6 +51,25 @@ public int getInt(int index) {
         return buffer.getInt(index);
     }
 
+    @Override
+    public int indexOf(byte marker, int from) {
+        final int remainingBytes = Math.max(length - from, 0);
+        Objects.checkFromIndexSize(from, remainingBytes, length);
+        if (buffer.hasArray()) {
+            int startIndex = from + buffer.arrayOffset();
+            int endIndex = startIndex + remainingBytes;
+            final byte[] array = buffer.array();
+            for (int i = startIndex; i < endIndex; i++) {
+                if (array[i] == marker) {
+                    return (i - buffer.arrayOffset());
+                }
+            }
+            return -1;
+        } else {
+            return super.indexOf(marker, from);
+        }
+    }
+
     @Override
     public int length() {
         return length;
diff --git a/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java b/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
index 84a2b06eb801f..19df6d3a90db7 100644
--- a/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
+++ b/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
@@ -70,6 +70,34 @@ public byte get(int index) {
         return references[i].get(index - offsets[i]);
     }
 
+    @Override
+    public int indexOf(byte marker, int from) {
+        final int remainingBytes = Math.max(length - from, 0);
+        Objects.checkFromIndexSize(from, remainingBytes, length);
+
+        int result = -1;
+        if (length == 0) {
+            return result;
+        }
+
+        final int firstReferenceIndex = getOffsetIndex(from);
+        for (int i = firstReferenceIndex; i < references.length; ++i) {
+            final BytesReference reference = references[i];
+            final int internalFrom;
+            if (i == firstReferenceIndex) {
+                internalFrom = from - offsets[firstReferenceIndex];
+            } else {
+                internalFrom = 0;
+            }
+            result = reference.indexOf(marker, internalFrom);
+            if (result != -1) {
+                result += offsets[i];
+                break;
+            }
+        }
+        return result;
+    }
+
     @Override
     public int length() {
         return length;
diff --git a/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java b/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
index 646e6c6138230..794a8db4960c6 100644
--- a/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
+++ b/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
@@ -85,7 +85,7 @@ public StreamInput streamInput(StreamInput in) throws IOException {
         final Inflater inflater = new Inflater(nowrap);
         InputStream decompressedIn = new InflaterInputStream(in, inflater, BUFFER_SIZE);
         decompressedIn = new BufferedInputStream(decompressedIn, BUFFER_SIZE);
-        final InputStreamStreamInput inputStreamStreamInput = new InputStreamStreamInput(decompressedIn) {
+        return new InputStreamStreamInput(decompressedIn) {
             final AtomicBoolean closed = new AtomicBoolean(false);
 
             public void close() throws IOException {
@@ -99,9 +99,6 @@ public void close() throws IOException {
                 }
             }
         };
-
-        inputStreamStreamInput.setVersion(in.getVersion());
-        return inputStreamStreamInput;
     }
 
     @Override
@@ -112,7 +109,7 @@ public StreamOutput streamOutput(StreamOutput out) throws IOException {
         final boolean syncFlush = true;
         DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(out, deflater, BUFFER_SIZE, syncFlush);
         OutputStream compressedOut = new BufferedOutputStream(deflaterOutputStream, BUFFER_SIZE);
-        final OutputStreamStreamOutput outputStreamStreamOutput = new OutputStreamStreamOutput(compressedOut) {
+        return new OutputStreamStreamOutput(compressedOut) {
             final AtomicBoolean closed = new AtomicBoolean(false);
 
             public void close() throws IOException {
@@ -126,7 +123,5 @@ public void close() throws IOException {
                 }
             }
         };
-        outputStreamStreamOutput.setVersion(out.getVersion());
-        return outputStreamStreamOutput;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/io/Streams.java b/server/src/main/java/org/elasticsearch/common/io/Streams.java
index 3747c4d895a20..222f94e65ef6a 100644
--- a/server/src/main/java/org/elasticsearch/common/io/Streams.java
+++ b/server/src/main/java/org/elasticsearch/common/io/Streams.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.common.io;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStream;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -297,15 +296,5 @@ public void reset() throws IOException {
         public BytesReference bytes() {
             return delegate.bytes();
         }
-
-        @Override
-        public Version getVersion() {
-            return delegate.getVersion();
-        }
-
-        @Override
-        public void setVersion(Version version) {
-            delegate.setVersion(version);
-        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
new file mode 100644
index 0000000000000..b27010017e5ee
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.io.stream;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesReference;
+
+/**
+ * A holder for {@link Writeable}s that can delays reading the underlying
+ * {@linkplain Writeable} when it is read from a remote node.
+ */
+public abstract class DelayableWriteable<T extends Writeable> implements Supplier<T>, Writeable {
+    /**
+     * Build a {@linkplain DelayableWriteable} that wraps an existing object
+     * but is serialized so that deserializing it can be delayed.
+     */
+    public static <T extends Writeable> DelayableWriteable<T> referencing(T reference) {
+        return new Referencing<>(reference);
+    }
+    /**
+     * Build a {@linkplain DelayableWriteable} that copies a buffer from
+     * the provided {@linkplain StreamInput} and deserializes the buffer
+     * when {@link Supplier#get()} is called.
+     */
+    public static <T extends Writeable> DelayableWriteable<T> delayed(Writeable.Reader<T> reader, StreamInput in) throws IOException {
+        return new Delayed<>(reader, in);
+    }
+
+    private DelayableWriteable() {}
+
+    public abstract boolean isDelayed();
+
+    private static class Referencing<T extends Writeable> extends DelayableWriteable<T> {
+        private T reference;
+
+        Referencing(T reference) {
+            this.reference = reference;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            try (BytesStreamOutput buffer = new BytesStreamOutput()) {
+                buffer.setVersion(out.getVersion());
+                reference.writeTo(buffer);
+                out.writeBytesReference(buffer.bytes());
+            }
+        }
+
+        @Override
+        public T get() {
+            return reference;
+        }
+
+        @Override
+        public boolean isDelayed() {
+            return false;
+        }
+    }
+
+    private static class Delayed<T extends Writeable> extends DelayableWriteable<T> {
+        private final Writeable.Reader<T> reader;
+        private final Version remoteVersion;
+        private final BytesReference serialized;
+        private final NamedWriteableRegistry registry;
+
+        Delayed(Writeable.Reader<T> reader, StreamInput in) throws IOException {
+            this.reader = reader;
+            remoteVersion = in.getVersion();
+            serialized = in.readBytesReference();
+            registry = in.namedWriteableRegistry();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            if (out.getVersion() == remoteVersion) {
+                /*
+                 * If the version *does* line up we can just copy the bytes
+                 * which is good because this is how shard request caching
+                 * works.
+                 */
+                out.writeBytesReference(serialized);
+            } else {
+                /*
+                 * If the version doesn't line up then we have to deserialize
+                 * into the Writeable and re-serialize it against the new
+                 * output stream so it can apply any backwards compatibility
+                 * differences in the wire protocol. This ain't efficient but
+                 * it should be quite rare.
+                 */
+                referencing(get()).writeTo(out);
+            }
+        }
+
+        @Override
+        public T get() {
+            try {
+                try (StreamInput in = registry == null ?
+                        serialized.streamInput() : new NamedWriteableAwareStreamInput(serialized.streamInput(), registry)) {
+                    in.setVersion(remoteVersion);
+                    return reader.read(in);
+                }
+            } catch (IOException e) {
+                throw new RuntimeException("unexpected error expanding aggregations", e);
+            }
+        }
+
+        @Override
+        public boolean isDelayed() {
+            return true;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
index 2fa700634b851..6d7d9fd6952fd 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
@@ -94,4 +94,9 @@ public void setVersion(Version version) {
     protected void ensureCanReadBytes(int length) throws EOFException {
         delegate.ensureCanReadBytes(length);
     }
+
+    @Override
+    public NamedWriteableRegistry namedWriteableRegistry() {
+        return delegate.namedWriteableRegistry();
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
index 5db80a711eeb9..149231b1fe433 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
@@ -52,4 +52,9 @@ public <C extends NamedWriteable> C readNamedWriteable(@SuppressWarnings("unused
             + "] than it was read from [" + name + "].";
         return c;
     }
+
+    @Override
+    public NamedWriteableRegistry namedWriteableRegistry() {
+        return namedWriteableRegistry;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index d30b226dfb2dc..7de344d95dc46 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -1097,6 +1097,14 @@ public <T extends Exception> T readException() throws IOException {
         return null;
     }
 
+    /**
+     * Get the registry of named writeables is his stream has one,
+     * {@code null} otherwise.
+     */
+    public NamedWriteableRegistry namedWriteableRegistry() {
+        return null;
+    }
+
     /**
      * Reads a {@link NamedWriteable} from the current stream, by first reading its name and then looking for
      * the corresponding entry in the registry by name, so that the proper object can be read and returned.
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 5c47b55b363d8..7fa616e1c6340 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -87,6 +87,7 @@
 public abstract class StreamOutput extends OutputStream {
 
     private static final Map<TimeUnit, Byte> TIME_UNIT_BYTE_MAP;
+    private static final int MAX_NESTED_EXCEPTION_LEVEL = 100;
 
     static {
         final Map<TimeUnit, Byte> timeUnitByteMap = new EnumMap<>(TimeUnit.class);
@@ -923,8 +924,15 @@ public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOExcep
     }
 
     public void writeException(Throwable throwable) throws IOException {
+        writeException(throwable, throwable, 0);
+    }
+
+    private void writeException(Throwable rootException, Throwable throwable, int nestedLevel) throws IOException {
         if (throwable == null) {
             writeBoolean(false);
+        } else if (nestedLevel > MAX_NESTED_EXCEPTION_LEVEL) {
+            assert failOnTooManyNestedExceptions(rootException);
+            writeException(new IllegalStateException("too many nested exceptions"));
         } else {
             writeBoolean(true);
             boolean writeCause = true;
@@ -1033,12 +1041,16 @@ public void writeException(Throwable throwable) throws IOException {
                 writeOptionalString(throwable.getMessage());
             }
             if (writeCause) {
-                writeException(throwable.getCause());
+                writeException(rootException, throwable.getCause(), nestedLevel + 1);
             }
-            ElasticsearchException.writeStackTraces(throwable, this);
+            ElasticsearchException.writeStackTraces(throwable, this, (o, t) -> o.writeException(rootException, t, nestedLevel + 1));
         }
     }
 
+    boolean failOnTooManyNestedExceptions(Throwable throwable) {
+        throw new AssertionError("too many nested exceptions", throwable);
+    }
+
     /**
      * Writes a {@link NamedWriteable} to the current stream, by first writing its name and then the object itself
      */
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index d121752017c58..719da2b89ff39 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.common.settings;
 
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.Build;
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
 import org.elasticsearch.action.search.TransportSearchAction;
@@ -98,7 +97,6 @@
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestController;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchService;
@@ -113,7 +111,6 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -183,9 +180,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         }
     }
 
-    public static final Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS;
-    static {
-        final Set<Setting<?>> alwaysEnabled = Set.of(
+    public static Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Set.of(
             AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING,
             AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING,
             BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING,
@@ -299,7 +294,6 @@ public void apply(Settings value, Settings current, Settings previous) {
             SniffConnectionStrategy.REMOTE_CONNECTIONS_PER_CLUSTER,
             RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING,
             RemoteClusterService.REMOTE_NODE_ATTRIBUTE,
-            RemoteClusterService.ENABLE_REMOTE_CLUSTERS,
             RemoteClusterService.REMOTE_CLUSTER_PING_SCHEDULE,
             RemoteClusterService.REMOTE_CLUSTER_COMPRESS,
             RemoteConnectionStrategy.REMOTE_CONNECTION_MODE,
@@ -407,6 +401,7 @@ public void apply(Settings value, Settings current, Settings previous) {
             Node.NODE_DATA_SETTING,
             Node.NODE_MASTER_SETTING,
             Node.NODE_INGEST_SETTING,
+            Node.NODE_REMOTE_CLUSTER_CLIENT,
             Node.NODE_ATTRIBUTES,
             Node.NODE_LOCAL_STORAGE_SETTING,
             AutoCreateIndex.AUTO_CREATE_INDEX_SETTING,
@@ -482,15 +477,6 @@ public void apply(Settings value, Settings current, Settings previous) {
             HandshakingTransportAddressConnector.PROBE_CONNECT_TIMEOUT_SETTING,
             HandshakingTransportAddressConnector.PROBE_HANDSHAKE_TIMEOUT_SETTING);
 
-        if (Build.CURRENT.isSnapshot()) {
-            Set<Setting<?>> modifiable = new HashSet<>(alwaysEnabled);
-            modifiable.add(RestController.RESTRICT_SYSTEM_INDICES);
-            BUILT_IN_CLUSTER_SETTINGS = Set.copyOf(modifiable);
-        } else {
-            BUILT_IN_CLUSTER_SETTINGS = alwaysEnabled;
-        }
-    }
-
-    static final List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList();
+    static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList();
 
 }
diff --git a/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java b/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
index 0b32091fb4fa1..a5d7cf7d641d5 100644
--- a/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
+++ b/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -35,7 +36,14 @@
 
 public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue>, ToXContentFragment {
 
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(ByteSizeValue.class));
+    /**
+     * We have to lazy initialize the deprecation logger as otherwise a static logger here would be constructed before logging is configured
+     * leading to a runtime failure (see {@link LogConfigurator#checkErrorListener()} ). The premature construction would come from any
+     * {@link ByteSizeValue} object constructed in, for example, settings in {@link org.elasticsearch.common.network.NetworkService}.
+     */
+    static class DeprecationLoggerHolder {
+        static DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(ByteSizeValue.class));
+    }
 
     public static final ByteSizeValue ZERO = new ByteSizeValue(0, ByteSizeUnit.BYTES);
 
@@ -227,7 +235,7 @@ private static ByteSizeValue parse(final String initialInput, final String norma
             } catch (final NumberFormatException e) {
                 try {
                     final double doubleValue = Double.parseDouble(s);
-                    deprecationLogger.deprecated(
+                    DeprecationLoggerHolder.deprecationLogger.deprecated(
                             "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [{}] found for setting [{}]",
                             initialInput, settingName);
                     return new ByteSizeValue((long) (doubleValue * unit.toBytes(1)));
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index edc2c563ae2a3..e4cb14857932f 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -20,10 +20,8 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.client.OriginSettingClient;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -50,7 +48,6 @@
 import java.util.stream.Collector;
 import java.util.stream.Stream;
 
-import static java.util.Collections.emptyList;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE;
 
@@ -65,7 +62,7 @@
  * Consumers of ThreadContext usually don't need to interact with adding or stashing contexts. Every elasticsearch thread is managed by
  * a thread pool or executor being responsible for stashing and restoring the threads context. For instance if a network request is
  * received, all headers are deserialized from the network and directly added as the headers of the threads {@link ThreadContext}
- * (see {@link #readFrom(StreamInput)}. In order to not modify the context that is currently active on this thread the network code
+ * (see {@link #readHeaders(StreamInput)}. In order to not modify the context that is currently active on this thread the network code
  * uses a try/with pattern to stash it's current context, read headers into a fresh one and once the request is handled or a handler thread
  * is forked (which in turn inherits the context) it restores the previous context. For instance:
  * </p>
@@ -233,18 +230,17 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     /**
-     * Reads the values from the stream into the current context
+     * Reads the headers from the stream into the current context
      */
-    public void readFrom(StreamInput in) throws IOException {
+    public void readHeaders(StreamInput in) throws IOException {
         final Tuple<Map<String, String>, Map<String, Set<String>>> streamTuple = readHeadersFromStream(in);
         final Map<String, String>  requestHeaders = streamTuple.v1();
         final Map<String, Set<String>> responseHeaders = streamTuple.v2();
-        final List<String> allowedSystemIndices = readAllowedSystemIndices(in);
         final ThreadContextStruct struct;
-        if (requestHeaders.isEmpty() && responseHeaders.isEmpty() && allowedSystemIndices.isEmpty()) {
+        if (requestHeaders.isEmpty() && responseHeaders.isEmpty()) {
             struct = ThreadContextStruct.EMPTY;
         } else {
-            struct = new ThreadContextStruct(requestHeaders, responseHeaders, Collections.emptyMap(), allowedSystemIndices, false, 0L);
+            struct = new ThreadContextStruct(requestHeaders, responseHeaders, Collections.emptyMap(), false);
         }
         threadLocal.set(struct);
     }
@@ -271,14 +267,6 @@ public static Tuple<Map<String, String>, Map<String, Set<String>>> readHeadersFr
         return new Tuple<>(requestHeaders, responseHeaders);
     }
 
-    public static List<String> readAllowedSystemIndices(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
-            return in.readOptionalStringList();
-        } else {
-            return emptyList();
-        }
-    }
-
     /**
      * Returns the header for the given key or <code>null</code> if not present
      */
@@ -422,36 +410,6 @@ public boolean isSystemContext() {
         return threadLocal.get().isSystemContext;
     }
 
-    /**
-     * Returns <code>true</code> if a request made within this context can access system indices
-     */
-    public boolean isSystemIndexAccessAllowed() {
-        return threadLocal.get().allowedSystemIndexPatterns != null;
-    }
-
-    /**
-     * Sets the context to disallow access to system indices
-     */
-    public void disallowSystemIndexAccess() {
-        threadLocal.set(threadLocal.get().setAllowSystemIndices(null));
-    }
-
-    /**
-     * Sets the context to allow access to system indices
-     */
-    public void allowSystemIndexAccess(List<String> patterns) {
-        threadLocal.set(threadLocal.get().setAllowSystemIndices(patterns));
-    }
-
-    /**
-     * Returns the list of allowed system index patterns or {@code null} if none are allowed. An
-     * empty list indicates that all system indices are allowed to be accessed.
-     */
-    @Nullable
-    public List<String> allowedSystemIndexPatterns() {
-        return threadLocal.get().allowedSystemIndexPatterns;
-    }
-
     @FunctionalInterface
     public interface StoredContext extends AutoCloseable {
         @Override
@@ -483,7 +441,6 @@ private static final class ThreadContextStruct {
         private final Map<String, String> requestHeaders;
         private final Map<String, Object> transientHeaders;
         private final Map<String, Set<String>> responseHeaders;
-        private final List<String> allowedSystemIndexPatterns;
         private final boolean isSystemContext;
         //saving current warning headers' size not to recalculate the size with every new warning header
         private final long warningHeadersSize;
@@ -498,40 +455,29 @@ private ThreadContextStruct setSystemContext() {
         private ThreadContextStruct(Map<String, String> requestHeaders,
                                     Map<String, Set<String>> responseHeaders,
                                     Map<String, Object> transientHeaders, boolean isSystemContext) {
-            this(requestHeaders, responseHeaders, transientHeaders, emptyList(), isSystemContext, 0L);
+            this.requestHeaders = requestHeaders;
+            this.responseHeaders = responseHeaders;
+            this.transientHeaders = transientHeaders;
+            this.isSystemContext = isSystemContext;
+            this.warningHeadersSize = 0L;
         }
 
         private ThreadContextStruct(Map<String, String> requestHeaders,
                                     Map<String, Set<String>> responseHeaders,
                                     Map<String, Object> transientHeaders, boolean isSystemContext,
                                     long warningHeadersSize) {
-            this(requestHeaders, responseHeaders, transientHeaders, emptyList(), isSystemContext, warningHeadersSize);
-        }
-
-        private ThreadContextStruct(Map<String, String> requestHeaders,
-                                    Map<String, Set<String>> responseHeaders,
-                                    Map<String, Object> transientHeaders,
-                                    List<String> allowedSystemIndexPatterns,
-                                    boolean isSystemContext,
-                                    long warningHeadersSize) {
             this.requestHeaders = requestHeaders;
             this.responseHeaders = responseHeaders;
             this.transientHeaders = transientHeaders;
             this.isSystemContext = isSystemContext;
             this.warningHeadersSize = warningHeadersSize;
-            this.allowedSystemIndexPatterns = allowedSystemIndexPatterns;
         }
 
         /**
          * This represents the default context and it should only ever be called by {@link #DEFAULT_CONTEXT}.
          */
         private ThreadContextStruct() {
-            this(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), emptyList(), false, 0L);
-        }
-
-        private ThreadContextStruct setAllowSystemIndices(List<String> allowedSystemIndexPatterns) {
-            final List<String> copy = allowedSystemIndexPatterns == null ? null : List.copyOf(allowedSystemIndexPatterns);
-            return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, copy, isSystemContext, warningHeadersSize);
+            this(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), false);
         }
 
         private ThreadContextStruct putRequest(String key, String value) {
@@ -575,8 +521,7 @@ private ThreadContextStruct putResponseHeaders(Map<String, Set<String>> headers)
                     newResponseHeaders.put(key, entry.getValue());
                 }
             }
-            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, allowedSystemIndexPatterns,
-                isSystemContext, 0L);
+            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext);
         }
 
         private ThreadContextStruct putResponse(final String key, final String value, final Function<String, String> uniqueValue,
@@ -626,8 +571,7 @@ private ThreadContextStruct putResponse(final String key, final String value, fi
                     return this;
                 }
             }
-            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, allowedSystemIndexPatterns,
-                isSystemContext, newWarningHeaderSize);
+            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext, newWarningHeaderSize);
         }
 
 
@@ -663,9 +607,6 @@ private void writeTo(StreamOutput out, Map<String, String> defaultHeaders) throw
             }
 
             out.writeMap(responseHeaders, StreamOutput::writeString, StreamOutput::writeStringCollection);
-            if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
-                out.writeOptionalStringCollection(allowedSystemIndexPatterns);
-            }
         }
     }
 
@@ -683,7 +624,7 @@ private ContextPreservingRunnable(Runnable in) {
 
         @Override
         public void run() {
-            try (ThreadContext.StoredContext ignore = stashContext()) {
+            try (ThreadContext.StoredContext ignore = stashContext()){
                 ctx.restore();
                 in.run();
             }
diff --git a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java
index ffef170754075..8cd0c9aa0fae5 100644
--- a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java
+++ b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java
@@ -136,7 +136,6 @@ public class PersistedClusterStateService {
     private final String nodeId;
     private final NamedXContentRegistry namedXContentRegistry;
     private final BigArrays bigArrays;
-    private final boolean preserveUnknownCustoms;
     private final LongSupplier relativeTimeMillisSupplier;
 
     private volatile TimeValue slowWriteLoggingThreshold;
@@ -144,18 +143,16 @@ public class PersistedClusterStateService {
     public PersistedClusterStateService(NodeEnvironment nodeEnvironment, NamedXContentRegistry namedXContentRegistry, BigArrays bigArrays,
                                         ClusterSettings clusterSettings, LongSupplier relativeTimeMillisSupplier) {
         this(nodeEnvironment.nodeDataPaths(), nodeEnvironment.nodeId(), namedXContentRegistry, bigArrays, clusterSettings,
-            relativeTimeMillisSupplier, false);
+            relativeTimeMillisSupplier);
     }
 
     public PersistedClusterStateService(Path[] dataPaths, String nodeId, NamedXContentRegistry namedXContentRegistry, BigArrays bigArrays,
-                                        ClusterSettings clusterSettings, LongSupplier relativeTimeMillisSupplier,
-                                        boolean preserveUnknownCustoms) {
+                                        ClusterSettings clusterSettings, LongSupplier relativeTimeMillisSupplier) {
         this.dataPaths = dataPaths;
         this.nodeId = nodeId;
         this.namedXContentRegistry = namedXContentRegistry;
         this.bigArrays = bigArrays;
         this.relativeTimeMillisSupplier = relativeTimeMillisSupplier;
-        this.preserveUnknownCustoms = preserveUnknownCustoms;
         this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD);
         clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold);
     }
@@ -383,8 +380,7 @@ private OnDiskState loadOnDiskState(Path dataPath, DirectoryReader reader) throw
         consumeFromType(searcher, GLOBAL_TYPE_NAME, bytes ->
         {
             final MetaData metaData = MetaData.Builder.fromXContent(XContentFactory.xContent(XContentType.SMILE)
-                .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, bytes.bytes, bytes.offset, bytes.length),
-                preserveUnknownCustoms);
+                .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, bytes.bytes, bytes.offset, bytes.length));
             logger.trace("found global metadata with last-accepted term [{}]", metaData.coordinationMetaData().term());
             if (builderReference.get() != null) {
                 throw new IllegalStateException("duplicate global metadata found in [" + dataPath + "]");
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 67bbdaf79ce76..1a80aa80a30b0 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -368,14 +368,14 @@ public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int
     }
 
     public Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) {
-        throw new QueryShardException(context, "Can only use prefix queries on keyword and text fields - not on [" + name
+        throw new QueryShardException(context, "Can only use prefix queries on keyword, text and wildcard fields - not on [" + name
             + "] which is of type [" + typeName() + "]");
     }
 
     public Query wildcardQuery(String value,
                                @Nullable MultiTermQuery.RewriteMethod method,
                                QueryShardContext context) {
-        throw new QueryShardException(context, "Can only use wildcard queries on keyword and text fields - not on [" + name
+        throw new QueryShardException(context, "Can only use wildcard queries on keyword, text and wildcard fields - not on [" + name
             + "] which is of type [" + typeName() + "]");
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
index fa7055cadd2a9..ace6e6d6f28ec 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.mapper;
 
 import com.fasterxml.jackson.core.JsonParseException;
-
 import org.apache.lucene.document.DoublePoint;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FloatPoint;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
index 05bf6b61d1de1..37c20b236ed56 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MultiTermQuery;
@@ -93,6 +94,36 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer
         return query;
     }
 
+    public static final String normalizeWildcardPattern(String fieldname, String value, Analyzer normalizer)  {
+        if (normalizer == null) {
+            return value;
+        }
+        // we want to normalize everything except wildcard characters, e.g. F?o Ba* to f?o ba*, even if e.g there
+        // is a char_filter that would otherwise remove them
+        Matcher wildcardMatcher = WILDCARD_PATTERN.matcher(value);
+        BytesRefBuilder sb = new BytesRefBuilder();
+        int last = 0;
+
+        while (wildcardMatcher.find()) {
+            if (wildcardMatcher.start() > 0) {
+                String chunk = value.substring(last, wildcardMatcher.start());
+
+                BytesRef normalized = normalizer.normalize(fieldname, chunk);
+                sb.append(normalized);
+            }
+            // append the matched group - without normalizing
+            sb.append(new BytesRef(wildcardMatcher.group()));
+
+            last = wildcardMatcher.end();
+        }
+        if (last < value.length()) {
+            String chunk = value.substring(last);
+            BytesRef normalized = normalizer.normalize(fieldname, chunk);
+            sb.append(normalized);
+        }
+        return sb.toBytesRef().utf8ToString();
+    }    
+    
     @Override
     public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
         failIfNotIndexed();
@@ -103,30 +134,8 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, Qu
 
         Term term;
         if (searchAnalyzer() != null) {
-            // we want to normalize everything except wildcard characters, e.g. F?o Ba* to f?o ba*, even if e.g there
-            // is a char_filter that would otherwise remove them
-            Matcher wildcardMatcher = WILDCARD_PATTERN.matcher(value);
-            BytesRefBuilder sb = new BytesRefBuilder();
-            int last = 0;
-
-            while (wildcardMatcher.find()) {
-                if (wildcardMatcher.start() > 0) {
-                    String chunk = value.substring(last, wildcardMatcher.start());
-
-                    BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
-                    sb.append(normalized);
-                }
-                // append the matched group - without normalizing
-                sb.append(new BytesRef(wildcardMatcher.group()));
-
-                last = wildcardMatcher.end();
-            }
-            if (last < value.length()) {
-                String chunk = value.substring(last);
-                BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
-                sb.append(normalized);
-            }
-            term = new Term(name(), sb.toBytesRef());
+            value = normalizeWildcardPattern(name(), value, searchAnalyzer());
+            term = new Term(name(), value);
         } else {
             term = new Term(name(), indexedValueForSearch(value));
         }
diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
index 3aa6118936e4d..bb9a007c60ca4 100644
--- a/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
+++ b/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
@@ -239,7 +239,7 @@ public static RangeQueryBuilder rangeQuery(String name) {
      * which matches any single character. Note this query can be slow, as it
      * needs to iterate over many terms. In order to prevent extremely slow WildcardQueries,
      * a Wildcard term should not start with one of the wildcards {@code *} or
-     * {@code ?}.
+     * {@code ?}. (The wildcard field type however, is optimised for leading wildcards)
      *
      * @param name  The field name
      * @param query The wildcard query string
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/RetryListener.java b/server/src/main/java/org/elasticsearch/index/reindex/RetryListener.java
index d197e1bed12f9..c77b73267203f 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/RetryListener.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/RetryListener.java
@@ -72,7 +72,6 @@ public void onRejection(Exception e) {
     }
 
     private void schedule(Runnable runnable, TimeValue delay) {
-        // schedule does not preserve context so have to do this manually
-        threadPool.schedule(threadPool.preserveContext(runnable), delay, ThreadPool.Names.SAME);
+        threadPool.schedule(runnable, delay, ThreadPool.Names.SAME);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskState.java b/server/src/main/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskState.java
index ae2a6a552cba4..5d50cf7791419 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskState.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskState.java
@@ -182,11 +182,11 @@ TimeValue throttledUntil() {
      * Schedule prepareBulkRequestRunnable to run after some delay. This is where throttling plugs into reindexing so the request can be
      * rescheduled over and over again.
      */
-    public void delayPrepareBulkRequest(ThreadPool threadPool, TimeValue lastBatchStartTime, int lastBatchSize,
+    public void delayPrepareBulkRequest(ThreadPool threadPool, long lastBatchStartTimeNS, int lastBatchSize,
                                         AbstractRunnable prepareBulkRequestRunnable) {
         // Synchronize so we are less likely to schedule the same request twice.
         synchronized (delayedPrepareBulkRequestReference) {
-            TimeValue delay = throttleWaitTime(lastBatchStartTime, timeValueNanos(System.nanoTime()), lastBatchSize);
+            TimeValue delay = throttleWaitTime(lastBatchStartTimeNS, System.nanoTime(), lastBatchSize);
             logger.debug("[{}]: preparing bulk request for [{}]", task.getId(), delay);
             try {
                 delayedPrepareBulkRequestReference.set(new DelayedPrepareBulkRequest(threadPool, getRequestsPerSecond(),
@@ -197,8 +197,8 @@ public void delayPrepareBulkRequest(ThreadPool threadPool, TimeValue lastBatchSt
         }
     }
 
-    public TimeValue throttleWaitTime(TimeValue lastBatchStartTime, TimeValue now, int lastBatchSize) {
-        long earliestNextBatchStartTime = now.nanos() + (long) perfectlyThrottledBatchTime(lastBatchSize);
+    public TimeValue throttleWaitTime(long lastBatchStartTimeNS, long nowNS, int lastBatchSize) {
+        long earliestNextBatchStartTime = nowNS + (long) perfectlyThrottledBatchTime(lastBatchSize);
         long waitTime = min(MAX_THROTTLE_WAIT_TIME.nanos(), max(0, earliestNextBatchStartTime - System.nanoTime()));
         return timeValueNanos(waitTime);
     }
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 56707bc198beb..b70ce0280dfc6 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1184,12 +1184,12 @@ public void failShard(String reason, @Nullable Exception e) {
     }
 
     /**
-     * Acquire the searcher without applying the additional reader wrapper.
+     * Acquire a lightweight searcher which can be used to rewrite shard search requests.
      */
-    public Engine.Searcher acquireSearcherNoWrap(String source) {
+    public Engine.Searcher acquireCanMatchSearcher() {
         readAllowed();
         markSearcherAccessed();
-        return getEngine().acquireSearcher(source, Engine.SearcherScope.EXTERNAL);
+        return getEngine().acquireSearcher("can_match", Engine.SearcherScope.EXTERNAL);
     }
 
     public Engine.Searcher acquireSearcher(String source) {
@@ -1208,10 +1208,7 @@ private Engine.Searcher acquireSearcher(String source, Engine.SearcherScope scop
         return wrapSearcher(searcher);
     }
 
-    /**
-     * Wraps the provided searcher acquired with {@link #acquireSearcherNoWrap(String)}.
-     */
-    public Engine.Searcher wrapSearcher(Engine.Searcher searcher) {
+    private Engine.Searcher wrapSearcher(Engine.Searcher searcher) {
         assert ElasticsearchDirectoryReader.unwrap(searcher.getDirectoryReader())
             != null : "DirectoryReader must be an instance or ElasticsearchDirectoryReader";
         boolean success = false;
diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
index 309a3c3a38694..c34a449ad7f79 100644
--- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
+++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
@@ -135,6 +135,7 @@ static final class Fields {
 
     static final class ParseFields {
         static final ParseField FILES = new ParseField("files");
+        static final ParseField SHARD_STATE_ID = new ParseField("shard_state_id");
         static final ParseField SNAPSHOTS = new ParseField("snapshots");
     }
 
@@ -207,6 +208,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.value(fileInfo.name());
             }
             builder.endArray();
+            if (snapshot.shardStateIdentifier() != null) {
+                builder.field(ParseFields.SHARD_STATE_ID.getPreferredName(), snapshot.shardStateIdentifier());
+            }
             builder.endObject();
         }
         builder.endObject();
@@ -219,6 +223,8 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t
             token = parser.nextToken();
         }
         Map<String, List<String>> snapshotsMap = new HashMap<>();
+        Map<String, String> historyUUIDs = new HashMap<>();
+        Map<String, Long> globalCheckpoints = new HashMap<>();
         Map<String, FileInfo> files = new HashMap<>();
         if (token == XContentParser.Token.START_OBJECT) {
             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -250,15 +256,16 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t
                         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                             if (token == XContentParser.Token.FIELD_NAME) {
                                 currentFieldName = parser.currentName();
-                                if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
-                                    if (ParseFields.FILES.match(currentFieldName, parser.getDeprecationHandler()) == false) {
-                                        throw new ElasticsearchParseException("unknown array [{}]", currentFieldName);
-                                    }
+                                if (ParseFields.FILES.match(currentFieldName, parser.getDeprecationHandler()) &&
+                                    parser.nextToken() == XContentParser.Token.START_ARRAY) {
                                     List<String> fileNames = new ArrayList<>();
                                     while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
                                         fileNames.add(parser.text());
                                     }
                                     snapshotsMap.put(snapshot, fileNames);
+                                } else if (ParseFields.SHARD_STATE_ID.match(currentFieldName, parser.getDeprecationHandler())) {
+                                    parser.nextToken();
+                                    historyUUIDs.put(snapshot, parser.text());
                                 }
                             }
                         }
@@ -277,7 +284,8 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t
                 assert fileInfo != null;
                 fileInfosBuilder.add(fileInfo);
             }
-            snapshots.add(new SnapshotFiles(entry.getKey(), Collections.unmodifiableList(fileInfosBuilder)));
+            snapshots.add(new SnapshotFiles(entry.getKey(), Collections.unmodifiableList(fileInfosBuilder),
+                historyUUIDs.get(entry.getKey())));
         }
         return new BlobStoreIndexShardSnapshots(files, Collections.unmodifiableList(snapshots));
     }
diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java
index 5da7b2b1f8c6b..039ea9405a158 100644
--- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java
+++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.index.snapshots.blobstore;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
 
 import java.util.HashMap;
@@ -33,6 +34,9 @@ public class SnapshotFiles {
 
     private final List<FileInfo> indexFiles;
 
+    @Nullable
+    private final String shardStateIdentifier;
+
     private Map<String, FileInfo> physicalFiles = null;
 
     /**
@@ -45,12 +49,23 @@ public String snapshot() {
     }
 
     /**
-     * @param snapshot   snapshot name
-     * @param indexFiles index files
+     * @param snapshot             snapshot name
+     * @param indexFiles           index files
+     * @param shardStateIdentifier unique identifier for the state of the shard that this snapshot was taken from
      */
-    public SnapshotFiles(String snapshot, List<FileInfo> indexFiles ) {
+    public SnapshotFiles(String snapshot, List<FileInfo> indexFiles, @Nullable String shardStateIdentifier) {
         this.snapshot = snapshot;
         this.indexFiles = indexFiles;
+        this.shardStateIdentifier = shardStateIdentifier;
+    }
+
+    /**
+     * Returns an identifier for the shard state that can be used to check whether a shard has changed between
+     * snapshots or not.
+     */
+    @Nullable
+    public String shardStateIdentifier() {
+        return shardStateIdentifier;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
index 49c2d070c0310..898e451aa3bc6 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
@@ -124,7 +124,8 @@ BytesReference getOrCompute(CacheEntity cacheEntity, Supplier<BytesReference> lo
         if (cacheLoader.isLoaded()) {
             key.entity.onMiss();
             if (logger.isTraceEnabled()) {
-                logger.trace("Cache miss for reader version [{}] and request:\n {}", reader.getVersion(), cacheKeyRenderer.get());
+                logger.trace("Cache miss for reader version [{}], max_doc[{}] and request:\n {}",
+                    reader.getVersion(), reader.maxDoc(), cacheKeyRenderer.get());
             }
             // see if its the first time we see this reader, and make sure to register a cleanup key
             CleanupKey cleanupKey = new CleanupKey(cacheEntity, reader.getReaderCacheHelper().getKey());
@@ -137,7 +138,8 @@ BytesReference getOrCompute(CacheEntity cacheEntity, Supplier<BytesReference> lo
         } else {
             key.entity.onHit();
             if (logger.isTraceEnabled()) {
-                logger.trace("Cache hit for reader version [{}] and request:\n {}", reader.getVersion(), cacheKeyRenderer.get());
+                logger.trace("Cache hit for reader version [{}], max_doc[{}] and request:\n {}",
+                    reader.getVersion(), reader.maxDoc(), cacheKeyRenderer.get());
             }
         }
         return value;
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index 224c640c941ff..01145e6281161 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -200,6 +200,8 @@ public class Node implements Closeable {
         Setting.boolSetting("node.master", true, Property.NodeScope);
     public static final Setting<Boolean> NODE_INGEST_SETTING =
         Setting.boolSetting("node.ingest", true, Property.NodeScope);
+    public static final Setting<Boolean> NODE_REMOTE_CLUSTER_CLIENT =
+        Setting.boolSetting("node.remote_cluster_client", true, Property.NodeScope);
 
     /**
     * controls whether the node is allowed to persist things like metadata to disk
@@ -440,7 +442,7 @@ protected Node(final Environment initialEnvironment,
                 .stream()
                 .collect(Collectors.toUnmodifiableMap(
                     plugin -> plugin.getClass().getSimpleName(),
-                    plugin -> plugin.getSystemIndexDescriptors(settings)));
+                    plugin -> plugin.getSystemIndexDescriptors()));
             SystemIndexDescriptor.checkForOverlappingPatterns(systemIndexDescriptorMap);
 
             final List<SystemIndexDescriptor> systemIndexDescriptors = systemIndexDescriptorMap.values().stream()
@@ -476,7 +478,7 @@ protected Node(final Environment initialEnvironment,
 
             ActionModule actionModule = new ActionModule(settings, clusterModule.getIndexNameExpressionResolver(),
                 settingsModule.getIndexScopedSettings(), settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(),
-                threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService);
+                threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService, clusterService);
             modules.add(actionModule);
 
             final RestController restController = actionModule.getRestController();
diff --git a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java
index 821f141b3eabc..a4d7a36f38670 100644
--- a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.plugins;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 
 import java.util.Collection;
@@ -34,10 +33,9 @@ public interface SystemIndexPlugin extends ActionPlugin {
     /**
      * Returns a {@link Collection} of {@link SystemIndexDescriptor}s that describe this plugin's system indices, including
      * name, mapping, and settings.
-     * @param settings The node's settings
      * @return Descriptions of the system indices managed by this plugin.
      */
-    default Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    default Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.emptyList();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
index f41137c763289..ec708976f4e75 100644
--- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
@@ -121,10 +121,10 @@ public boolean isReadOnly() {
 
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                              Map<String, Object> userMetadata, ActionListener<String> listener) {
-        in.snapshotShard(
-            store, mapperService, snapshotId, indexId, snapshotIndexCommit, snapshotStatus, repositoryMetaVersion, userMetadata, listener);
+                              IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                              Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
+        in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, shardStateIdentifier, snapshotStatus,
+            repositoryMetaVersion, userMetadata, listener);
     }
     @Override
     public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState,
diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java
index bca034564bdd9..c2e93c172eb94 100644
--- a/server/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.ShardId;
@@ -199,14 +200,17 @@ void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations,
      * @param snapshotId            snapshot id
      * @param indexId               id for the index being snapshotted
      * @param snapshotIndexCommit   commit point
+     * @param shardStateIdentifier  a unique identifier of the state of the shard that is stored with the shard's snapshot and used
+     *                              to detect if the shard has changed between snapshots. If {@code null} is passed as the identifier
+     *                              snapshotting will be done by inspecting the physical files referenced by {@code snapshotIndexCommit}
      * @param snapshotStatus        snapshot status
      * @param repositoryMetaVersion version of the updated repository metadata to write
      * @param userMetadata          user metadata of the snapshot found in {@link SnapshotsInProgress.Entry#userMetadata()}
      * @param listener              listener invoked on completion
      */
     void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit,
-                       IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion, Map<String, Object> userMetadata,
-                       ActionListener<String> listener);
+                       @Nullable String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
+                       Map<String, Object> userMetadata, ActionListener<String> listener);
 
     /**
      * Restores snapshot of the shard.
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index fdb70cffb133f..7102bbaa943f9 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -125,6 +125,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
@@ -1534,8 +1535,8 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b
 
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                              Map<String, Object> userMetadata, ActionListener<String> listener) {
+                              IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                              Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
         final ShardId shardId = store.shardId();
         final long startTime = threadPool.absoluteTimeInMillis();
         try {
@@ -1561,76 +1562,92 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s
                 throw new IndexShardSnapshotFailedException(shardId,
                     "Duplicate snapshot name [" + snapshotId.getName() + "] detected, aborting");
             }
-
-            final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles = new ArrayList<>();
-            final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new LinkedBlockingQueue<>();
-            store.incRef();
-            final Collection<String> fileNames;
-            final Store.MetadataSnapshot metadataFromStore;
-            try {
-                // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
-                try {
-                    logger.trace(
-                        "[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit);
-                    metadataFromStore = store.getMetadata(snapshotIndexCommit);
-                    fileNames = snapshotIndexCommit.getFileNames();
-                } catch (IOException e) {
-                    throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
+            // First inspect all known SegmentInfos instances to see if we already have an equivalent commit in the repository
+            final List<BlobStoreIndexShardSnapshot.FileInfo> filesFromSegmentInfos = Optional.ofNullable(shardStateIdentifier).map(id -> {
+                for (SnapshotFiles snapshotFileSet : snapshots.snapshots()) {
+                    if (id.equals(snapshotFileSet.shardStateIdentifier())) {
+                        return snapshotFileSet.indexFiles();
+                    }
                 }
-            } finally {
-                store.decRef();
-            }
+                return null;
+            }).orElse(null);
+
+            final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles;
             int indexIncrementalFileCount = 0;
             int indexTotalNumberOfFiles = 0;
             long indexIncrementalSize = 0;
-            long indexTotalFileCount = 0;
-            for (String fileName : fileNames) {
-                if (snapshotStatus.isAborted()) {
-                    logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
-                    throw new IndexShardSnapshotFailedException(shardId, "Aborted");
+            long indexTotalFileSize = 0;
+            final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new LinkedBlockingQueue<>();
+            // If we did not find a set of files that is equal to the current commit we determine the files to upload by comparing files
+            // in the commit with files already in the repository
+            if (filesFromSegmentInfos == null) {
+                indexCommitPointFiles = new ArrayList<>();
+                store.incRef();
+                final Collection<String> fileNames;
+                final Store.MetadataSnapshot metadataFromStore;
+                try {
+                    // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
+                    try {
+                        logger.trace(
+                            "[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit);
+                        metadataFromStore = store.getMetadata(snapshotIndexCommit);
+                        fileNames = snapshotIndexCommit.getFileNames();
+                    } catch (IOException e) {
+                        throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
+                    }
+                } finally {
+                    store.decRef();
                 }
+                for (String fileName : fileNames) {
+                    if (snapshotStatus.isAborted()) {
+                        logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
+                        throw new IndexShardSnapshotFailedException(shardId, "Aborted");
+                    }
 
-                logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName);
-                final StoreFileMetaData md = metadataFromStore.get(fileName);
-                BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null;
-                List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
-                if (filesInfo != null) {
-                    for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) {
-                        if (fileInfo.isSame(md)) {
-                            // a commit point file with the same name, size and checksum was already copied to repository
-                            // we will reuse it for this snapshot
-                            existingFileInfo = fileInfo;
-                            break;
+                    logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName);
+                    final StoreFileMetaData md = metadataFromStore.get(fileName);
+                    BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null;
+                    List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
+                    if (filesInfo != null) {
+                        for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) {
+                            if (fileInfo.isSame(md)) {
+                                // a commit point file with the same name, size and checksum was already copied to repository
+                                // we will reuse it for this snapshot
+                                existingFileInfo = fileInfo;
+                                break;
+                            }
                         }
                     }
-                }
 
-                // We can skip writing blobs where the metadata hash is equal to the blob's contents because we store the hash/contents
-                // directly in the shard level metadata in this case
-                final boolean needsWrite = md.hashEqualsContents() == false;
-                indexTotalFileCount += md.length();
-                indexTotalNumberOfFiles++;
-
-                if (existingFileInfo == null) {
-                    indexIncrementalFileCount++;
-                    indexIncrementalSize += md.length();
-                    // create a new FileInfo
-                    BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo =
-                        new BlobStoreIndexShardSnapshot.FileInfo(
-                            (needsWrite ? UPLOADED_DATA_BLOB_PREFIX : VIRTUAL_DATA_BLOB_PREFIX) + UUIDs.randomBase64UUID(),
-                            md, chunkSize());
-                    indexCommitPointFiles.add(snapshotFileInfo);
-                    if (needsWrite) {
-                        filesToSnapshot.add(snapshotFileInfo);
+                    // We can skip writing blobs where the metadata hash is equal to the blob's contents because we store the hash/contents
+                    // directly in the shard level metadata in this case
+                    final boolean needsWrite = md.hashEqualsContents() == false;
+                    indexTotalFileSize += md.length();
+                    indexTotalNumberOfFiles++;
+
+                    if (existingFileInfo == null) {
+                        indexIncrementalFileCount++;
+                        indexIncrementalSize += md.length();
+                        // create a new FileInfo
+                        BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo =
+                            new BlobStoreIndexShardSnapshot.FileInfo(
+                                (needsWrite ? UPLOADED_DATA_BLOB_PREFIX : VIRTUAL_DATA_BLOB_PREFIX) + UUIDs.randomBase64UUID(),
+                                md, chunkSize());
+                        indexCommitPointFiles.add(snapshotFileInfo);
+                        if (needsWrite) {
+                            filesToSnapshot.add(snapshotFileInfo);
+                        }
+                        assert needsWrite || assertFileContentsMatchHash(snapshotFileInfo, store);
+                    } else {
+                        indexCommitPointFiles.add(existingFileInfo);
                     }
-                    assert needsWrite || assertFileContentsMatchHash(snapshotFileInfo, store);
-                } else {
-                    indexCommitPointFiles.add(existingFileInfo);
                 }
+            } else {
+                indexCommitPointFiles = filesFromSegmentInfos;
             }
 
             snapshotStatus.moveToStarted(startTime, indexIncrementalFileCount,
-                indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileCount);
+                indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileSize);
 
             final StepListener<Collection<Void>> allFilesUploadedListener = new StepListener<>();
             allFilesUploadedListener.whenComplete(v -> {
@@ -1655,7 +1672,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s
                 }
                 // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones
                 List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
-                newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
+                newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), shardStateIdentifier));
                 for (SnapshotFiles point : snapshots) {
                     newSnapshotsList.add(point);
                 }
@@ -1742,7 +1759,7 @@ public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, Sh
         final BlobContainer container = shardContainer(indexId, snapshotShardId);
         executor.execute(ActionRunnable.wrap(restoreListener, l -> {
             final BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(container, snapshotId);
-            final SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
+            final SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles(), null);
             new FileRestoreContext(metadata.name(), shardId, snapshotId, recoveryState) {
                 @Override
                 protected void restoreFiles(List<BlobStoreIndexShardSnapshot.FileInfo> filesToRecover, Store store,
diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
index 101c3182fbb62..f2fd12bfa79c2 100644
--- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
@@ -183,57 +183,4 @@ protected Set<String> responseParams() {
         return Collections.emptySet();
     }
 
-    public static class Wrapper extends BaseRestHandler {
-
-        protected final BaseRestHandler delegate;
-
-        public Wrapper(BaseRestHandler delegate) {
-            this.delegate = delegate;
-        }
-
-        @Override
-        public String getName() {
-            return delegate.getName();
-        }
-
-        @Override
-        public List<Route> routes() {
-            return delegate.routes();
-        }
-
-        @Override
-        public List<DeprecatedRoute> deprecatedRoutes() {
-            return delegate.deprecatedRoutes();
-        }
-
-        @Override
-        public List<ReplacedRoute> replacedRoutes() {
-            return delegate.replacedRoutes();
-        }
-
-        @Override
-        protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-            return delegate.prepareRequest(request, client);
-        }
-
-        @Override
-        protected Set<String> responseParams() {
-            return delegate.responseParams();
-        }
-
-        @Override
-        public boolean canTripCircuitBreaker() {
-            return delegate.canTripCircuitBreaker();
-        }
-
-        @Override
-        public boolean supportsContentStream() {
-            return delegate.supportsContentStream();
-        }
-
-        @Override
-        public boolean allowsUnsafeBuffers() {
-            return delegate.allowsUnsafeBuffers();
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 0537b4d40a044..bb015d6632e6b 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -31,8 +31,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.path.PathTrie;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -66,10 +64,6 @@ public class RestController implements HttpServerTransport.Dispatcher {
     private static final Logger logger = LogManager.getLogger(RestController.class);
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
-    // TODO once we are ready, this should default to true
-    public static final Setting<Boolean> RESTRICT_SYSTEM_INDICES =
-        Setting.boolSetting("rest.restrict_system_indices", false, Property.NodeScope);
-
     private final PathTrie<MethodHandlers> handlers = new PathTrie<>(RestUtils.REST_DECODER);
 
     private final UnaryOperator<RestHandler> handlerWrapper;
@@ -81,10 +75,9 @@ public class RestController implements HttpServerTransport.Dispatcher {
     /** Rest headers that are copied to internal requests made during a rest request. */
     private final Set<RestHeaderDefinition> headersToCopy;
     private final UsageService usageService;
-    private final boolean restrictSystemIndices;
 
     public RestController(Set<RestHeaderDefinition> headersToCopy, UnaryOperator<RestHandler> handlerWrapper,
-            NodeClient client, CircuitBreakerService circuitBreakerService, UsageService usageService, boolean restrictSystemIndices) {
+            NodeClient client, CircuitBreakerService circuitBreakerService, UsageService usageService) {
         this.headersToCopy = headersToCopy;
         this.usageService = usageService;
         if (handlerWrapper == null) {
@@ -93,7 +86,6 @@ public RestController(Set<RestHeaderDefinition> headersToCopy, UnaryOperator<Res
         this.handlerWrapper = handlerWrapper;
         this.client = client;
         this.circuitBreakerService = circuitBreakerService;
-        this.restrictSystemIndices = restrictSystemIndices;
     }
 
     /**
@@ -178,13 +170,6 @@ public void dispatchRequest(RestRequest request, RestChannel channel, ThreadCont
             handleFavicon(request.method(), request.uri(), channel);
             return;
         }
-
-        if (restrictSystemIndices) {
-            threadContext.disallowSystemIndexAccess();
-        } else {
-            assert threadContext.isSystemIndexAccessAllowed();
-        }
-
         try {
             tryAllHandlers(request, channel, threadContext);
         } catch (Exception e) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java b/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java
index 7d8e71502eb24..224c266bd2b18 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java
@@ -38,7 +38,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN;
@@ -86,41 +85,28 @@ static int getNumTasks(HttpChannel channel) {
     @Override
     public <Request extends ActionRequest, Response extends ActionResponse> void doExecute(
         ActionType<Response> action, Request request, ActionListener<Response> listener) {
-        final AtomicBoolean created = new AtomicBoolean(false);
-        CloseListener closeListener = httpChannels.computeIfAbsent(httpChannel, channel -> {
-            created.set(true);
-            return new CloseListener();
-        });
+        CloseListener closeListener = httpChannels.computeIfAbsent(httpChannel, channel -> new CloseListener());
         TaskHolder taskHolder = new TaskHolder();
-        final Task task;
-        boolean success = false;
-        try {
-            task = client.executeLocally(action, request,
-                new ActionListener<>() {
-                    @Override
-                    public void onResponse(Response response) {
-                        try {
-                            closeListener.unregisterTask(taskHolder);
-                        } finally {
-                            listener.onResponse(response);
-                        }
+        Task task = client.executeLocally(action, request,
+            new ActionListener<>() {
+                @Override
+                public void onResponse(Response response) {
+                    try {
+                        closeListener.unregisterTask(taskHolder);
+                    } finally {
+                        listener.onResponse(response);
                     }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        try {
-                            closeListener.unregisterTask(taskHolder);
-                        } finally {
-                            listener.onFailure(e);
-                        }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    try {
+                        closeListener.unregisterTask(taskHolder);
+                    } finally {
+                        listener.onFailure(e);
                     }
-                });
-            success = true;
-        } finally {
-            if (success == false && created.get()) {
-                httpChannels.remove(httpChannel);
-            }
-        }
+                }
+            });
         final TaskId taskId = new TaskId(client.getLocalNodeId(), task.getId());
         closeListener.registerTask(taskHolder, taskId);
         closeListener.maybeRegisterChannel(httpChannel);
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
index 0b3d8b698cf58..ea7f97c3815f7 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
@@ -34,23 +34,64 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Locale;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
-
 public class RestNodesHotThreadsAction extends BaseRestHandler {
 
+    private static final String formatDeprecatedMessageWithoutNodeID = "[%s] is a deprecated endpoint. " + 
+        "Please use [/_nodes/hot_threads] instead.";
+    private static final String formatDeprecatedMessageWithNodeID = "[%s] is a deprecated endpoint. " + 
+        "Please use [/_nodes/{nodeId}/hot_threads] instead.";
+    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_HOT_THREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithoutNodeID, 
+        "/_cluster/nodes/hot_threads"
+    ); 
+    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOT_THREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithNodeID, 
+        "/_cluster/nodes/{nodeId}/hot_threads"
+    );
+    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_HOTTHREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithoutNodeID, 
+        "/_cluster/nodes/hotthreads"
+    ); 
+    private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOTTHREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithNodeID, 
+        "/_cluster/nodes/{nodeId}/hotthreads"
+    ); 
+    private static final String DEPRECATED_MESSAGE_NODES_HOTTHREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithoutNodeID, 
+        "/_nodes/hotthreads"
+    ); 
+    private static final String DEPRECATED_MESSAGE_NODES_NODEID_HOTTHREADS = String.format(Locale.ROOT, 
+        formatDeprecatedMessageWithNodeID, 
+        "/_nodes/{nodeId}/hotthreads"
+    );
+
+    @Override
+    public List<DeprecatedRoute> deprecatedRoutes() {
+        return List.of(
+            new DeprecatedRoute(GET, "/_cluster/nodes/hot_threads",
+                    DEPRECATED_MESSAGE_CLUSTER_NODES_HOT_THREADS),
+            new DeprecatedRoute(GET, "/_cluster/nodes/{nodeId}/hot_threads",
+                    DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOT_THREADS),
+            new DeprecatedRoute(GET, "/_cluster/nodes/hotthreads",
+                    DEPRECATED_MESSAGE_CLUSTER_NODES_HOTTHREADS),
+            new DeprecatedRoute(GET, "/_cluster/nodes/{nodeId}/hotthreads",
+                    DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOTTHREADS),
+            new DeprecatedRoute(GET, "/_nodes/hotthreads",
+                    DEPRECATED_MESSAGE_NODES_HOTTHREADS),
+            new DeprecatedRoute(GET, "/_nodes/{nodeId}/hotthreads",
+                    DEPRECATED_MESSAGE_NODES_NODEID_HOTTHREADS));
+    }
+
     @Override
     public List<Route> routes() {
         return List.of(
-            new Route(GET, "/_cluster/nodes/hotthreads"),
-            new Route(GET, "/_cluster/nodes/hot_threads"),
-            new Route(GET, "/_cluster/nodes/{nodeId}/hotthreads"),
-            new Route(GET, "/_cluster/nodes/{nodeId}/hot_threads"),
-            new Route(GET, "/_nodes/hotthreads"),
             new Route(GET, "/_nodes/hot_threads"),
-            new Route(GET, "/_nodes/{nodeId}/hotthreads"),
-            new Route(GET, "/_nodes/{nodeId}/hot_threads"));
+            new Route(GET, "/_nodes/{nodeId}/hot_threads")
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
index 12920599fbd19..363a17978843e 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
@@ -36,17 +36,7 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestNodesInfoAction extends BaseRestHandler {
-    static final Set<String> ALLOWED_METRICS = Sets.newHashSet(
-            "http",
-            "ingest",
-            "indices",
-            "jvm",
-            "os",
-            "plugins",
-            "process",
-            "settings",
-            "thread_pool",
-            "transport");
+    static final Set<String> ALLOWED_METRICS = NodesInfoRequest.Metric.allMetrics();
 
     private final SettingsFilter settingsFilter;
 
@@ -108,16 +98,9 @@ static NodesInfoRequest prepareRequest(final RestRequest request) {
             nodesInfoRequest.all();
         } else {
             nodesInfoRequest.clear();
-            nodesInfoRequest.settings(metrics.contains("settings"));
-            nodesInfoRequest.os(metrics.contains("os"));
-            nodesInfoRequest.process(metrics.contains("process"));
-            nodesInfoRequest.jvm(metrics.contains("jvm"));
-            nodesInfoRequest.threadPool(metrics.contains("thread_pool"));
-            nodesInfoRequest.transport(metrics.contains("transport"));
-            nodesInfoRequest.http(metrics.contains("http"));
-            nodesInfoRequest.plugins(metrics.contains("plugins"));
-            nodesInfoRequest.ingest(metrics.contains("ingest"));
-            nodesInfoRequest.indices(metrics.contains("indices"));
+            // disregard unknown metrics
+            metrics.retainAll(ALLOWED_METRICS);
+            nodesInfoRequest.addMetrics(metrics.toArray(String[]::new));
         }
         return nodesInfoRequest;
     }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
index 13c2a5c581eb3..557266e302af5 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
@@ -67,7 +67,8 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
-                nodesInfoRequest.clear().jvm(false).os(false).process(true);
+                nodesInfoRequest.clear()
+                    .addMetric(NodesInfoRequest.Metric.PROCESS.metricName());
                 client.admin().cluster().nodesInfo(nodesInfoRequest, new RestResponseListener<NodesInfoResponse>(channel) {
                     @Override
                     public RestResponse buildResponse(NodesInfoResponse nodesInfoResponse) throws Exception {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
index 0833a04ff45a1..f391a10b18b3f 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -97,7 +97,11 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
-                nodesInfoRequest.clear().jvm(true).os(true).process(true).http(true);
+                nodesInfoRequest.clear().addMetrics(
+                        NodesInfoRequest.Metric.JVM.metricName(),
+                        NodesInfoRequest.Metric.OS.metricName(),
+                        NodesInfoRequest.Metric.PROCESS.metricName(),
+                        NodesInfoRequest.Metric.HTTP.metricName());
                 client.admin().cluster().nodesInfo(nodesInfoRequest, new RestActionListener<NodesInfoResponse>(channel) {
                     @Override
                     public void processResponse(final NodesInfoResponse nodesInfoResponse) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
index 9d4b4b08d625e..c8fa82b773567 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
@@ -67,7 +67,8 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) throws Exception {
                 NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
-                nodesInfoRequest.clear().plugins(true);
+                nodesInfoRequest.clear()
+                    .addMetric(NodesInfoRequest.Metric.PLUGINS.metricName());
                 client.admin().cluster().nodesInfo(nodesInfoRequest, new RestResponseListener<NodesInfoResponse>(channel) {
                     @Override
                     public RestResponse buildResponse(final NodesInfoResponse nodesInfoResponse) throws Exception {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 40971a425e07e..fd83b2d34bea9 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -200,6 +200,9 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("warmer.total", "alias:wto,warmerTotal;default:false;text-align:right;desc:total warmer ops");
         table.addCell("warmer.total_time", "alias:wtt,warmerTotalTime;default:false;text-align:right;desc:time spent in warmers");
 
+        table.addCell("path.data", "alias:pd,dataPath;default:false;text-align:right;desc:shard data path");
+        table.addCell("path.state", "alias:ps,statsPath;default:false;text-align:right;desc:shard state path");
+
         table.endHeaders();
         return table;
     }
@@ -214,7 +217,8 @@ private static <S, T> Object getOrNull(S stats, Function<S, T> accessor, Functio
         return null;
     }
 
-    private Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) {
+    // package private for testing
+    Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) {
         Table table = getTableWithHeader(request);
 
         for (ShardRouting shard : state.getState().routingTable().allShards()) {
@@ -351,6 +355,9 @@ private Table buildTable(RestRequest request, ClusterStateResponse state, Indice
             table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::total));
             table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::totalTime));
 
+            table.addCell(getOrNull(shardStats, ShardStats::getDataPath, s -> s));
+            table.addCell(getOrNull(shardStats, ShardStats::getStatePath, s -> s));
+
             table.endRow();
         }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
index 480c329808f86..f53c43565a068 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
@@ -80,7 +80,9 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
-                nodesInfoRequest.clear().process(true).threadPool(true);
+                nodesInfoRequest.clear().addMetrics(
+                        NodesInfoRequest.Metric.PROCESS.metricName(),
+                        NodesInfoRequest.Metric.THREAD_POOL.metricName());
                 client.admin().cluster().nodesInfo(nodesInfoRequest, new RestActionListener<NodesInfoResponse>(channel) {
                     @Override
                     public void processResponse(final NodesInfoResponse nodesInfoResponse) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
index f155b09aafb42..4a86ca118b176 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -33,7 +33,6 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
-import java.util.function.Supplier;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
@@ -82,10 +81,10 @@ void validateOpType(String opType) {
 
     public static final class AutoIdHandler extends RestIndexAction {
 
-        private final Supplier<DiscoveryNodes> nodesInCluster;
+        private final ClusterService clusterService;
 
-        public AutoIdHandler(Supplier<DiscoveryNodes> nodesInCluster) {
-            this.nodesInCluster = nodesInCluster;
+        public AutoIdHandler(ClusterService clusterService) {
+            this.clusterService = clusterService;
         }
 
         @Override
@@ -101,7 +100,7 @@ public List<Route> routes() {
         @Override
         public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException {
             assert request.params().get("id") == null : "non-null id: " + request.params().get("id");
-            if (request.params().get("op_type") == null && nodesInCluster.get().getMinNodeVersion().onOrAfter(Version.V_7_5_0)) {
+            if (request.params().get("op_type") == null && clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.V_7_5_0)) {
                 // default to op_type create
                 request.params().put("op_type", "create");
             }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestClearReaderAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestClearReaderAction.java
new file mode 100644
index 0000000000000..c672ad821778f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestClearReaderAction.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.search;
+
+import org.elasticsearch.action.search.ClearReaderAction;
+import org.elasticsearch.action.search.ClearReaderRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestStatusToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.DELETE;
+
+public class RestClearReaderAction extends BaseRestHandler {
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(DELETE, "/_search/reader"));
+    }
+
+    @Override
+    public String getName() {
+        return "clear_reader_action";
+    }
+
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        final ClearReaderRequest clearRequest;
+        try (XContentParser parser = request.contentOrSourceParamParser()) {
+            clearRequest = ClearReaderRequest.fromXContent(parser);
+        }
+        return channel -> client.execute(ClearReaderAction.INSTANCE, clearRequest, new RestStatusToXContentListener<>(channel));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index 4cef66b5da4b1..ee6cba31d26f1 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -94,7 +94,10 @@ public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean a
             multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0));
         }
 
-        int preFilterShardSize = restRequest.paramAsInt("pre_filter_shard_size", SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE);
+        Integer preFilterShardSize = null;
+        if (restRequest.hasParam("pre_filter_shard_size")) {
+            preFilterShardSize = restRequest.paramAsInt("pre_filter_shard_size", SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE);
+        }
 
         final Integer maxConcurrentShardRequests;
         if (restRequest.hasParam("max_concurrent_shard_requests")) {
@@ -111,10 +114,11 @@ public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean a
             multiRequest.add(searchRequest);
         });
         List<SearchRequest> requests = multiRequest.requests();
-        preFilterShardSize = Math.max(1, preFilterShardSize / (requests.size()+1));
         for (SearchRequest request : requests) {
             // preserve if it's set on the request
-            request.setPreFilterShardSize(Math.min(preFilterShardSize, request.getPreFilterShardSize()));
+            if (preFilterShardSize != null && request.getPreFilterShardSize() == null) {
+                request.setPreFilterShardSize(preFilterShardSize);
+            }
             if (maxConcurrentShardRequests != null) {
                 request.setMaxConcurrentShardRequests(maxConcurrentShardRequests);
             }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index b95aaf4b86018..37f674d70e57e 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -128,7 +128,9 @@ public static void parseSearchRequest(SearchRequest searchRequest, RestRequest r
 
         final int batchedReduceSize = request.paramAsInt("batched_reduce_size", searchRequest.getBatchedReduceSize());
         searchRequest.setBatchedReduceSize(batchedReduceSize);
-        searchRequest.setPreFilterShardSize(request.paramAsInt("pre_filter_shard_size", searchRequest.getPreFilterShardSize()));
+        if (request.hasParam("pre_filter_shard_size")) {
+            searchRequest.setPreFilterShardSize(request.paramAsInt("pre_filter_shard_size", SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE));
+        }
 
         if (request.hasParam("max_concurrent_shard_requests")) {
             // only set if we have the parameter since we auto adjust the max concurrency on the coordinator
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index a2a0013718104..2bdff5cafc965 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -96,6 +96,7 @@ final class DefaultSearchContext extends SearchContext {
     private final QuerySearchResult queryResult;
     private final FetchSearchResult fetchResult;
     private final float queryBoost;
+    private final boolean lowLevelCancellation;
     private TimeValue timeout;
     // terminate after count
     private int terminateAfter = DEFAULT_TERMINATE_AFTER;
@@ -115,7 +116,6 @@ final class DefaultSearchContext extends SearchContext {
     private int trackTotalHitsUpTo = SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO;
     private FieldDoc searchAfter;
     private CollapseContext collapse;
-    private boolean lowLevelCancellation;
     // filter for sliced scroll
     private SliceBuilder sliceBuilder;
     private SearchShardTask task;
@@ -147,9 +147,17 @@ final class DefaultSearchContext extends SearchContext {
     private final QueryShardContext queryShardContext;
     private final FetchPhase fetchPhase;
 
-    DefaultSearchContext(ReaderContext readerContext, ShardSearchRequest request, SearchShardTarget shardTarget,
-                         ClusterService clusterService, IndexService indexService, IndexShard indexShard, BigArrays bigArrays,
-                         LongSupplier relativeTimeSupplier, TimeValue timeout, FetchPhase fetchPhase) throws IOException {
+    DefaultSearchContext(ReaderContext readerContext,
+                         ShardSearchRequest request,
+                         SearchShardTarget shardTarget,
+                         ClusterService clusterService,
+                         IndexService indexService,
+                         IndexShard indexShard,
+                         BigArrays bigArrays,
+                         LongSupplier relativeTimeSupplier,
+                         TimeValue timeout,
+                         FetchPhase fetchPhase,
+                         boolean lowLevelCancellation) throws IOException {
         this.readerContext = readerContext;
         this.request = request;
         this.fetchPhase = fetchPhase;
@@ -165,12 +173,13 @@ final class DefaultSearchContext extends SearchContext {
         this.clusterService = clusterService;
         final Engine.Searcher engineSearcher = readerContext.engineSearcher();
         this.searcher = new ContextIndexSearcher(engineSearcher.getIndexReader(), engineSearcher.getSimilarity(),
-            engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy());
+            engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy(), lowLevelCancellation);
         this.relativeTimeSupplier = relativeTimeSupplier;
         this.timeout = timeout;
         queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher,
             request::nowInMillis, shardTarget.getClusterAlias());
         queryBoost = request.indexBoost();
+        this.lowLevelCancellation = lowLevelCancellation;
     }
 
     @Override
@@ -545,10 +554,6 @@ public boolean lowLevelCancellation() {
         return lowLevelCancellation;
     }
 
-    public void lowLevelCancellation(boolean lowLevelCancellation) {
-        this.lowLevelCancellation = lowLevelCancellation;
-    }
-
     @Override
     public FieldDoc searchAfter() {
         return searchAfter;
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index 3ecf9e2371682..dea00a9b3012d 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -396,23 +396,23 @@ private void registerAggregations(List<SearchPlugin> plugins) {
         registerAggregation(new AggregationSpec(SignificantTextAggregationBuilder.NAME, SignificantTextAggregationBuilder::new,
                 SignificantTextAggregationBuilder::parse));
         registerAggregation(new AggregationSpec(RangeAggregationBuilder.NAME, RangeAggregationBuilder::new,
-                RangeAggregationBuilder::parse).addResultReader(InternalRange::new));
+                RangeAggregationBuilder.PARSER).addResultReader(InternalRange::new));
         registerAggregation(new AggregationSpec(DateRangeAggregationBuilder.NAME, DateRangeAggregationBuilder::new,
-                DateRangeAggregationBuilder::parse).addResultReader(InternalDateRange::new));
+                DateRangeAggregationBuilder.PARSER).addResultReader(InternalDateRange::new));
         registerAggregation(new AggregationSpec(IpRangeAggregationBuilder.NAME, IpRangeAggregationBuilder::new,
-                IpRangeAggregationBuilder::parse).addResultReader(InternalBinaryRange::new));
+                IpRangeAggregationBuilder.PARSER).addResultReader(InternalBinaryRange::new));
         registerAggregation(new AggregationSpec(HistogramAggregationBuilder.NAME, HistogramAggregationBuilder::new,
-                HistogramAggregationBuilder::parse).addResultReader(InternalHistogram::new));
+                HistogramAggregationBuilder.PARSER).addResultReader(InternalHistogram::new));
         registerAggregation(new AggregationSpec(DateHistogramAggregationBuilder.NAME, DateHistogramAggregationBuilder::new,
-                DateHistogramAggregationBuilder::parse).addResultReader(InternalDateHistogram::new));
+                DateHistogramAggregationBuilder.PARSER).addResultReader(InternalDateHistogram::new));
         registerAggregation(new AggregationSpec(AutoDateHistogramAggregationBuilder.NAME, AutoDateHistogramAggregationBuilder::new,
-                AutoDateHistogramAggregationBuilder::parse).addResultReader(InternalAutoDateHistogram::new));
+                AutoDateHistogramAggregationBuilder.PARSER).addResultReader(InternalAutoDateHistogram::new));
         registerAggregation(new AggregationSpec(GeoDistanceAggregationBuilder.NAME, GeoDistanceAggregationBuilder::new,
                 GeoDistanceAggregationBuilder::parse).addResultReader(InternalGeoDistance::new));
         registerAggregation(new AggregationSpec(GeoHashGridAggregationBuilder.NAME, GeoHashGridAggregationBuilder::new,
-                GeoHashGridAggregationBuilder::parse).addResultReader(InternalGeoHashGrid::new));
+                GeoHashGridAggregationBuilder.PARSER).addResultReader(InternalGeoHashGrid::new));
         registerAggregation(new AggregationSpec(GeoTileGridAggregationBuilder.NAME, GeoTileGridAggregationBuilder::new,
-                GeoTileGridAggregationBuilder::parse).addResultReader(InternalGeoTileGrid::new));
+                GeoTileGridAggregationBuilder.PARSER).addResultReader(InternalGeoTileGrid::new));
         registerAggregation(new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new,
                 NestedAggregationBuilder::parse).addResultReader(InternalNested::new));
         registerAggregation(new AggregationSpec(ReverseNestedAggregationBuilder.NAME, ReverseNestedAggregationBuilder::new,
@@ -420,9 +420,9 @@ private void registerAggregations(List<SearchPlugin> plugins) {
         registerAggregation(new AggregationSpec(TopHitsAggregationBuilder.NAME, TopHitsAggregationBuilder::new,
                 TopHitsAggregationBuilder::parse).addResultReader(InternalTopHits::new));
         registerAggregation(new AggregationSpec(GeoBoundsAggregationBuilder.NAME, GeoBoundsAggregationBuilder::new,
-                GeoBoundsAggregationBuilder::parse).addResultReader(InternalGeoBounds::new));
+                GeoBoundsAggregationBuilder.PARSER).addResultReader(InternalGeoBounds::new));
         registerAggregation(new AggregationSpec(GeoCentroidAggregationBuilder.NAME, GeoCentroidAggregationBuilder::new,
-                GeoCentroidAggregationBuilder::parse).addResultReader(InternalGeoCentroid::new));
+                GeoCentroidAggregationBuilder.PARSER).addResultReader(InternalGeoCentroid::new));
         registerAggregation(new AggregationSpec(ScriptedMetricAggregationBuilder.NAME, ScriptedMetricAggregationBuilder::new,
                 ScriptedMetricAggregationBuilder.PARSER).addResultReader(InternalScriptedMetric::new));
         registerAggregation((new AggregationSpec(CompositeAggregationBuilder.NAME, CompositeAggregationBuilder::new,
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 991d435b0e7f4..7bb06f4f6b98d 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -243,7 +243,7 @@ public SearchService(ClusterService clusterService, IndicesService indicesServic
 
     private void validateKeepAlives(TimeValue defaultKeepAlive, TimeValue maxKeepAlive) {
         if (defaultKeepAlive.millis() > maxKeepAlive.millis()) {
-            throw new IllegalArgumentException("Default keep alive setting for scroll [" + DEFAULT_KEEPALIVE_SETTING.getKey() + "]" +
+            throw new IllegalArgumentException("Default keep alive setting for request [" + DEFAULT_KEEPALIVE_SETTING.getKey() + "]" +
                 " should be smaller than max keep alive [" + MAX_KEEPALIVE_SETTING.getKey() + "], " +
                 "was (" + defaultKeepAlive + " > " + maxKeepAlive + ")");
         }
@@ -316,33 +316,27 @@ protected void doClose() {
 
     public void executeDfsPhase(ShardSearchRequest request, boolean keepStatesInContext,
                                 SearchShardTask task, ActionListener<SearchPhaseResult> listener) {
-        rewriteShardRequest(request, ActionListener.wrap(
-            // fork the execution in the search thread pool and wraps the searcher
-            // to execute the query
-            context -> {
-                try {
-                    context.wrapSearcher().execute(() -> {
-                        final SearchPhaseResult result;
-                        try {
-                            result = executeDfsPhase(context, keepStatesInContext, task);
-                        } catch (Exception exc) {
-                            listener.onFailure(exc);
-                            return;
-                        }
-                        listener.onResponse(result);
-                    });
-                } catch (Exception exc) {
-                    // if the execution is rejected we need to close the searcher
-                    IOUtils.closeWhileHandlingException(context.searcher);
-                    listener.onFailure(exc);
-                }
-            }, listener::onFailure));
+        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
+        IndexShard shard = indexService.getShard(request.shardId().id());
+        rewriteAndFetchShardRequest(shard, request, new ActionListener<ShardSearchRequest>() {
+            @Override
+            public void onResponse(ShardSearchRequest rewritten) {
+                // fork the execution in the search thread pool
+                runAsync(shard, () -> executeDfsPhase(request, task, keepStatesInContext), listener);
+            }
+
+            @Override
+            public void onFailure(Exception exc) {
+                listener.onFailure(exc);
+            }
+        });
     }
 
-    private DfsSearchResult executeDfsPhase(SearchRewriteContext rewriteContext, boolean keepStatesInContext, SearchShardTask task)
-        throws IOException {
-        final ReaderContext reader = createAndPutReaderContext(rewriteContext, keepStatesInContext);
-        try (SearchContext context = createContext(reader, rewriteContext.request, task, true)) {
+    private DfsSearchResult executeDfsPhase(ShardSearchRequest request,
+                                            SearchShardTask task,
+                                            boolean keepStatesInContext) throws IOException {
+        ReaderContext reader = createOrGetReaderContext(request, keepStatesInContext);
+        try (SearchContext context = createContext(reader, request, task, true)) {
             dfsPhase.execute(context);
             return context.dfsResult();
         } catch (Exception e) {
@@ -369,66 +363,91 @@ public void executeQueryPhase(ShardSearchRequest request, boolean keepStatesInCo
                                   SearchShardTask task, ActionListener<SearchPhaseResult> listener) {
         assert request.canReturnNullResponseIfMatchNoDocs() == false || request.numberOfShards() > 1
             : "empty responses require more than one shard";
-        rewriteShardRequest(request, ActionListener.wrap(
-            context -> {
-                try {
-                    ShardSearchRequest rewritten = context.request;
-                    if (rewritten.canReturnNullResponseIfMatchNoDocs()
-                            && canRewriteToMatchNone(rewritten.source())
-                            && rewritten.source().query() instanceof MatchNoneQueryBuilder) {
-                        assert request.scroll() == null : "must always create search context for scroll requests";
-                        onMatchNoDocs(context, listener);
-                    } else {
-                        // fork the execution in the search thread pool and wraps the searcher
-                        // to execute the query
-                        context.wrapSearcher().execute(() -> {
-                            final SearchPhaseResult result;
-                            try {
-                                result = executeQueryPhase(context, keepStatesInContext, task);
-                            } catch (Exception exc) {
-                                listener.onFailure(exc);
+        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
+        IndexShard shard = indexService.getShard(request.shardId().id());
+        rewriteAndFetchShardRequest(shard, request, new ActionListener<ShardSearchRequest>() {
+            @Override
+            public void onResponse(ShardSearchRequest orig) {
+                if (orig.canReturnNullResponseIfMatchNoDocs()) {
+                    assert orig.shouldCreatePersistentReader() == false;
+                    // we clone the shard request and perform a quick rewrite using a lightweight
+                    // searcher since we are outside of the search thread pool.
+                    // If the request rewrites to "match none" we can shortcut the query phase
+                    // entirely. Otherwise we fork the execution in the search thread pool.
+                    ShardSearchRequest canMatchRequest = new ShardSearchRequest(orig);
+                    try (Engine.Searcher searcher = shard.acquireCanMatchSearcher()) {
+                        QueryShardContext context = indexService.newQueryShardContext(canMatchRequest.shardId().id(), searcher,
+                            canMatchRequest::nowInMillis, canMatchRequest.getClusterAlias());
+                        Rewriteable.rewrite(canMatchRequest.getRewriteable(), context, true);
+                    } catch (Exception exc) {
+                        listener.onFailure(exc);
+                        return;
+                    }
+                    if (canRewriteToMatchNone(canMatchRequest.source())
+                            && canMatchRequest.source().query() instanceof MatchNoneQueryBuilder) {
+                        if (canMatchRequest.readerId() != null) {
+                            ReaderContext readerContext = getReaderContext(canMatchRequest.readerId());
+                            if (readerContext == null) {
+                                listener.onFailure(new SearchContextMissingException(canMatchRequest.readerId()));
                                 return;
                             }
-                            listener.onResponse(result);
-                        });
+                            try (Releasable r = readerContext.markAsUsed()) {
+                                listener.onResponse(QuerySearchResult.nullInstance());
+                            }
+                        } else {
+                            listener.onResponse(QuerySearchResult.nullInstance());
+                        }
+                        return;
                     }
-                } catch (Exception exc) {
-                    // if the execution is rejected we need to close the searcher
-                    IOUtils.closeWhileHandlingException(context.searcher);
-                    listener.onFailure(exc);
                 }
-            }, listener::onFailure));
+                // fork the execution in the search thread pool
+                runAsync(shard, () -> executeQueryPhase(orig, task, keepStatesInContext), listener);
+            }
+
+            @Override
+            public void onFailure(Exception exc) {
+                listener.onFailure(exc);
+            }
+        });
     }
 
-    private void onMatchNoDocs(SearchRewriteContext rewriteContext, ActionListener<SearchPhaseResult> listener) {
+    private <T> void runAsync(IndexShard shard, CheckedSupplier<T, Exception> command, ActionListener<T> listener) {
+        Executor executor = getExecutor(shard);
         try {
-            rewriteContext.searcher.close();
+            executor.execute(() -> {
+                T result;
+                try {
+                    result = command.get();
+                } catch (Exception exc) {
+                    listener.onFailure(exc);
+                    return;
+                }
+                listener.onResponse(result);
+            });
         } catch (Exception exc) {
             listener.onFailure(exc);
-            return;
         }
-        listener.onResponse(QuerySearchResult.nullInstance());
     }
 
     private <T> void runAsync(SearchContextId id, CheckedSupplier<T, IOException> executable, ActionListener<T> listener) {
         getExecutor(id).execute(ActionRunnable.supply(listener, executable::get));
     }
 
-    private SearchPhaseResult executeQueryPhase(SearchRewriteContext rewriteContext, boolean keepStatesInContext,
-                                                SearchShardTask task) throws Exception {
-        final ReaderContext reader = createAndPutReaderContext(rewriteContext, keepStatesInContext);
-        final ShardSearchRequest request = rewriteContext.request;
+    private SearchPhaseResult executeQueryPhase(ShardSearchRequest request,
+                                                SearchShardTask task,
+                                                boolean keepStatesInContext) throws Exception {
+        ReaderContext reader = createOrGetReaderContext(request, keepStatesInContext);
         try (SearchContext context = createContext(reader, request, task, true)) {
             final long afterQueryTime;
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
                 loadOrExecuteQueryPhase(request, context);
-                if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) {
+                if (context.queryResult().hasSearchContext() == false && reader.singleSession()) {
                     freeReaderContext(reader.id());
                 }
                 afterQueryTime = executor.success();
             }
             if (request.numberOfShards() == 1) {
-                return executeFetchPhase(context, afterQueryTime);
+                return executeFetchPhase(reader, context, afterQueryTime);
             }
             return context.queryResult();
         } catch (Exception e) {
@@ -443,12 +462,12 @@ private SearchPhaseResult executeQueryPhase(SearchRewriteContext rewriteContext,
         }
     }
 
-    private QueryFetchSearchResult executeFetchPhase(SearchContext context, long afterQueryTime) {
+    private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchContext context, long afterQueryTime) {
         try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, afterQueryTime)){
             shortcutDocIdsToLoad(context);
             fetchPhase.execute(context);
-            if (fetchPhaseShouldFreeContext(context)) {
-                freeReaderContext(context.id());
+            if (reader.singleSession()) {
+                freeReaderContext(reader.id());
             }
             executor.success();
         }
@@ -462,6 +481,10 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
             final LegacyReaderContext reader = (LegacyReaderContext) findReaderContext(request.contextId());
             try (SearchContext context = createContext(reader, reader.getShardSearchRequest(null), task, false);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                if (request.scroll() != null && request.scroll().keepAlive() != null) {
+                    checkKeepAliveLimit(request.scroll().keepAlive().millis());
+                    reader.keepAlive(request.scroll().keepAlive().millis());
+                }
                 reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 context.searcher().setAggregatedDfs(reader.getAggregatedDfs(null));
                 processScroll(request, reader, context);
@@ -488,7 +511,7 @@ public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task,
                 reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 context.searcher().setAggregatedDfs(request.dfs());
                 queryPhase.execute(context);
-                if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) {
+                if (context.queryResult().hasSearchContext() == false && reader.singleSession()) {
                     // no hits, we can release the context since there will be no fetch phase
                     freeReaderContext(reader.id());
                 }
@@ -505,16 +528,6 @@ public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task,
         }, listener);
     }
 
-    private boolean fetchPhaseShouldFreeContext(SearchContext context) {
-        if (context.scrollContext() == null) {
-            // simple search, no scroll
-            return true;
-        } else {
-            // scroll request, but the scroll was not extended
-            return context.scrollContext().scroll == null;
-        }
-    }
-
     final Executor getExecutor(SearchContextId id) {
         return getExecutor(findReaderContext(id).indexShard());
     }
@@ -530,13 +543,17 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
             final LegacyReaderContext reader = (LegacyReaderContext) findReaderContext(request.contextId());
             try (SearchContext context = createContext(reader, reader.getShardSearchRequest(null), task, false);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+                if (request.scroll() != null && request.scroll().keepAlive() != null) {
+                    checkKeepAliveLimit(request.scroll().keepAlive().millis());
+                    reader.keepAlive(request.scroll().keepAlive().millis());
+                }
                 context.assignRescoreDocIds(reader.getRescoreDocIds(null));
                 context.searcher().setAggregatedDfs(reader.getAggregatedDfs(null));
                 reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 processScroll(request, reader, context);
                 queryPhase.execute(context);
                 final long afterQueryTime = executor.success();
-                QueryFetchSearchResult fetchSearchResult = executeFetchPhase(context, afterQueryTime);
+                QueryFetchSearchResult fetchSearchResult = executeFetchPhase(reader, context, afterQueryTime);
                 return new ScrollQueryFetchSearchResult(fetchSearchResult, context.shardTarget());
             } catch (Exception e) {
                 logger.trace("Fetch phase failed", e);
@@ -560,7 +577,7 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
                 context.docIdsToLoad(request.docIds(), 0, request.docIdsSize());
                 try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, System.nanoTime())) {
                     fetchPhase.execute(context);
-                    if (fetchPhaseShouldFreeContext(context)) {
+                    if (reader.singleSession()) {
                         freeReaderContext(request.contextId());
                     }
                     executor.success();
@@ -593,10 +610,21 @@ private ReaderContext findReaderContext(SearchContextId id) throws SearchContext
         return reader;
     }
 
-    final ReaderContext createAndPutReaderContext(SearchRewriteContext rewriteContext, boolean keepStatesInContext) {
-        final ShardSearchRequest request = rewriteContext.request;
-        final SearchOperationListener searchOperationListener = rewriteContext.shard.getSearchOperationListener();
-        Engine.Searcher engineSearcher = rewriteContext.searcher;
+    final ReaderContext createOrGetReaderContext(ShardSearchRequest request, boolean keepStatesInContext) {
+        if (request.readerId() != null) {
+            ReaderContext readerContext = findReaderContext(request.readerId());
+            readerContext.keepAlive(getKeepAlive(request));
+            checkKeepAliveLimit(request.scroll().keepAlive().millis());
+            return readerContext;
+        }
+        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
+        IndexShard shard = indexService.getShard(request.shardId().id());
+        Engine.Searcher searcher = shard.acquireSearcher("search");
+        return createAndPutReaderContext(request, shard, searcher, keepStatesInContext);
+    }
+
+    final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexShard shard,
+                                                  Engine.Searcher engineSearcher, boolean keepStatesInContext) {
         ReaderContext readerContext = null;
         Releasable decreaseScrollContexts = null;
         try {
@@ -609,17 +637,21 @@ final ReaderContext createAndPutReaderContext(SearchRewriteContext rewriteContex
                             + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
                 }
             }
+            final long keepAlive = getKeepAlive(request);
+            checkKeepAliveLimit(keepAlive);
             if (keepStatesInContext || request.scroll() != null) {
-                readerContext = new LegacyReaderContext(idGenerator.incrementAndGet(), rewriteContext.shard, engineSearcher, request);
+                readerContext = new LegacyReaderContext(idGenerator.incrementAndGet(), shard, engineSearcher, request, keepAlive);
+                if (request.scroll() != null) {
+                    readerContext.addOnClose(decreaseScrollContexts);
+                    decreaseScrollContexts = null;
+                }
             } else {
-                readerContext = new ReaderContext(idGenerator.incrementAndGet(), rewriteContext.shard, engineSearcher);
-            }
-            if (request.scroll() != null) {
-                readerContext.addOnClose(decreaseScrollContexts);
-                decreaseScrollContexts = null;
+                readerContext = new ReaderContext(idGenerator.incrementAndGet(), shard, engineSearcher, keepAlive,
+                    request.keepAlive() == null);
             }
             engineSearcher = null;
             final ReaderContext finalReaderContext = readerContext;
+            final SearchOperationListener searchOperationListener = shard.getSearchOperationListener();
             searchOperationListener.onNewReaderContext(finalReaderContext);
             if (finalReaderContext.scrollContext() != null) {
                 searchOperationListener.onNewScrollContext(finalReaderContext.scrollContext());
@@ -641,8 +673,10 @@ final ReaderContext createAndPutReaderContext(SearchRewriteContext rewriteContex
         }
     }
 
-    final SearchContext createContext(ReaderContext reader, ShardSearchRequest request, SearchShardTask task, boolean includeAggregations)
-        throws IOException {
+    final SearchContext createContext(ReaderContext reader,
+                                      ShardSearchRequest request,
+                                      SearchShardTask task,
+                                      boolean includeAggregations) throws IOException {
         final DefaultSearchContext context = createSearchContext(reader, request, defaultSearchTimeout);
         try {
             context.setTask(task);
@@ -663,17 +697,8 @@ final SearchContext createContext(ReaderContext reader, ShardSearchRequest reque
             dfsPhase.preProcess(context);
             queryPhase.preProcess(context);
             fetchPhase.preProcess(context);
-
-            // compute the context keep alive
-            long keepAlive = defaultKeepAlive;
-            if (request.scroll() != null && request.scroll().keepAlive() != null) {
-                keepAlive = request.scroll().keepAlive().millis();
-            }
-            contextScrollKeepAlive(reader, keepAlive);
-            context.lowLevelCancellation(lowLevelCancellation);
             // Disable timeout while searching and mark the reader as accessed when releasing the context.
-            context.addReleasable(() -> reader.accessed(threadPool.relativeTimeInMillis()));
-            reader.accessed(-1L);
+            context.addReleasable(reader.markAsUsed());
             context.addReleasable(() -> reader.indexShard().getSearchOperationListener().onFreeSearchContext(context));
         } catch (Exception e) {
             context.close();
@@ -686,7 +711,7 @@ final SearchContext createContext(ReaderContext reader, ShardSearchRequest reque
     public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout) throws IOException {
         IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().getId());
         Engine.Searcher engineSearcher = indexShard.acquireSearcher("search");
-        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, engineSearcher)) {
+        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, engineSearcher, -1L, true)) {
             engineSearcher = null; // transfer ownership to readerContext
             return createSearchContext(readerContext, request, timeout);
         } finally {
@@ -704,8 +729,8 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
             IndexShard indexShard = indexService.getShard(request.shardId().getId());
             SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(),
                 indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE);
-            searchContext = new DefaultSearchContext(reader, request, shardTarget,
-                clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase);
+            searchContext = new DefaultSearchContext(reader, request, shardTarget, clusterService, indexService, indexShard,
+                bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase, lowLevelCancellation);
             assert searchContext.getQueryShardContext().isCacheable();
             success = true;
             return searchContext;
@@ -750,14 +775,23 @@ public void freeAllScrollContexts() {
         }
     }
 
-    private void contextScrollKeepAlive(ReaderContext readerContext, long keepAlive) {
+    private long getKeepAlive(ShardSearchRequest request) {
+        if (request.scroll() != null && request.scroll().keepAlive() != null) {
+            return request.scroll().keepAlive().millis();
+        } else if (request.keepAlive() != null) {
+            return request.keepAlive().millis();
+        } else {
+            return defaultKeepAlive;
+        }
+    }
+
+    private void checkKeepAliveLimit(long keepAlive) {
         if (keepAlive > maxKeepAlive) {
             throw new IllegalArgumentException(
-                "Keep alive for scroll (" + TimeValue.timeValueMillis(keepAlive) + ") is too large. " +
+                "Keep alive for request (" + TimeValue.timeValueMillis(keepAlive) + ") is too large. " +
                     "It must be less than (" + TimeValue.timeValueMillis(maxKeepAlive) + "). " +
                     "This limit can be set by changing the [" + MAX_KEEPALIVE_SETTING.getKey() + "] cluster level setting.");
         }
-        readerContext.keepAlive(keepAlive);
     }
 
     private void processFailure(ReaderContext context, Exception e) {
@@ -1008,10 +1042,6 @@ private void processScroll(InternalScrollSearchRequest request, ReaderContext re
         // process scroll
         context.from(context.from() + context.size());
         context.scrollContext().scroll = request.scroll();
-        // update the context keep alive based on the new scroll value
-        if (request.scroll() != null && request.scroll().keepAlive() != null) {
-            contextScrollKeepAlive(reader, request.scroll().keepAlive().millis());
-        }
     }
 
     /**
@@ -1029,17 +1059,9 @@ public ResponseCollectorService getResponseCollectorService() {
     class Reaper implements Runnable {
         @Override
         public void run() {
-            final long time = threadPool.relativeTimeInMillis();
             for (ReaderContext context : activeReaders.values()) {
-                // Use the same value for both checks since lastAccessTime can
-                // be modified by another thread between checks!
-                final long lastAccessTime = context.lastAccessTime();
-                if (lastAccessTime == -1L) { // its being processed or timeout is disabled
-                    continue;
-                }
-                if ((time - lastAccessTime > context.keepAlive())) {
-                    logger.debug("freeing search context [{}], time [{}], lastAccessTime [{}], keepAlive [{}]", context.id(), time,
-                        lastAccessTime, context.keepAlive());
+                if (context.isKeepAliveLapsed()) {
+                    logger.debug("freeing search context [{}]", context.id());
                     freeReaderContext(context.id());
                 }
             }
@@ -1061,7 +1083,7 @@ public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException
         IndexShard indexShard = indexService.getShard(request.shardId().getId());
         // we don't want to use the reader wrapper since it could run costly operations
         // and we can afford false positives.
-        try (Engine.Searcher searcher = indexShard.acquireSearcherNoWrap("can_match")) {
+        try (Engine.Searcher searcher = indexShard.acquireCanMatchSearcher()) {
             QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), searcher,
                 request::nowInMillis, request.getClusterAlias());
             Rewriteable.rewrite(request.getRewriteable(), context, false);
@@ -1097,50 +1119,17 @@ public static boolean canRewriteToMatchNone(SearchSourceBuilder source) {
         return aggregations == null || aggregations.mustVisitAllDocs() == false;
     }
 
-    /*
-     * Rewrites the search request with a light weight rewrite context in order to fetch resources asynchronously
-     * and then rewrites with a searcher when the shard is active.
-     * The provided action listener is executed on the same thread or in a listener threadpool.
-     */
-    private void rewriteShardRequest(ShardSearchRequest request, ActionListener<SearchRewriteContext> listener) {
-        IndexShard shard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id());
+    private void rewriteAndFetchShardRequest(IndexShard shard, ShardSearchRequest request, ActionListener<ShardSearchRequest> listener) {
         ActionListener<Rewriteable> actionListener = ActionListener.wrap(r ->
                 // now we need to check if there is a pending refresh and register
-                shard.awaitShardSearchActive(b -> {
-                    try {
-                        // we can now acquire a searcher and rewrite the request with it
-                        SearchRewriteContext rewriteContext = acquireSearcherAndRewrite(request, shard);
-                        listener.onResponse(rewriteContext);
-                    } catch (Exception e) {
-                        listener.onFailure(e);
-                    }
-                }), listener::onFailure);
+                shard.awaitShardSearchActive(b -> listener.onResponse(request)),
+            listener::onFailure);
         // we also do rewrite on the coordinating node (TransportSearchService) but we also need to do it here for BWC as well as
         // AliasFilters that might need to be rewritten. These are edge-cases but we are every efficient doing the rewrite here so it's not
         // adding a lot of overhead
         Rewriteable.rewriteAndFetch(request.getRewriteable(), indicesService.getRewriteContext(request::nowInMillis), actionListener);
     }
 
-    SearchRewriteContext acquireSearcherAndRewrite(ShardSearchRequest request, IndexShard shard) throws IOException {
-        // acquire the searcher for rewrite with no wrapping in order to avoid costly
-        // operations. We'll wrap the searcher at a later stage (when executing the query).
-        Engine.Searcher searcher = shard.acquireSearcherNoWrap("search");
-        boolean success = false;
-        try {
-            IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-            QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), searcher,
-                request::nowInMillis, request.getClusterAlias());
-            Rewriteable.rewrite(request.getRewriteable(), context, true);
-            SearchRewriteContext rewrite = new SearchRewriteContext(request, shard, searcher, getExecutor(shard));
-            success = true;
-            return rewrite;
-        } finally {
-            if (success == false) {
-                IOUtils.closeWhileHandlingException(searcher);
-            }
-        }
-    }
-
     /**
      * Returns a new {@link QueryRewriteContext} with the given {@code now} provider
      */
@@ -1160,7 +1149,8 @@ public InternalAggregation.ReduceContextBuilder aggReduceContextBuilder(SearchRe
         return new InternalAggregation.ReduceContextBuilder() {
             @Override
             public InternalAggregation.ReduceContext forPartialReduction() {
-                return InternalAggregation.ReduceContext.forPartialReduction(bigArrays, scriptService);
+                return InternalAggregation.ReduceContext.forPartialReduction(bigArrays, scriptService,
+                        () -> requestToPipelineTree(request));
             }
 
             @Override
@@ -1179,37 +1169,6 @@ private static PipelineTree requestToPipelineTree(SearchRequest request) {
         return request.source().aggregations().buildPipelineTree();
     }
 
-    static class SearchRewriteContext {
-        private final ShardSearchRequest request;
-        private final IndexShard shard;
-        private Engine.Searcher searcher;
-        private final Executor executor;
-
-        private boolean isWrapped;
-
-        private SearchRewriteContext(ShardSearchRequest request,
-                                     IndexShard shard,
-                                     Engine.Searcher searcher,
-                                     Executor executor) {
-            this.request = request;
-            this.shard = shard;
-            this.searcher = searcher;
-            this.executor = executor;
-        }
-
-        SearchRewriteContext wrapSearcher() {
-            assert isWrapped == false : "searcher already wrapped";
-            isWrapped = true;
-            searcher = shard.wrapSearcher(searcher);
-            return this;
-        }
-
-        void execute(Runnable runnable) {
-            assert isWrapped : "searcher is not wrapped";
-            executor.execute(runnable);
-        }
-    }
-
     public static final class CanMatchResponse extends SearchPhaseResult {
         private final boolean canMatch;
         private final MinAndMax<?> minAndMax;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
index fca5b14e34be3..f1e8281fb8b51 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
@@ -131,17 +131,16 @@ public void execute(SearchContext context) {
             }
         }
         List<PipelineAggregator> pipelineAggregators = context.aggregations().factories().createPipelineAggregators();
-        List<SiblingPipelineAggregator> siblingPipelineAggregators = new ArrayList<>(pipelineAggregators.size());
         for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
-            if (pipelineAggregator instanceof SiblingPipelineAggregator) {
-                siblingPipelineAggregators.add((SiblingPipelineAggregator) pipelineAggregator);
-            } else {
+            if (false == pipelineAggregator instanceof SiblingPipelineAggregator) {
+                // TODO move this to request validation after #53669
                 throw new AggregationExecutionException("Invalid pipeline aggregation named [" + pipelineAggregator.name()
                     + "] of type [" + pipelineAggregator.getWriteableName() + "]. Only sibling pipeline aggregations are "
                     + "allowed at the top level");
             }
         }
-        context.queryResult().aggregations(new InternalAggregations(aggregations, siblingPipelineAggregators));
+        context.queryResult().aggregations(new InternalAggregations(aggregations,
+                context.request().source().aggregations()::buildPipelineTree));
 
         // disable aggregations so that they don't run on next pages in case of scrolling
         context.aggregations(null);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index 1998a53ad06b0..397c6ea762cf1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -283,8 +284,6 @@ public boolean mustVisitAllDocs() {
             return false;
         }
 
-
-
         public Builder addAggregator(AggregationBuilder factory) {
             if (!names.add(factory.name)) {
                 throw new IllegalArgumentException("Two sibling aggregations cannot have the same name: [" + factory.name + "]");
@@ -298,15 +297,51 @@ public Builder addPipelineAggregator(PipelineAggregationBuilder pipelineAggregat
             return this;
         }
 
+        /**
+         * Validate the root of the aggregation tree.
+         */
+        public ActionRequestValidationException validate(ActionRequestValidationException e) {
+            PipelineAggregationBuilder.ValidationContext context =
+                    PipelineAggregationBuilder.ValidationContext.forTreeRoot(aggregationBuilders, pipelineAggregatorBuilders, e);
+            validatePipelines(context);
+            return validateChildren(context.getValidationException());
+        }
+
+        /**
+         * Validate a the pipeline aggregations in this factory.
+         */
+        private void validatePipelines(PipelineAggregationBuilder.ValidationContext context) {
+            List<PipelineAggregationBuilder> orderedPipelineAggregators;
+            try {
+                orderedPipelineAggregators = resolvePipelineAggregatorOrder(pipelineAggregatorBuilders, aggregationBuilders);
+            } catch (IllegalArgumentException iae) {
+                context.addValidationError(iae.getMessage());
+                return;
+            }
+            for (PipelineAggregationBuilder builder : orderedPipelineAggregators) {
+                builder.validate(context);
+            }
+        }
+
+        /**
+         * Validate a the children of this factory.
+         */
+        private ActionRequestValidationException validateChildren(ActionRequestValidationException e) {
+            for (AggregationBuilder agg : aggregationBuilders) {
+                PipelineAggregationBuilder.ValidationContext context =
+                        PipelineAggregationBuilder.ValidationContext.forInsideTree(agg, e);
+                agg.factoriesBuilder.validatePipelines(context);
+                e = agg.factoriesBuilder.validateChildren(context.getValidationException());
+            }
+            return e;
+        }
+
         public AggregatorFactories build(QueryShardContext queryShardContext, AggregatorFactory parent) throws IOException {
             if (aggregationBuilders.isEmpty() && pipelineAggregatorBuilders.isEmpty()) {
                 return EMPTY;
             }
-            List<PipelineAggregationBuilder> orderedpipelineAggregators = null;
-            orderedpipelineAggregators = resolvePipelineAggregatorOrder(this.pipelineAggregatorBuilders, this.aggregationBuilders);
-            for (PipelineAggregationBuilder builder : orderedpipelineAggregators) {
-                builder.validate(parent, aggregationBuilders, pipelineAggregatorBuilders);
-            }
+            List<PipelineAggregationBuilder> orderedPipelineAggregators =
+                    resolvePipelineAggregatorOrder(pipelineAggregatorBuilders, aggregationBuilders);
             AggregatorFactory[] aggFactories = new AggregatorFactory[aggregationBuilders.size()];
 
             int i = 0;
@@ -314,7 +349,7 @@ public AggregatorFactories build(QueryShardContext queryShardContext, Aggregator
                 aggFactories[i] = agg.build(queryShardContext, parent);
                 ++i;
             }
-            return new AggregatorFactories(aggFactories, orderedpipelineAggregators);
+            return new AggregatorFactories(aggFactories, orderedPipelineAggregators);
         }
 
         private List<PipelineAggregationBuilder> resolvePipelineAggregatorOrder(
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index 0015756d4d933..cffc610900458 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -37,7 +38,9 @@
 import java.util.Objects;
 import java.util.function.Function;
 import java.util.function.IntConsumer;
+import java.util.function.Supplier;
 
+import static java.util.Collections.emptyList;
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -62,12 +65,19 @@ public static class ReduceContext {
         private final ScriptService scriptService;
         private final IntConsumer multiBucketConsumer;
         private final PipelineTree pipelineTreeRoot;
+        /**
+         * Supplies the pipelines when the result of the reduce is serialized
+         * to node versions that need pipeline aggregators to be serialized
+         * to them.
+         */
+        private final Supplier<PipelineTree> pipelineTreeForBwcSerialization;
 
         /**
          * Build a {@linkplain ReduceContext} to perform a partial reduction.
          */
-        public static ReduceContext forPartialReduction(BigArrays bigArrays, ScriptService scriptService) {
-            return new ReduceContext(bigArrays, scriptService, (s) -> {}, null);
+        public static ReduceContext forPartialReduction(BigArrays bigArrays, ScriptService scriptService,
+                Supplier<PipelineTree> pipelineTreeForBwcSerialization) {
+            return new ReduceContext(bigArrays, scriptService, (s) -> {}, null, pipelineTreeForBwcSerialization);
         }
 
         /**
@@ -77,15 +87,16 @@ public static ReduceContext forPartialReduction(BigArrays bigArrays, ScriptServi
         public static ReduceContext forFinalReduction(BigArrays bigArrays, ScriptService scriptService,
                 IntConsumer multiBucketConsumer, PipelineTree pipelineTreeRoot) {
             return new ReduceContext(bigArrays, scriptService, multiBucketConsumer,
-                    requireNonNull(pipelineTreeRoot, "prefer EMPTY to null"));
+                    requireNonNull(pipelineTreeRoot, "prefer EMPTY to null"), () -> pipelineTreeRoot);
         }
 
         private ReduceContext(BigArrays bigArrays, ScriptService scriptService, IntConsumer multiBucketConsumer,
-                PipelineTree pipelineTreeRoot) {
+                PipelineTree pipelineTreeRoot, Supplier<PipelineTree> pipelineTreeForBwcSerialization) {
             this.bigArrays = bigArrays;
             this.scriptService = scriptService;
             this.multiBucketConsumer = multiBucketConsumer;
             this.pipelineTreeRoot = pipelineTreeRoot;
+            this.pipelineTreeForBwcSerialization = pipelineTreeForBwcSerialization;
         }
 
         /**
@@ -112,6 +123,15 @@ public PipelineTree pipelineTreeRoot() {
             return pipelineTreeRoot;
         }
 
+        /**
+         * Supplies the pipelines when the result of the reduce is serialized
+         * to node versions that need pipeline aggregators to be serialized
+         * to them.
+         */
+        public Supplier<PipelineTree> pipelineTreeForBwcSerialization() {
+            return pipelineTreeForBwcSerialization;
+        }
+
         /**
          * Adds {@code count} buckets to the global count for the request and fails if this number is greater than
          * the maximum number of buckets allowed in a response
@@ -129,9 +149,9 @@ public void consumeBucketsAndMaybeBreak(int size) {
     private final List<PipelineAggregator> pipelineAggregators;
 
     /**
-     * Constructs an get with a given name.
+     * Constructs an aggregation result with a given name.
      *
-     * @param name The name of the get.
+     * @param name The name of the aggregation.
      */
     protected InternalAggregation(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         this.name = name;
@@ -145,14 +165,20 @@ protected InternalAggregation(String name, List<PipelineAggregator> pipelineAggr
     protected InternalAggregation(StreamInput in) throws IOException {
         name = in.readString();
         metaData = in.readMap();
-        pipelineAggregators = in.readNamedWriteableList(PipelineAggregator.class);
+        if (in.getVersion().before(Version.V_7_8_0)) {
+            pipelineAggregators = in.readNamedWriteableList(PipelineAggregator.class);
+        } else {
+            pipelineAggregators = emptyList();
+        }
     }
 
     @Override
     public final void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeGenericValue(metaData);
-        out.writeNamedWriteableList(pipelineAggregators);
+        if (out.getVersion().before(Version.V_7_8_0)) {
+            out.writeNamedWriteableList(pipelineAggregators);
+        }
         doWriteTo(out);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
index f763ececfaf36..c3b5d5ee70038 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -34,9 +35,13 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
+import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.toList;
+
 /**
  * An internal implementation of {@link Aggregations}.
  */
@@ -54,35 +59,57 @@ public final class InternalAggregations extends Aggregations implements Writeabl
         }
     };
 
-    private final List<SiblingPipelineAggregator> topLevelPipelineAggregators;
+    /**
+     * The way to build a tree of pipeline aggregators. Used only for
+     * serialization backwards compatibility.
+     */
+    private final Supplier<PipelineAggregator.PipelineTree> pipelineTreeForBwcSerialization;
 
     /**
      * Constructs a new aggregation.
      */
     public InternalAggregations(List<InternalAggregation> aggregations) {
         super(aggregations);
-        this.topLevelPipelineAggregators = Collections.emptyList();
+        this.pipelineTreeForBwcSerialization = null;
     }
 
     /**
-     * Constructs a new aggregation providing its {@link InternalAggregation}s and {@link SiblingPipelineAggregator}s
+     * Constructs a node in the aggregation tree.
+     * @param pipelineTreeSource must be null inside the tree or after final reduction. Should reference the
+     *                           search request otherwise so we can properly serialize the response to
+     *                           versions of Elasticsearch that require the pipelines to be serialized.
      */
-    public InternalAggregations(List<InternalAggregation> aggregations, List<SiblingPipelineAggregator> topLevelPipelineAggregators) {
+    public InternalAggregations(List<InternalAggregation> aggregations, Supplier<PipelineAggregator.PipelineTree> pipelineTreeSource) {
         super(aggregations);
-        this.topLevelPipelineAggregators = Objects.requireNonNull(topLevelPipelineAggregators);
+        this.pipelineTreeForBwcSerialization = pipelineTreeSource;
     }
 
     public InternalAggregations(StreamInput in) throws IOException {
         super(in.readList(stream -> in.readNamedWriteable(InternalAggregation.class)));
-        this.topLevelPipelineAggregators = in.readList(
-            stream -> (SiblingPipelineAggregator)in.readNamedWriteable(PipelineAggregator.class));
+        if (in.getVersion().before(Version.V_7_8_0)) {
+            in.readNamedWriteableList(PipelineAggregator.class); 
+        }
+        /*
+         * Setting the pipeline tree source to null is here is correct but
+         * only because we don't immediately pass the InternalAggregations
+         * off to another node. Instead, we always reduce together with
+         * many aggregations and that always adds the tree read from the
+         * current request.
+         */
+        pipelineTreeForBwcSerialization = null;
     }
 
     @Override
     @SuppressWarnings("unchecked")
     public void writeTo(StreamOutput out) throws IOException {
         out.writeNamedWriteableList((List<InternalAggregation>)aggregations);
-        out.writeNamedWriteableList(topLevelPipelineAggregators);
+        if (out.getVersion().before(Version.V_7_8_0)) {
+            if (pipelineTreeForBwcSerialization == null) {
+                out.writeNamedWriteableList(emptyList());
+            } else {
+                out.writeNamedWriteableList(pipelineTreeForBwcSerialization.get().aggregators());
+            }
+        }
     }
 
     /**
@@ -95,12 +122,17 @@ public List<InternalAggregation> copyResults() {
     }
 
     /**
-     * Returns the top-level pipeline aggregators.
-     * Note that top-level pipeline aggregators become normal aggregation once the final reduction has been performed, after which they
-     * become part of the list of {@link InternalAggregation}s.
+     * Get the top level pipeline aggregators.
+     * @deprecated these only exist for BWC serialization
      */
+    @Deprecated
     public List<SiblingPipelineAggregator> getTopLevelPipelineAggregators() {
-        return topLevelPipelineAggregators;
+        if (pipelineTreeForBwcSerialization == null) {
+            return emptyList();
+        }
+        return pipelineTreeForBwcSerialization.get().aggregators().stream()
+                .map(p -> (SiblingPipelineAggregator) p)
+                .collect(toList());
     }
 
     @SuppressWarnings("unchecked")
@@ -131,7 +163,8 @@ public double sortValue(AggregationPath.PathElement head, Iterator<AggregationPa
      * aggregations (both embedded parent/sibling as well as top-level sibling pipelines)
      */
     public static InternalAggregations topLevelReduce(List<InternalAggregations> aggregationsList, ReduceContext context) {
-        InternalAggregations reduced = reduce(aggregationsList, context);
+        InternalAggregations reduced = reduce(aggregationsList, context,
+                reducedAggregations -> new InternalAggregations(reducedAggregations, context.pipelineTreeForBwcSerialization()));
         if (reduced == null) {
             return null;
         }
@@ -157,12 +190,16 @@ public static InternalAggregations topLevelReduce(List<InternalAggregations> agg
      * {@link InternalAggregations} object found in the list.
      * Note that pipeline aggregations _are not_ reduced by this method.  Pipelines are handled
      * separately by {@link InternalAggregations#topLevelReduce(List, ReduceContext)}
+     * @param ctor used to build the {@link InternalAggregations}. The top level reduce specifies a constructor
+     *            that adds pipeline aggregation information that is used to send pipeline aggregations to
+     *            older versions of Elasticsearch that require the pipeline aggregations to be returned
+     *            as part of the aggregation tree
      */
-    public static InternalAggregations reduce(List<InternalAggregations> aggregationsList, ReduceContext context) {
+    public static InternalAggregations reduce(List<InternalAggregations> aggregationsList, ReduceContext context,
+            Function<List<InternalAggregation>, InternalAggregations> ctor) {
         if (aggregationsList.isEmpty()) {
             return null;
         }
-        List<SiblingPipelineAggregator> topLevelPipelineAggregators = aggregationsList.get(0).getTopLevelPipelineAggregators();
 
         // first we collect all aggregations of the same type and list them together
         Map<String, List<InternalAggregation>> aggByName = new HashMap<>();
@@ -185,6 +222,14 @@ public static InternalAggregations reduce(List<InternalAggregations> aggregation
             reducedAggregations.add(first.reduce(aggregations, context));
         }
 
-        return new InternalAggregations(reducedAggregations, topLevelPipelineAggregators);
+        return ctor.apply(reducedAggregations);
     }
+
+    /**
+     * Version of {@link #reduce(List, ReduceContext, Function)} for nodes inside the aggregation tree.
+     */
+    public static InternalAggregations reduce(List<InternalAggregations> aggregationsList, ReduceContext context) {
+        return reduce(aggregationsList, context, InternalAggregations::new);
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
index 85b9a7dcb0ca5..aaffcfc440bce 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
@@ -18,14 +18,20 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.util.Collection;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * A factory that knows how to create an {@link PipelineAggregator} of a
@@ -64,11 +70,145 @@ public final String[] getBucketsPaths() {
     }
 
     /**
-     * Internal: Validates the state of this factory (makes sure the factory is properly
-     * configured)
+     * Makes sure this builder is properly configured.
      */
-    protected abstract void validate(AggregatorFactory parent, Collection<AggregationBuilder> aggregationBuilders,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorBuilders);
+    protected abstract void validate(ValidationContext context);
+    public abstract static class ValidationContext {
+        /**
+         * Build the context for the root of the aggregation tree.
+         */
+        public static ValidationContext forTreeRoot(Collection<AggregationBuilder> siblingAggregations,
+                Collection<PipelineAggregationBuilder> siblingPipelineAggregations,
+                ActionRequestValidationException validationFailuresSoFar) {
+            return new ForTreeRoot(siblingAggregations, siblingPipelineAggregations, validationFailuresSoFar);
+        }
+
+        /**
+         * Build the context for a node inside the aggregation tree.
+         */
+        public static ValidationContext forInsideTree(AggregationBuilder parent,
+                ActionRequestValidationException validationFailuresSoFar) {
+            return new ForInsideTree(parent, validationFailuresSoFar);
+        }
+
+
+        private ActionRequestValidationException e;
+
+        private ValidationContext(ActionRequestValidationException validationFailuresSoFar) {
+            this.e = validationFailuresSoFar;
+        }
+
+        private static class ForTreeRoot extends ValidationContext {
+            private final Collection<AggregationBuilder> siblingAggregations;
+            private final Collection<PipelineAggregationBuilder> siblingPipelineAggregations;
+
+            ForTreeRoot(Collection<AggregationBuilder> siblingAggregations,
+                    Collection<PipelineAggregationBuilder> siblingPipelineAggregations,
+                    ActionRequestValidationException validationFailuresSoFar) {
+                super(validationFailuresSoFar);
+                this.siblingAggregations = Objects.requireNonNull(siblingAggregations);
+                this.siblingPipelineAggregations = Objects.requireNonNull(siblingPipelineAggregations);
+            }
+
+            @Override
+            public Collection<AggregationBuilder> getSiblingAggregations() {
+                return siblingAggregations;
+            }
+
+            @Override
+            public Collection<PipelineAggregationBuilder> getSiblingPipelineAggregations() {
+                return siblingPipelineAggregations;
+            }
+
+            @Override
+            public void validateParentAggSequentiallyOrdered(String type, String name) {
+                addValidationError(type + " aggregation [" + name
+                        + "] must have a histogram, date_histogram or auto_date_histogram as parent but doesn't have a parent");
+            }
+        }
+
+        private static class ForInsideTree extends ValidationContext {
+            private final AggregationBuilder parent;
+
+            ForInsideTree(AggregationBuilder parent, ActionRequestValidationException validationFailuresSoFar) {
+                super(validationFailuresSoFar);
+                this.parent = Objects.requireNonNull(parent);
+            }
+
+            @Override
+            public Collection<AggregationBuilder> getSiblingAggregations() {
+                return parent.getSubAggregations();
+            }
+
+            @Override
+            public Collection<PipelineAggregationBuilder> getSiblingPipelineAggregations() {
+                return parent.getPipelineAggregations();
+            }
+
+            @Override
+            public void validateParentAggSequentiallyOrdered(String type, String name) {
+                if (parent instanceof HistogramAggregationBuilder) {
+                    HistogramAggregationBuilder histoParent = (HistogramAggregationBuilder) parent;
+                    if (histoParent.minDocCount() != 0) {
+                        addValidationError(
+                                "parent histogram of " + type + " aggregation [" + name + "] must have min_doc_count of 0");
+                    }
+                } else if (parent instanceof DateHistogramAggregationBuilder) {
+                    DateHistogramAggregationBuilder histoParent = (DateHistogramAggregationBuilder) parent;
+                    if (histoParent.minDocCount() != 0) {
+                        addValidationError(
+                                "parent histogram of " + type + " aggregation [" + name + "] must have min_doc_count of 0");
+                    }
+                } else if (parent instanceof AutoDateHistogramAggregationBuilder) {
+                    // Nothing to check
+                } else {
+                    addValidationError(
+                            type + " aggregation [" + name + "] must have a histogram, date_histogram or auto_date_histogram as parent");
+                }
+            }
+        }
+
+        /**
+         * Aggregations that are siblings to the aggregation being validated.
+         */
+        public abstract Collection<AggregationBuilder> getSiblingAggregations();
+
+        /**
+         * Pipeline aggregations that are siblings to the aggregation being validated.
+         */
+        public abstract Collection<PipelineAggregationBuilder> getSiblingPipelineAggregations();
+
+        /**
+         * Add a validation error to this context. All validation errors
+         * are accumulated in a list and, if there are any, the request
+         * is not executed and the entire list is returned as the error
+         * response.
+         */
+        public void addValidationError(String error) {
+            e = ValidateActions.addValidationError(error, e);
+        }
+
+        /**
+         * Add a validation error about the {@code buckets_path}.
+         */
+        public void addBucketPathValidationError(String error) {
+            addValidationError(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName() + ' ' + error);
+        }
+
+        /**
+         * Validates that the parent is sequentially ordered.
+         */
+        public abstract void validateParentAggSequentiallyOrdered(String type, String name);
+
+        /**
+         * The validation exception, if there is one. It'll be {@code null}
+         * if the context wasn't provided with any exception on creation
+         * and none were added.
+         */
+        public ActionRequestValidationException getValidationException() {
+            return e;
+        }
+    }
 
     /**
      * Creates the pipeline aggregator
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java
index cf8c951a6c5de..60092f1d353c5 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java
@@ -116,7 +116,7 @@ public static CompositeValuesSourceBuilder<?> fromXContent(XContentParser parser
                 builder = TermsValuesSourceBuilder.parse(name, parser);
                 break;
             case DateHistogramValuesSourceBuilder.TYPE:
-                builder = DateHistogramValuesSourceBuilder.parse(name, parser);
+                builder = DateHistogramValuesSourceBuilder.PARSER.parse(parser, name);
                 break;
             case HistogramValuesSourceBuilder.TYPE:
                 builder = HistogramValuesSourceBuilder.parse(name, parser);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
index 2c76872479074..dfac9ee524c0b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.search.aggregations.bucket.composite;
 
+import java.io.IOException;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.util.Objects;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Rounding;
@@ -41,11 +46,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
-import java.io.IOException;
-import java.time.ZoneId;
-import java.time.ZoneOffset;
-import java.util.Objects;
-
 /**
  * A {@link CompositeValuesSourceBuilder} that builds a {@link RoundingValuesSource} from a {@link Script} or
  * a field name using the provided interval.
@@ -54,9 +54,9 @@ public class DateHistogramValuesSourceBuilder
     extends CompositeValuesSourceBuilder<DateHistogramValuesSourceBuilder> implements DateIntervalConsumer {
     static final String TYPE = "date_histogram";
 
-    private static final ObjectParser<DateHistogramValuesSourceBuilder, Void> PARSER;
+    static final ObjectParser<DateHistogramValuesSourceBuilder, String> PARSER =
+            ObjectParser.fromBuilder(TYPE, DateHistogramValuesSourceBuilder::new);
     static {
-        PARSER = new ObjectParser<>(DateHistogramValuesSourceBuilder.TYPE);
         PARSER.declareString(DateHistogramValuesSourceBuilder::format, new ParseField("format"));
         DateIntervalWrapper.declareIntervalFields(PARSER);
         PARSER.declareField(DateHistogramValuesSourceBuilder::offset, p -> {
@@ -75,9 +75,6 @@ public class DateHistogramValuesSourceBuilder
         }, new ParseField("time_zone"), ObjectParser.ValueType.LONG);
         CompositeValuesSourceParserHelper.declareValuesSourceFields(PARSER, ValueType.NUMERIC);
     }
-    static DateHistogramValuesSourceBuilder parse(String name, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new DateHistogramValuesSourceBuilder(name), null);
-    }
 
     private ZoneId timeZone = null;
     private DateIntervalWrapper dateHistogramInterval = new DateIntervalWrapper();
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
index 0a7c918231c73..71a79818828ed 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
@@ -42,10 +47,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
 public abstract class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoGridAggregationBuilder>
         implements MultiBucketAggregationBuilder {
     /* recognized field names in JSON */
@@ -64,8 +65,9 @@ protected interface PrecisionParser {
         int parse(XContentParser parser) throws IOException;
     }
 
-    public static ObjectParser<GeoGridAggregationBuilder, Void> createParser(String name, PrecisionParser precisionParser) {
-        ObjectParser<GeoGridAggregationBuilder, Void> parser = new ObjectParser<>(name);
+    public static <T extends GeoGridAggregationBuilder> ObjectParser<T, String> createParser(
+            String name, PrecisionParser precisionParser, Function<String, T> ctor) {
+        ObjectParser<T, String> parser = ObjectParser.fromBuilder(name, ctor);
         ValuesSourceParserHelper.declareGeoFields(parser, false, false);
         parser.declareField((p, builder, context) -> builder.precision(precisionParser.parse(p)), FIELD_PRECISION,
             org.elasticsearch.common.xcontent.ObjectParser.ValueType.INT);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
index acc9cde113164..e685a82b00e86 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
@@ -19,11 +19,13 @@
 
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
+import java.io.IOException;
+import java.util.Map;
+
 import org.elasticsearch.common.geo.GeoBoundingBox;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -32,15 +34,13 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
-import java.io.IOException;
-import java.util.Map;
-
 public class GeoHashGridAggregationBuilder extends GeoGridAggregationBuilder {
     public static final String NAME = "geohash_grid";
     public static final int DEFAULT_PRECISION = 5;
     public static final int DEFAULT_MAX_NUM_CELLS = 10000;
 
-    private static final ObjectParser<GeoGridAggregationBuilder, Void> PARSER = createParser(NAME, GeoUtils::parsePrecision);
+    public static final ObjectParser<GeoHashGridAggregationBuilder, String> PARSER =
+            createParser(NAME, GeoUtils::parsePrecision, GeoHashGridAggregationBuilder::new);
 
     public GeoHashGridAggregationBuilder(String name) {
         super(name);
@@ -79,10 +79,6 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu
         return new GeoHashGridAggregationBuilder(this, factoriesBuilder, metaData);
     }
 
-    public static GeoGridAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new GeoHashGridAggregationBuilder(aggregationName), null);
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
index 595c6cab6e718..01a670caf6cfe 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
@@ -19,10 +19,12 @@
 
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
+import java.io.IOException;
+import java.util.Map;
+
 import org.elasticsearch.common.geo.GeoBoundingBox;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -31,15 +33,13 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
-import java.io.IOException;
-import java.util.Map;
-
 public class GeoTileGridAggregationBuilder extends GeoGridAggregationBuilder {
     public static final String NAME = "geotile_grid";
     public static final int DEFAULT_PRECISION = 7;
     private static final int DEFAULT_MAX_NUM_CELLS = 10000;
 
-    private static final ObjectParser<GeoGridAggregationBuilder, Void> PARSER = createParser(NAME, GeoTileUtils::parsePrecision);
+    public static final ObjectParser<GeoTileGridAggregationBuilder, String> PARSER =
+        createParser(NAME, GeoTileUtils::parsePrecision, GeoTileGridAggregationBuilder::new);
 
     public GeoTileGridAggregationBuilder(String name) {
         super(name);
@@ -77,10 +77,6 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu
         return new GeoTileGridAggregationBuilder(this, factoriesBuilder, metaData);
     }
 
-    public static GeoGridAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new GeoTileGridAggregationBuilder(aggregationName), null);
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
index 313d34bbbad17..06cf86d852e11 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
@@ -19,6 +19,14 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import static java.util.Map.entry;
+
+import java.io.IOException;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Objects;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Rounding;
@@ -28,7 +36,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
@@ -43,14 +50,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 
-import java.io.IOException;
-import java.time.ZoneId;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Objects;
-
-import static java.util.Map.entry;
-
 public class AutoDateHistogramAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, AutoDateHistogramAggregationBuilder> {
 
@@ -59,9 +58,9 @@ public class AutoDateHistogramAggregationBuilder
     private static final ParseField NUM_BUCKETS_FIELD = new ParseField("buckets");
     private static final ParseField MINIMUM_INTERVAL_FIELD = new ParseField("minimum_interval");
 
-    private static final ObjectParser<AutoDateHistogramAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<AutoDateHistogramAggregationBuilder, String> PARSER =
+        ObjectParser.fromBuilder(NAME, AutoDateHistogramAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(AutoDateHistogramAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, true);
         PARSER.declareInt(AutoDateHistogramAggregationBuilder::setNumBuckets, NUM_BUCKETS_FIELD);
         PARSER.declareStringOrNull(AutoDateHistogramAggregationBuilder::setMinimumIntervalExpression, MINIMUM_INTERVAL_FIELD);
@@ -110,10 +109,6 @@ static RoundingInfo[] buildRoundings(ZoneId timeZone, String minimumInterval) {
         return Arrays.copyOfRange(roundings, indexToSliceFrom, roundings.length);
     }
 
-    public static AutoDateHistogramAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new AutoDateHistogramAggregationBuilder(aggregationName), null);
-    }
-
     private int numBuckets = 10;
 
     private String minimumIntervalExpression;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
index c60f9c7f52cef..03aaaa89f3757 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
@@ -19,6 +19,17 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import static java.util.Map.entry;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.zone.ZoneOffsetTransition;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedNumericDocValues;
@@ -26,14 +37,12 @@
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
@@ -55,17 +64,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
-import java.io.IOException;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZoneOffset;
-import java.time.zone.ZoneOffsetTransition;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import static java.util.Map.entry;
-
 /**
  * A builder for histograms on date fields.
  */
@@ -73,7 +71,6 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuil
         implements MultiBucketAggregationBuilder, DateIntervalConsumer {
 
     public static final String NAME = "date_histogram";
-    private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser();
 
     public static final Map<String, Rounding.DateTimeUnit> DATE_FIELD_UNITS = Map.ofEntries(
             entry("year", Rounding.DateTimeUnit.YEAR_OF_CENTURY),
@@ -93,11 +90,10 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuil
             entry("second", Rounding.DateTimeUnit.SECOND_OF_MINUTE),
             entry("1s", Rounding.DateTimeUnit.SECOND_OF_MINUTE));
 
-    private static final ObjectParser<DateHistogramAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<DateHistogramAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME, DateHistogramAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(DateHistogramAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareAnyFields(PARSER, true, true, true);
-
         DateIntervalWrapper.declareIntervalFields(PARSER);
 
         PARSER.declareField(DateHistogramAggregationBuilder::offset, p -> {
@@ -119,10 +115,6 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuil
                 Histogram.ORDER_FIELD);
     }
 
-    public static DateHistogramAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new DateHistogramAggregationBuilder(aggregationName), null);
-    }
-
     private DateIntervalWrapper dateHistogramInterval = new DateIntervalWrapper();
     private long offset = 0;
     private ExtendedBounds extendedBounds;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
index 18e7d8720607e..04874c592c967 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
@@ -86,7 +86,7 @@ public String value() {
     private DateHistogramInterval dateHistogramInterval;
     private IntervalTypeEnum intervalType = IntervalTypeEnum.NONE;
 
-    public static <T extends DateIntervalConsumer> void declareIntervalFields(ObjectParser<T, Void> parser) {
+    public static <T extends DateIntervalConsumer> void declareIntervalFields(ObjectParser<T, String> parser) {
 
         // NOTE: this field is deprecated and will be removed
         parser.declareField((wrapper, interval) -> {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
index 11a792ed25d6f..1a5a2cce36c86 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
@@ -19,12 +19,16 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -43,11 +47,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
 /**
  * A builder for histograms on numeric fields.  This builder can operate on either base numeric fields, or numeric range fields.  IP range
  * fields are unsupported, and will throw at the factory layer.
@@ -64,9 +63,9 @@ public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder<
         EXTENDED_BOUNDS_PARSER.declareDouble((bounds, d) -> bounds[1] = d, new ParseField("max"));
     }
 
-    private static final ObjectParser<HistogramAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<HistogramAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME, HistogramAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(HistogramAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareAnyFields(PARSER, true, true);
 
         PARSER.declareDouble(HistogramAggregationBuilder::interval, Histogram.INTERVAL_FIELD);
@@ -85,10 +84,6 @@ public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder<
             Histogram.ORDER_FIELD);
     }
 
-    public static HistogramAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new HistogramAggregationBuilder(aggregationName), null);
-    }
-
     private double interval;
     private double offset = 0;
     private double minBound = Double.POSITIVE_INFINITY;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
index fcefc81b69845..92880c2011df0 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
@@ -38,9 +38,9 @@
 public class DateRangeAggregationBuilder extends AbstractRangeBuilder<DateRangeAggregationBuilder, RangeAggregator.Range> {
     public static final String NAME = "date_range";
 
-    private static final ObjectParser<DateRangeAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<DateRangeAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME,  DateRangeAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(DateRangeAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, true);
         PARSER.declareBoolean(DateRangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD);
 
@@ -51,9 +51,6 @@ public class DateRangeAggregationBuilder extends AbstractRangeBuilder<DateRangeA
         }, (p, c) -> DateRangeAggregationBuilder.parseRange(p), RangeAggregator.RANGES_FIELD);
     }
 
-    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new DateRangeAggregationBuilder(aggregationName), null);
-    }
 
     private static RangeAggregator.Range parseRange(XContentParser parser) throws IOException {
         return RangeAggregator.Range.fromXContent(parser);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
index e0b3fad96f6bc..eda8c0524e6bf 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
@@ -18,6 +18,15 @@
  */
 package org.elasticsearch.search.aggregations.bucket.range;
 
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParseField;
@@ -44,24 +53,15 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
 
 public final class IpRangeAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Bytes, IpRangeAggregationBuilder> {
     public static final String NAME = "ip_range";
     private static final ParseField MASK_FIELD = new ParseField("mask");
 
-    private static final ObjectParser<IpRangeAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<IpRangeAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME, IpRangeAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(IpRangeAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareBytesFields(PARSER, false, false);
 
         PARSER.declareBoolean(IpRangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD);
@@ -71,10 +71,6 @@ public final class IpRangeAggregationBuilder
         }, (p, c) -> IpRangeAggregationBuilder.parseRange(p), RangeAggregator.RANGES_FIELD);
     }
 
-    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new IpRangeAggregationBuilder(aggregationName), null);
-    }
-
     private static Range parseRange(XContentParser parser) throws IOException {
         String key = null;
         String from = null;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
index 6a95447287799..7744cae764c57 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
@@ -38,9 +38,9 @@
 public class RangeAggregationBuilder extends AbstractRangeBuilder<RangeAggregationBuilder, Range> {
     public static final String NAME = "range";
 
-    private static final ObjectParser<RangeAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<RangeAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME, RangeAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(RangeAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
         PARSER.declareBoolean(RangeAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD);
 
@@ -51,10 +51,6 @@ public class RangeAggregationBuilder extends AbstractRangeBuilder<RangeAggregati
         }, (p, c) -> RangeAggregationBuilder.parseRange(p), RangeAggregator.RANGES_FIELD);
     }
 
-    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new RangeAggregationBuilder(aggregationName), null);
-    }
-
     private static Range parseRange(XContentParser parser) throws IOException {
         return Range.fromXContent(parser);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
index 0687c81a64802..3de703099bcb2 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -88,7 +88,12 @@ public SignificantTermsAggregatorFactory(String name,
                                              Map<String, Object> metaData) throws IOException {
         super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData);
 
-        if (!config.unmapped()) {
+        if (config.unmapped() == false) {
+            if (config.fieldContext().fieldType().isSearchable() == false) {
+                throw new IllegalArgumentException("SignificantText aggregation requires fields to be searchable, but ["
+                    + config.fieldContext().fieldType().name() + "] is not");
+            }
+
             this.fieldType = config.fieldContext().fieldType();
             this.indexedFieldName = fieldType.name();
         }
@@ -129,6 +134,10 @@ private FilterableTermsEnum getTermsEnum(String field) throws IOException {
     }
 
     private long getBackgroundFrequency(String value) throws IOException {
+        // fieldType can be null if the field is unmapped, but theoretically this method should only be called
+        // when constructing buckets.  Assert to ensure this is the case
+        // TODO this is a bad setup and it should be refactored
+        assert fieldType != null;
         Query query = fieldType.termQuery(value, queryShardContext);
         if (query instanceof TermQuery) {
             // for types that use the inverted index, we prefer using a caching terms
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
index 4930c213443b8..950bde076908d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
@@ -83,6 +83,10 @@ public SignificantTextAggregatorFactory(String name,
         // Note that if the field is unmapped (its field type is null), we don't fail,
         // and just use the given field name as a placeholder.
         this.fieldType = queryShardContext.fieldMapper(fieldName);
+        if (fieldType != null && fieldType.indexAnalyzer() == null) {
+            throw new IllegalArgumentException("Field [" + fieldType.name() + "] has no analyzer, but SignificantText " +
+                "requires an analyzed field");
+        }
         this.indexedFieldName = fieldType != null ? fieldType.name() : fieldName;
         this.sourceFieldNames = sourceFieldNames == null
             ? new String[] { indexedFieldName }
@@ -124,6 +128,10 @@ private FilterableTermsEnum getTermsEnum(String field) throws IOException {
     }
 
     private long getBackgroundFrequency(String value) throws IOException {
+        // fieldType can be null if the field is unmapped, but theoretically this method should only be called
+        // when constructing buckets.  Assert to ensure this is the case
+        // TODO this is a bad setup and it should be refactored
+        assert fieldType != null;
         Query query = fieldType.termQuery(value, queryShardContext);
         if (query instanceof TermQuery) {
             // for types that use the inverted index, we prefer using a caching terms
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
index 8b9f89fd54209..80658243a0794 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
@@ -19,40 +19,35 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
-import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
 
 public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoBoundsAggregationBuilder> {
     public static final String NAME = "geo_bounds";
 
-    private static final ObjectParser<GeoBoundsAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<GeoBoundsAggregationBuilder, String> PARSER =
+            ObjectParser.fromBuilder(NAME, GeoBoundsAggregationBuilder::new); 
     static {
-        PARSER = new ObjectParser<>(GeoBoundsAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareGeoFields(PARSER, false, false);
         PARSER.declareBoolean(GeoBoundsAggregationBuilder::wrapLongitude, GeoBoundsAggregator.WRAP_LONGITUDE_FIELD);
     }
 
-    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new GeoBoundsAggregationBuilder(aggregationName), null);
-    }
-
     private boolean wrapLongitude = true;
 
     public GeoBoundsAggregationBuilder(String name) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
index daf357186f36c..afa0329433a6e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
@@ -19,11 +19,13 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
+import java.io.IOException;
+import java.util.Map;
+
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
@@ -35,23 +37,16 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 
-import java.io.IOException;
-import java.util.Map;
-
 public class GeoCentroidAggregationBuilder
         extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.GeoPoint, GeoCentroidAggregationBuilder> {
     public static final String NAME = "geo_centroid";
 
-    private static final ObjectParser<GeoCentroidAggregationBuilder, Void> PARSER;
+    public static final ObjectParser<GeoCentroidAggregationBuilder, String> PARSER =
+        ObjectParser.fromBuilder(NAME, GeoCentroidAggregationBuilder::new);
     static {
-        PARSER = new ObjectParser<>(GeoCentroidAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareGeoFields(PARSER, true, false);
     }
 
-    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new GeoCentroidAggregationBuilder(aggregationName), null);
-    }
-
     public GeoCentroidAggregationBuilder(String name) {
         super(name, CoreValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
index 35eab881bb643..c2bffc0450677 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
@@ -22,16 +22,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregatorFactory;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorFactory;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 
@@ -79,16 +73,6 @@ public String type() {
         return type;
     }
 
-    /**
-     * Validates the state of this factory (makes sure the factory is properly
-     * configured)
-     */
-    @Override
-    public final void validate(AggregatorFactory parent, Collection<AggregationBuilder> factories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
-        doValidate(parent, factories, pipelineAggregatorFactories);
-    }
-
     protected abstract PipelineAggregator createInternal(Map<String, Object> metaData);
 
     /**
@@ -102,32 +86,6 @@ public final PipelineAggregator create() {
         return aggregator;
     }
 
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> factories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
-    }
-
-    /**
-     * Validates pipeline aggregations that need sequentially ordered data.
-     */
-    public static void validateSequentiallyOrderedParentAggs(AggregatorFactory parent, String type, String name) {
-        if ((parent instanceof HistogramAggregatorFactory || parent instanceof DateHistogramAggregatorFactory
-                || parent instanceof AutoDateHistogramAggregatorFactory) == false) {
-            throw new IllegalStateException(
-                    type + " aggregation [" + name + "] must have a histogram, date_histogram or auto_date_histogram as parent");
-        }
-        if (parent instanceof HistogramAggregatorFactory) {
-            HistogramAggregatorFactory histoParent = (HistogramAggregatorFactory) parent;
-            if (histoParent.minDocCount() != 0) {
-                throw new IllegalStateException("parent histogram of " + type + " aggregation [" + name + "] must have min_doc_count of 0");
-            }
-        } else if (parent instanceof DateHistogramAggregatorFactory) {
-            DateHistogramAggregatorFactory histoParent = (DateHistogramAggregatorFactory) parent;
-            if (histoParent.minDocCount() != 0) {
-                throw new IllegalStateException("parent histogram of " + type + " aggregation [" + name + "] must have min_doc_count of 0");
-            }
-        }
-    }
-
     @SuppressWarnings("unchecked")
     @Override
     public PAB setMetaData(Map<String, Object> metaData) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java
index 0c4aa6f1359bd..753f3ca38821c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java
@@ -24,13 +24,10 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -107,28 +104,27 @@ public GapPolicy gapPolicy() {
     protected abstract PipelineAggregator createInternal(Map<String, Object> metaData);
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggBuilders,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
+    protected void validate(ValidationContext context) {
         if (bucketsPaths.length != 1) {
-            throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                    + " must contain a single entry for aggregation [" + name + "]");
+            context.addBucketPathValidationError("must contain a single entry for aggregation [" + name + "]");
+            return;
         }
         // Need to find the first agg name in the buckets path to check its a
         // multi bucket agg: aggs are split with '>' and can optionally have a
         // metric name after them by using '.' so need to split on both to get
         // just the agg name
         final String firstAgg = bucketsPaths[0].split("[>\\.]")[0];
-        Optional<AggregationBuilder> aggBuilder = aggBuilders.stream().filter((builder) -> builder.getName().equals(firstAgg))
+        Optional<AggregationBuilder> aggBuilder = context.getSiblingAggregations().stream()
+                .filter(builder -> builder.getName().equals(firstAgg))
                 .findAny();
-        if (aggBuilder.isPresent()) {
-            if ((aggBuilder.get() instanceof MultiBucketAggregationBuilder) == false) {
-                throw new IllegalArgumentException("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                        + " must be a multi-bucket aggregation for aggregation [" + name + "] found :"
-                        + aggBuilder.get().getClass().getName() + " for buckets path: " + bucketsPaths[0]);
-            }
-        } else {
-            throw new IllegalArgumentException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                    + " aggregation does not exist for aggregation [" + name + "]: " + bucketsPaths[0]);
+        if (aggBuilder.isEmpty()) {
+            context.addBucketPathValidationError("aggregation does not exist for aggregation [" + name + "]: " + bucketsPaths[0]);
+            return;
+        }
+        if ((aggBuilder.get() instanceof MultiBucketAggregationBuilder) == false) {
+            context.addValidationError("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                    + " must be a multi-bucket aggregation for aggregation [" + name + "] found :"
+                    + aggBuilder.get().getClass().getName() + " for buckets path: " + bucketsPaths[0]);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
index 5fddd5f1a9e1c..fa6c6b7639413 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
@@ -198,6 +198,11 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param
         return builder;
     }
 
+    @Override
+    protected void validate(ValidationContext context) {
+        // Nothing to check
+    }
+
     @Override
     protected boolean overrideBucketsPath() {
         return true;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
index 6bb83c60dea83..0981acbc9bafd 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
@@ -193,6 +193,11 @@ public static BucketSelectorPipelineAggregationBuilder parse(String reducerName,
         return factory;
     }
 
+    @Override
+    protected void validate(ValidationContext context) {
+        // Nothing to check
+    }
+
     @Override
     protected boolean overrideBucketsPath() {
         return true;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
index 1e80c6f78c3c5..7f6b491975868 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
@@ -25,9 +25,6 @@
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -36,7 +33,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -144,10 +140,9 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-                           Collection<PipelineAggregationBuilder> pipelineAggregatoractories) {
+    protected void validate(ValidationContext context) {
         if (sorts.isEmpty() && size == null && from == 0) {
-            throw new IllegalStateException("[" + name + "] is configured to perform nothing. Please set either of "
+            context.addValidationError("[" + name + "] is configured to perform nothing. Please set either of "
                     + Arrays.asList(SearchSourceBuilder.SORT_FIELD.getPreferredName(), SIZE.getPreferredName(), FROM.getPreferredName())
                     + " to use " + NAME);
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
index fc9b8caf72769..c1af597b4c275 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
@@ -25,13 +25,9 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -93,14 +89,11 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
+    protected void validate(ValidationContext context) {
         if (bucketsPaths.length != 1) {
-            throw new IllegalStateException(BUCKETS_PATH.getPreferredName()
-                    + " must contain a single entry for aggregation [" + name + "]");
+            context.addBucketPathValidationError("must contain a single entry for aggregation [" + name + "]");
         }
-        
-        validateSequentiallyOrderedParentAggs(parent, NAME, name);
+        context.validateParentAggSequentiallyOrdered(NAME, name);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
index 4aa4155ac063f..106a8a97f5728 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
@@ -28,16 +28,12 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -152,14 +148,13 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatoractories) {
+    protected void validate(ValidationContext context) {
         if (bucketsPaths.length != 1) {
-            throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+            context.addValidationError(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
         }
-        
-        validateSequentiallyOrderedParentAggs(parent, NAME, name);
+
+        context.validateParentAggSequentiallyOrdered(NAME, name);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
index 89816a1fb22e8..97948d0e8638a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
@@ -22,12 +22,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 
@@ -80,13 +76,10 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggBuilders,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
-        super.doValidate(parent, aggBuilders, pipelineAggregatorFactories);
-
-        if (sigma < 0.0 ) {
-            throw new IllegalStateException(ExtendedStatsBucketParser.SIGMA.getPreferredName()
-                    + " must be a non-negative double");
+    protected void validate(ValidationContext context) {
+        super.validate(context);
+        if (sigma < 0.0) {
+            context.addValidationError(ExtendedStatsBucketParser.SIGMA.getPreferredName() + " must be a non-negative double");
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
index d24710631d8fd..e5ec2780b0c14 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
@@ -30,13 +30,9 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
@@ -179,13 +175,11 @@ public void setShift(int shift) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-                           Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
+    protected void validate(ValidationContext context) {
         if (window <= 0) {
-            throw new IllegalArgumentException("[" + WINDOW.getPreferredName() + "] must be a positive, non-zero integer.");
+            context.addValidationError("[" + WINDOW.getPreferredName() + "] must be a positive, non-zero integer.");
         }
-        
-        validateSequentiallyOrderedParentAggs(parent, NAME, name);
+        context.validateParentAggSequentiallyOrdered(NAME, name);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
index 06a940af89c77..a806b54772140 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
@@ -20,18 +20,15 @@
 package org.elasticsearch.search.aggregations.pipeline;
 
 import com.carrotsearch.hppc.DoubleArrayList;
+
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 
@@ -109,14 +106,13 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
-        super.doValidate(parent, aggFactories, pipelineAggregatorFactories);
-
+    protected void validate(ValidationContext context) {
+        super.validate(context);
         for (Double p : percents) {
             if (p == null || p < 0.0 || p > 100.0) {
-                throw new IllegalStateException(PERCENTS_FIELD.getPreferredName()
+                context.addValidationError(PERCENTS_FIELD.getPreferredName()
                         + " must only contain non-null doubles from 0.0-100.0 inclusive");
+                return;
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
index 3b02edf51579a..fe6bb921b9129 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
@@ -26,14 +26,10 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -138,11 +134,10 @@ protected DocValueFormat formatter() {
     protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new SerialDiffPipelineAggregator(name, bucketsPaths, formatter(), gapPolicy, lag, metaData);
     }
-    
+
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-            Collection<PipelineAggregationBuilder> pipelineAggregatoractories) {        
-        validateSequentiallyOrderedParentAggs(parent, NAME, name);
+    protected void validate(ValidationContext context) {
+        context.validateParentAggSequentiallyOrdered(NAME, name);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 14c61311bddf6..09948955629e0 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -31,6 +32,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -112,6 +114,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R
     public static final ParseField SEARCH_AFTER = new ParseField("search_after");
     public static final ParseField COLLAPSE = new ParseField("collapse");
     public static final ParseField SLICE = new ParseField("slice");
+    public static final ParseField READER_CONTEXT = new ParseField("reader");
 
     public static SearchSourceBuilder fromXContent(XContentParser parser) throws IOException {
         return fromXContent(parser, true);
@@ -190,6 +193,8 @@ public static HighlightBuilder highlight() {
 
     private CollapseBuilder collapse = null;
 
+    private ReaderBuilder reader = null;
+
     /**
      * Constructs a new search source builder.
      */
@@ -244,6 +249,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException {
         sliceBuilder = in.readOptionalWriteable(SliceBuilder::new);
         collapse = in.readOptionalWriteable(CollapseBuilder::new);
         trackTotalHitsUpTo = in.readOptionalInt();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            reader = in.readOptionalWriteable(ReaderBuilder::new);
+        }
     }
 
     @Override
@@ -298,6 +306,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(sliceBuilder);
         out.writeOptionalWriteable(collapse);
         out.writeOptionalInt(trackTotalHitsUpTo);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalWriteable(reader);
+        }
     }
 
     /**
@@ -916,6 +927,21 @@ public boolean isSuggestOnly() {
             && queryBuilder == null && aggregations == null;
     }
 
+    /**
+     * Returns the reader context that is configured with this query
+     */
+    public ReaderBuilder reader() {
+        return reader;
+    }
+
+    /**
+     * Specify the reader context that this query should use to execute.
+     */
+    public SearchSourceBuilder reader(ReaderBuilder reader) {
+        this.reader = reader;
+        return this;
+    }
+
     /**
      * Rewrites this search source builder into its primitive form. e.g. by
      * rewriting the QueryBuilder. If the builder did not change the identity
@@ -1000,6 +1026,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder
         rewrittenBuilder.version = version;
         rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm;
         rewrittenBuilder.collapse = collapse;
+        rewrittenBuilder.reader = reader;
         return rewrittenBuilder;
     }
 
@@ -1121,6 +1148,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th
                     sliceBuilder = SliceBuilder.fromXContent(parser);
                 } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) {
                     collapse = CollapseBuilder.fromXContent(parser);
+                } else if (READER_CONTEXT.match(currentFieldName, parser.getDeprecationHandler())) {
+                    reader = ReaderBuilder.fromXContent(parser);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -1317,6 +1346,9 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         if (collapse != null) {
             builder.field(COLLAPSE.getPreferredName(), collapse);
         }
+        if (reader != null) {
+            builder.field(READER_CONTEXT.getPreferredName(), reader);
+        }
         return builder;
     }
 
@@ -1529,7 +1561,7 @@ public int hashCode() {
         return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder,
                 indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size,
                 sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version,
-                seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo);
+                seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, reader);
     }
 
     @Override
@@ -1568,7 +1600,8 @@ public boolean equals(Object obj) {
                 && Objects.equals(profile, other.profile)
                 && Objects.equals(extBuilders, other.extBuilders)
                 && Objects.equals(collapse, other.collapse)
-                && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo);
+                && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo)
+                && Objects.equals(reader, other.reader);
     }
 
     @Override
@@ -1583,4 +1616,85 @@ public String toString(Params params) {
             throw new ElasticsearchException(e);
         }
     }
+
+    /**
+     * Specify whether this search should use specific reader contexts instead of the latest ones.
+     */
+    public static final class ReaderBuilder implements Writeable, ToXContentObject {
+        private static final ParseField ID_FIELD = new ParseField("id");
+        private static final ParseField KEEP_ALIVE_FIELD = new ParseField("keep_alive");
+        private static final ObjectParser<XContentParams, Void> PARSER;
+
+        static {
+            PARSER = new ObjectParser<>(READER_CONTEXT.getPreferredName(), XContentParams::new);
+            PARSER.declareString((params, id) -> params.id = id, ID_FIELD);
+            PARSER.declareField((params, keepAlive) -> params.keepAlive = keepAlive,
+                (p, c) -> TimeValue.parseTimeValue(p.text(), KEEP_ALIVE_FIELD.getPreferredName()),
+                KEEP_ALIVE_FIELD, ObjectParser.ValueType.STRING);
+        }
+
+        private static final class XContentParams {
+            private String id;
+            private TimeValue keepAlive;
+        }
+
+        private final String id;
+        private final TimeValue keepAlive;
+
+        public ReaderBuilder(String id, TimeValue keepAlive) {
+            if (id == null && keepAlive == null) {
+                throw new IllegalArgumentException("id or keep_alive must be specified");
+            }
+            this.id = id;
+            this.keepAlive = keepAlive;
+        }
+
+        public ReaderBuilder(StreamInput in) throws IOException {
+            id = in.readOptionalString();
+            keepAlive = in.readOptionalTimeValue();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeOptionalString(id);
+            out.writeOptionalTimeValue(keepAlive);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            if (id != null) {
+                builder.field(ID_FIELD.getPreferredName(), id);
+            }
+            if (keepAlive != null) {
+                builder.field(KEEP_ALIVE_FIELD.getPreferredName(), keepAlive);
+            }
+            return builder;
+        }
+
+        public static ReaderBuilder fromXContent(XContentParser parser) throws IOException {
+            final XContentParams params = PARSER.parse(parser, null);
+            return new ReaderBuilder(params.id, params.keepAlive);
+        }
+
+        public TimeValue getKeepAlive() {
+            return keepAlive;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            final ReaderBuilder that = (ReaderBuilder) o;
+            return Objects.equals(id, that.id) && Objects.equals(keepAlive, that.keepAlive);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(id, keepAlive);
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index d3c4f5641dc23..ed41939b1cb9c 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -82,23 +82,20 @@ public class ContextIndexSearcher extends IndexSearcher {
     private MutableQueryTimeout cancellable;
 
     public ContextIndexSearcher(IndexReader reader, Similarity similarity,
-                                QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) throws IOException {
-        this(reader, similarity, queryCache, queryCachingPolicy, new MutableQueryTimeout());
+                                QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
+                                boolean wrapWithExitableDirectoryReader) throws IOException {
+        this(reader, similarity, queryCache, queryCachingPolicy, new MutableQueryTimeout(), wrapWithExitableDirectoryReader);
     }
 
-    // TODO: Make the 2nd constructor private so that the IndexReader is always wrapped.
-    // Some issues must be fixed:
-    //   - regarding tests deriving from AggregatorTestCase and more specifically the use of searchAndReduce and
-    //     the ShardSearcher sub-searchers.
-    //   - tests that use a MultiReader
-    public ContextIndexSearcher(IndexReader reader, Similarity similarity,
-                                QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
-                                MutableQueryTimeout cancellable) throws IOException {
-        super(cancellable != null ? new ExitableDirectoryReader((DirectoryReader) reader, cancellable) : reader);
+    private ContextIndexSearcher(IndexReader reader, Similarity similarity,
+                                 QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
+                                 MutableQueryTimeout cancellable,
+                                 boolean wrapWithExitableDirectoryReader) throws IOException {
+        super(wrapWithExitableDirectoryReader ? new ExitableDirectoryReader((DirectoryReader) reader, cancellable) : reader);
         setSimilarity(similarity);
         setQueryCache(queryCache);
         setQueryCachingPolicy(queryCachingPolicy);
-        this.cancellable = cancellable != null ? cancellable : new MutableQueryTimeout();
+        this.cancellable = cancellable;
     }
 
     public void setProfiler(QueryProfiler profiler) {
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
index 51a94f4023b65..0a2dd476cf3b1 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
@@ -245,7 +245,7 @@ public int getDocCount() {
 
     private static class ExitableIntersectVisitor implements PointValues.IntersectVisitor {
 
-        private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 10) - 1; // 1023
+        private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 13) - 1; // 8191
 
         private final PointValues.IntersectVisitor in;
         private final QueryCancellation queryCancellation;
diff --git a/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
index e97aae9fb152a..d219c2c5621f4 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
@@ -32,8 +32,11 @@ public class LegacyReaderContext extends ReaderContext {
     private AggregatedDfs aggregatedDfs;
     private RescoreDocIds rescoreDocIds;
 
-    public LegacyReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher, ShardSearchRequest shardSearchRequest) {
-        super(id, indexShard, engineSearcher);
+    public LegacyReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher,
+                               ShardSearchRequest shardSearchRequest, long keepAliveInMillis) {
+        super(id, indexShard, engineSearcher, keepAliveInMillis, false);
+        assert shardSearchRequest.readerId() == null;
+        assert shardSearchRequest.keepAlive() == null;
         this.shardSearchRequest = Objects.requireNonNull(shardSearchRequest);
         if (shardSearchRequest.scroll() != null) {
             this.scrollContext = new ScrollContext();
@@ -71,4 +74,9 @@ public RescoreDocIds getRescoreDocIds(RescoreDocIds other) {
     public void setRescoreDocIds(RescoreDocIds rescoreDocIds) {
         this.rescoreDocIds = rescoreDocIds;
     }
+
+    @Override
+    public boolean singleSession() {
+        return scrollContext == null || scrollContext.scroll == null;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
index 9ba2939ef1c51..eb94b8f91e615 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
@@ -32,6 +32,8 @@
 import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * Holds a reference to a point in time {@link Engine.Searcher} that will be used to construct {@link SearchContext}.
@@ -47,17 +49,26 @@ public class ReaderContext extends AbstractRefCounted implements Releasable {
     private final IndexShard indexShard;
     private final Engine.Searcher engineSearcher;
     private final AtomicBoolean closed = new AtomicBoolean(false);
+    private final boolean singleSession;
 
-    private volatile long keepAlive;
-    private volatile long lastAccessTime = -1L;
+    private final AtomicLong keepAlive;
+    private final AtomicLong lastAccessTime;
+    private final AtomicInteger accessors = new AtomicInteger();
 
     private final List<Releasable> onCloses = new CopyOnWriteArrayList<>();
 
-    public ReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher) {
+    public ReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher, long keepAliveInMillis, boolean singleSession) {
         super("reader_context");
         this.id = new SearchContextId(UUIDs.base64UUID(), id);
         this.indexShard = indexShard;
         this.engineSearcher = engineSearcher;
+        this.singleSession = singleSession;
+        this.keepAlive = new AtomicLong(keepAliveInMillis);
+        this.lastAccessTime = new AtomicLong(nowInMillis());
+    }
+
+    private long nowInMillis() {
+        return indexShard.getThreadPool().relativeTimeInMillis();
     }
 
     @Override
@@ -94,20 +105,29 @@ public String source() {
         return engineSearcher.source();
     }
 
-    public void accessed(long accessTime) {
-        this.lastAccessTime = accessTime;
-    }
-
-    public long lastAccessTime() {
-        return this.lastAccessTime;
+    public void keepAlive(long keepAlive) {
+        this.keepAlive.updateAndGet(curr -> Math.max(curr, keepAlive));
     }
 
-    public long keepAlive() {
-        return this.keepAlive;
+    /**
+     * Marks this reader as being used so its time to live should not be expired.
+     *
+     * @return a releasable to indicate the caller has stopped using this reader
+     */
+    public Releasable markAsUsed() {
+        accessors.incrementAndGet();
+        return Releasables.releaseOnce(() -> {
+            this.lastAccessTime.updateAndGet(curr -> Math.max(curr, nowInMillis()));
+            accessors.decrementAndGet();
+        });
     }
 
-    public void keepAlive(long keepAlive) {
-        this.keepAlive = keepAlive;
+    public boolean isKeepAliveLapsed() {
+        if (accessors.get() > 0) {
+            return false; // being used
+        }
+        final long elapsed = nowInMillis() - lastAccessTime.get();
+        return elapsed > keepAlive.get();
     }
 
     // BWC
@@ -134,4 +154,13 @@ public RescoreDocIds getRescoreDocIds(RescoreDocIds other) {
     public void setRescoreDocIds(RescoreDocIds rescoreDocIds) {
 
     }
+
+    /**
+     * Returns {@code true} for readers that are intended to use in a single query. For readers that are intended
+     * to use in multiple queries (i.e., scroll or readers), we should not release them after the fetch phase
+     * or the query phase with empty results.
+     */
+    public boolean singleSession() {
+        return singleSession;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
index 38513dcc5b7d3..8cb64322d1938 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
@@ -27,7 +27,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-
+// TODO: Rename this to ReaderId?
 public final class SearchContextId implements Writeable {
     private final String readerId;
     private final long id;
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index 96d0d67812d3c..1bfaf2efb7b2b 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
@@ -88,6 +89,8 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque
     //these are the only mutable fields, as they are subject to rewriting
     private AliasFilter aliasFilter;
     private SearchSourceBuilder source;
+    private final SearchContextId readerId;
+    private final TimeValue keepAlive;
 
     public ShardSearchRequest(OriginalIndices originalIndices,
                               SearchRequest searchRequest,
@@ -98,6 +101,21 @@ public ShardSearchRequest(OriginalIndices originalIndices,
                               long nowInMillis,
                               @Nullable String clusterAlias,
                               String[] indexRoutings) {
+        this(originalIndices, searchRequest, shardId, numberOfShards, aliasFilter,
+            indexBoost, nowInMillis, clusterAlias, indexRoutings, null, null);
+    }
+
+    public ShardSearchRequest(OriginalIndices originalIndices,
+                              SearchRequest searchRequest,
+                              ShardId shardId,
+                              int numberOfShards,
+                              AliasFilter aliasFilter,
+                              float indexBoost,
+                              long nowInMillis,
+                              @Nullable String clusterAlias,
+                              String[] indexRoutings,
+                              SearchContextId readerId,
+                              TimeValue keepAlive) {
         this(originalIndices,
             shardId,
             numberOfShards,
@@ -111,7 +129,9 @@ public ShardSearchRequest(OriginalIndices originalIndices,
             searchRequest.preference(),
             searchRequest.scroll(),
             nowInMillis,
-            clusterAlias);
+            clusterAlias,
+            readerId,
+            keepAlive);
         // If allowPartialSearchResults is unset (ie null), the cluster-level default should have been substituted
         // at this stage. Any NPEs in the above are therefore an error in request preparation logic.
         assert searchRequest.allowPartialSearchResults() != null;
@@ -121,7 +141,7 @@ public ShardSearchRequest(ShardId shardId,
                               long nowInMillis,
                               AliasFilter aliasFilter) {
         this(OriginalIndices.NONE, shardId, -1, null, null, null,
-            aliasFilter, 1.0f, false, Strings.EMPTY_ARRAY, null, null, nowInMillis, null);
+            aliasFilter, 1.0f, false, Strings.EMPTY_ARRAY, null, null, nowInMillis, null, null, null);
     }
 
     private ShardSearchRequest(OriginalIndices originalIndices,
@@ -137,7 +157,9 @@ private ShardSearchRequest(OriginalIndices originalIndices,
                                String preference,
                                Scroll scroll,
                                long nowInMillis,
-                               @Nullable String clusterAlias) {
+                               @Nullable String clusterAlias,
+                               SearchContextId readerId,
+                               TimeValue keepAlive) {
         this.shardId = shardId;
         this.numberOfShards = numberOfShards;
         this.searchType = searchType;
@@ -152,6 +174,8 @@ private ShardSearchRequest(OriginalIndices originalIndices,
         this.nowInMillis = nowInMillis;
         this.clusterAlias = clusterAlias;
         this.originalIndices = originalIndices;
+        this.readerId = readerId;
+        this.keepAlive = keepAlive;
     }
 
     public ShardSearchRequest(StreamInput in) throws IOException {
@@ -184,9 +208,37 @@ public ShardSearchRequest(StreamInput in) throws IOException {
             canReturnNullResponseIfMatchNoDocs = false;
             bottomSortValues = null;
         }
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            this.readerId = in.readOptionalWriteable(SearchContextId::new);
+            this.keepAlive = in.readOptionalTimeValue();
+        } else {
+            this.readerId = null;
+            this.keepAlive = null;
+        }
         originalIndices = OriginalIndices.readOriginalIndices(in);
     }
 
+    public ShardSearchRequest(ShardSearchRequest clone) {
+        this.shardId = clone.shardId;
+        this.searchType = clone.searchType;
+        this.numberOfShards = clone.numberOfShards;
+        this.scroll = clone.scroll;
+        this.source = clone.source;
+        this.aliasFilter = clone.aliasFilter;
+        this.indexBoost = clone.indexBoost;
+        this.nowInMillis = clone.nowInMillis;
+        this.requestCache = clone.requestCache;
+        this.clusterAlias = clone.clusterAlias;
+        this.allowPartialSearchResults = clone.allowPartialSearchResults;
+        this.indexRoutings = clone.indexRoutings;
+        this.preference = clone.preference;
+        this.canReturnNullResponseIfMatchNoDocs = clone.canReturnNullResponseIfMatchNoDocs;
+        this.bottomSortValues = clone.bottomSortValues;
+        this.originalIndices = clone.originalIndices;
+        this.readerId = clone.readerId;
+        this.keepAlive = clone.keepAlive;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -218,10 +270,14 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce
             out.writeStringArray(indexRoutings);
             out.writeOptionalString(preference);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (out.getVersion().onOrAfter(Version.V_7_7_0) && asKey == false) {
             out.writeBoolean(canReturnNullResponseIfMatchNoDocs);
             out.writeOptionalWriteable(bottomSortValues);
         }
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalWriteable(readerId);
+            out.writeOptionalTimeValue(keepAlive);
+        }
     }
 
     @Override
@@ -323,6 +379,28 @@ public void canReturnNullResponseIfMatchNoDocs(boolean value) {
         this.canReturnNullResponseIfMatchNoDocs = value;
     }
 
+    /**
+     * Returns <code>true</code> if the request needs to create a long-lived context.
+     */
+    public boolean shouldCreatePersistentReader() {
+        return scroll != null  || (keepAlive != null && readerId == null);
+    }
+
+    /**
+     * Returns a non-null value if this request should execute using a specific point-in-time reader;
+     * otherwise, using the most up to date point-in-time reader.
+     */
+    public SearchContextId readerId() {
+        return readerId;
+    }
+
+    /**
+     * Returns a non-null to specify the time to live of the point-in-time reader that is used to execute this request.
+     */
+    public TimeValue keepAlive() {
+        return keepAlive;
+    }
+
     /**
      * Returns the cache key for this shard search request, based on its content
      */
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 8f6515e27a54f..0a78447388e35 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -112,9 +112,9 @@ public QueryPhase() {
     public void preProcess(SearchContext context) {
         final Runnable cancellation;
         if (context.lowLevelCancellation()) {
-            SearchShardTask task = context.getTask();
             cancellation = context.searcher().addQueryCancellation(() -> {
-                if (task.isCancelled()) {
+                SearchShardTask task = context.getTask();
+                if (task != null && task.isCancelled()) {
                     throw new TaskCancelledException("cancelled");
                 }
             });
@@ -282,9 +282,9 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe
             }
 
             if (searchContext.lowLevelCancellation()) {
-                SearchShardTask task = searchContext.getTask();
                 searcher.addQueryCancellation(() -> {
-                    if (task.isCancelled()) {
+                    SearchShardTask task = searchContext.getTask();
+                    if (task != null && task.isCancelled()) {
                         throw new TaskCancelledException("cancelled");
                     }
                 });
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 770c901d29026..20ccab1e0c067 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -19,9 +19,16 @@
 
 package org.elasticsearch.search.query;
 
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.common.lucene.Lucene.readTopDocs;
+import static org.elasticsearch.common.lucene.Lucene.writeTopDocs;
+
+import java.io.IOException;
+
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.DelayableWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
@@ -29,24 +36,14 @@
 import org.elasticsearch.search.RescoreDocIds;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
 import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.common.lucene.Lucene.readTopDocs;
-import static org.elasticsearch.common.lucene.Lucene.writeTopDocs;
-
 public final class QuerySearchResult extends SearchPhaseResult {
 
     private int from;
@@ -56,7 +53,14 @@ public final class QuerySearchResult extends SearchPhaseResult {
     private TotalHits totalHits;
     private float maxScore = Float.NaN;
     private DocValueFormat[] sortValueFormats;
-    private InternalAggregations aggregations;
+    /**
+     * Aggregation results. We wrap them in
+     * {@linkplain DelayableWriteable} because
+     * {@link InternalAggregation} is usually made up of many small objects
+     * which have a fairly high overhead in the JVM. So we delay deserializing
+     * them until just before we need them.
+     */
+    private DelayableWriteable<InternalAggregations> aggregations;
     private boolean hasAggs;
     private Suggest suggest;
     private boolean searchTimedOut;
@@ -199,21 +203,21 @@ public boolean hasAggs() {
      * Returns and nulls out the aggregation for this search results. This allows to free up memory once the aggregation is consumed.
      * @throws IllegalStateException if the aggregations have already been consumed.
      */
-    public Aggregations consumeAggs() {
+    public DelayableWriteable<InternalAggregations> consumeAggs() {
         if (aggregations == null) {
             throw new IllegalStateException("aggs already consumed");
         }
-        Aggregations aggs = aggregations;
+        DelayableWriteable<InternalAggregations> aggs = aggregations;
         aggregations = null;
         return aggs;
     }
 
     public void aggregations(InternalAggregations aggregations) {
-        this.aggregations = aggregations;
+        this.aggregations = aggregations == null ? null : DelayableWriteable.referencing(aggregations);
         hasAggs = aggregations != null;
     }
 
-    public InternalAggregations aggregations() {
+    public DelayableWriteable<InternalAggregations> aggregations() {
         return aggregations;
     }
 
@@ -316,19 +320,17 @@ public void readFromWithId(SearchContextId id, StreamInput in) throws IOExceptio
             }
         }
         setTopDocs(readTopDocs(in));
-        if (hasAggs = in.readBoolean()) {
-            aggregations = new InternalAggregations(in);
-        }
-        if (in.getVersion().before(Version.V_7_2_0)) {
-            List<SiblingPipelineAggregator> pipelineAggregators = in.readNamedWriteableList(PipelineAggregator.class).stream()
-                .map(a -> (SiblingPipelineAggregator) a).collect(Collectors.toList());
-            if (hasAggs && pipelineAggregators.isEmpty() == false) {
-                List<InternalAggregation> internalAggs = aggregations.asList().stream()
-                    .map(agg -> (InternalAggregation) agg).collect(Collectors.toList());
-                //Earlier versions serialize sibling pipeline aggs separately as they used to be set to QuerySearchResult directly, while
-                //later versions include them in InternalAggregations. Note that despite serializing sibling pipeline aggs as part of
-                //InternalAggregations is supported since 6.7.0, the shards set sibling pipeline aggs to InternalAggregations only from 7.1.
-                this.aggregations = new InternalAggregations(internalAggs, pipelineAggregators);
+        if (in.getVersion().before(Version.V_7_7_0)) {
+            if (hasAggs = in.readBoolean()) {
+                aggregations = DelayableWriteable.referencing(new InternalAggregations(in));
+            }
+            if (in.getVersion().before(Version.V_7_2_0)) {
+                // The list of PipelineAggregators is sent by old versions. We don't need it anyway.
+                in.readNamedWriteableList(PipelineAggregator.class);
+            }
+        } else {
+            if (hasAggs = in.readBoolean()) {
+                aggregations = DelayableWriteable.delayed(InternalAggregations::new, in);
             }
         }
         if (in.readBoolean()) {
@@ -371,18 +373,37 @@ public void writeToNoId(StreamOutput out) throws IOException {
         writeTopDocs(out, topDocsAndMaxScore);
         if (aggregations == null) {
             out.writeBoolean(false);
+            if (out.getVersion().before(Version.V_7_2_0)) {
+                /*
+                 * Earlier versions expect sibling pipeline aggs separately
+                 * as they used to be set to QuerySearchResult directly, while
+                 * later versions expect them in InternalAggregations. Note
+                 * that despite serializing sibling pipeline aggs as part of
+                 * InternalAggregations is supported since 6.7.0, the shards
+                 * set sibling pipeline aggs to InternalAggregations only from
+                 * 7.1 on.
+                 */
+                out.writeNamedWriteableList(emptyList());
+            }
         } else {
             out.writeBoolean(true);
-            aggregations.writeTo(out);
-        }
-        if (out.getVersion().before(Version.V_7_2_0)) {
-            //Earlier versions expect sibling pipeline aggs separately as they used to be set to QuerySearchResult directly,
-            //while later versions expect them in InternalAggregations. Note that despite serializing sibling pipeline aggs as part of
-            //InternalAggregations is supported since 6.7.0, the shards set sibling pipeline aggs to InternalAggregations only from 7.1 on.
-            if (aggregations == null) {
-                out.writeNamedWriteableList(Collections.emptyList());
+            if (out.getVersion().before(Version.V_7_7_0)) {
+                InternalAggregations aggs = aggregations.get();
+                aggs.writeTo(out);
+                if (out.getVersion().before(Version.V_7_2_0)) {
+                    /*
+                     * Earlier versions expect sibling pipeline aggs separately
+                     * as they used to be set to QuerySearchResult directly, while
+                     * later versions expect them in InternalAggregations. Note
+                     * that despite serializing sibling pipeline aggs as part of
+                     * InternalAggregations is supported since 6.7.0, the shards
+                     * set sibling pipeline aggs to InternalAggregations only from
+                     * 7.1 on.
+                     */
+                    out.writeNamedWriteableList(aggs.getTopLevelPipelineAggregators());
+                }
             } else {
-                out.writeNamedWriteableList(aggregations.getTopLevelPipelineAggregators());
+                aggregations.writeTo(out);
             }
         }
         if (suggest == null) {
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index 33ec18da99660..a39c2691a9db4 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -23,6 +23,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.index.IndexCommit;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -55,6 +56,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
@@ -340,8 +342,9 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde
             try {
                 // we flush first to make sure we get the latest writes snapshotted
                 snapshotRef = indexShard.acquireLastIndexCommit(true);
+                final IndexCommit snapshotIndexCommit = snapshotRef.getIndexCommit();
                 repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId,
-                    snapshotRef.getIndexCommit(), snapshotStatus, version, userMetadata,
+                    snapshotRef.getIndexCommit(), getShardStateId(indexShard, snapshotIndexCommit), snapshotStatus, version, userMetadata,
                     ActionListener.runBefore(listener, snapshotRef::close));
             } catch (Exception e) {
                 IOUtils.close(snapshotRef);
@@ -352,6 +355,31 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde
         }
     }
 
+    /**
+     * Generates an identifier from the current state of a shard that can be used to detect whether a shard's contents
+     * have changed between two snapshots.
+     * A shard is assumed to have unchanged contents if its global- and local checkpoint are equal, its maximum
+     * sequence number has not changed and its history- and force-merge-uuid have not changed.
+     * The method returns {@code null} if global and local checkpoint are different for a shard since no safe unique
+     * shard state id can be used in this case because of the possibility of a primary failover leading to different
+     * shard content for the same sequence number on a subsequent snapshot.
+     *
+     * @param indexShard          Shard
+     * @param snapshotIndexCommit IndexCommit for shard
+     * @return shard state id or {@code null} if none can be used
+     */
+    @Nullable
+    private static String getShardStateId(IndexShard indexShard, IndexCommit snapshotIndexCommit) throws IOException {
+        final Map<String, String> userCommitData = snapshotIndexCommit.getUserData();
+        final SequenceNumbers.CommitInfo seqNumInfo = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userCommitData.entrySet());
+        final long maxSeqNo = seqNumInfo.maxSeqNo;
+        if (maxSeqNo != seqNumInfo.localCheckpoint || maxSeqNo != indexShard.getLastSyncedGlobalCheckpoint()) {
+            return null;
+        }
+        return userCommitData.get(Engine.HISTORY_UUID_KEY) + "-" +
+            userCommitData.getOrDefault(Engine.FORCE_MERGE_UUID_KEY, "na") + "-" + maxSeqNo;
+    }
+
     /**
      * Checks if any shards were processed that the new master doesn't know about
      */
diff --git a/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java b/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java
index edf42cc4546b2..29a374077177d 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java
@@ -83,17 +83,7 @@ static boolean awaitTermination(final ScheduledThreadPoolExecutor scheduledThrea
     }
 
     /**
-     * Does nothing by default but can be used by subclasses to save the current thread context and wraps the command in a Runnable
-     * that restores that context before running the command.
-     */
-    default Runnable preserveContext(Runnable command) {
-        return command;
-    }
-
-    /**
-     * Schedules a one-shot command to be run after a given delay. The command is not run in the context of the calling thread.
-     * To preserve the context of the calling thread you may call {@link #preserveContext(Runnable)} on the runnable before passing
-     * it to this method.
+     * Schedules a one-shot command to be run after a given delay. The command is run in the context of the calling thread.
      * The command runs on scheduler thread. Do not run blocking calls on the scheduler thread. Subclasses may allow
      * to execute on a different executor, in which case blocking calls are allowed.
      *
diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 5a2b83c67222d..99e3143f315e6 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -323,9 +323,7 @@ public ExecutorService executor(String name) {
     }
 
     /**
-     * Schedules a one-shot command to run after a given delay. The command is not run in the context of the calling thread. To preserve the
-     * context of the calling thread you may call <code>threadPool.getThreadContext().preserveContext</code> on the runnable before passing
-     * it to this method.
+     * Schedules a one-shot command to run after a given delay. The command is run in the context of the calling thread.
      *
      * @param command the command to run
      * @param delay delay before the task executes
@@ -339,6 +337,7 @@ public ExecutorService executor(String name) {
      */
     @Override
     public ScheduledCancellable schedule(Runnable command, TimeValue delay, String executor) {
+        command = threadContext.preserveContext(command);
         if (!Names.SAME.equals(executor)) {
             command = new ThreadedRunnable(command, executor(executor));
         }
@@ -371,11 +370,6 @@ public Cancellable scheduleWithFixedDelay(Runnable command, TimeValue interval,
                         command, executor), e));
     }
 
-    @Override
-    public Runnable preserveContext(Runnable command) {
-        return getThreadContext().preserveContext(command);
-    }
-
     protected final void stopCachedTimeThread() {
         cachedTimeThread.running = false;
         cachedTimeThread.interrupt();
diff --git a/server/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java b/server/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java
index 7cb60a4326aa5..4116f88b14224 100644
--- a/server/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java
+++ b/server/src/main/java/org/elasticsearch/transport/CompressibleBytesOutputStream.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
@@ -103,14 +102,4 @@ public void close() throws IOException {
     public void reset() throws IOException {
         throw new UnsupportedOperationException();
     }
-
-    @Override
-    public Version getVersion() {
-        return stream.getVersion();
-    }
-
-    @Override
-    public void setVersion(Version version) {
-        stream.setVersion(version);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/InboundMessage.java b/server/src/main/java/org/elasticsearch/transport/InboundMessage.java
index 14db21bc86ec4..7aa0b1b6368f7 100644
--- a/server/src/main/java/org/elasticsearch/transport/InboundMessage.java
+++ b/server/src/main/java/org/elasticsearch/transport/InboundMessage.java
@@ -61,8 +61,7 @@ InboundMessage deserialize(BytesReference reference) throws IOException {
             try (ThreadContext.StoredContext existing = threadContext.stashContext()) {
                 long requestId = streamInput.readLong();
                 byte status = streamInput.readByte();
-                final Version remoteVersion = Version.fromId(streamInput.readInt());
-                streamInput.setVersion(remoteVersion);
+                Version remoteVersion = Version.fromId(streamInput.readInt());
                 final boolean isHandshake = TransportStatus.isHandshake(status);
                 ensureVersionCompatibility(remoteVersion, version, isHandshake);
 
@@ -70,11 +69,10 @@ InboundMessage deserialize(BytesReference reference) throws IOException {
                     // Consume the variable header size
                     streamInput.readInt();
                 } else {
-                    streamInput = decompressingStream(status, streamInput);
-                    assertRemoteVersion(streamInput, remoteVersion);
+                    streamInput = decompressingStream(status, remoteVersion, streamInput);
                 }
 
-                threadContext.readFrom(streamInput);
+                threadContext.readHeaders(streamInput);
 
                 InboundMessage message;
                 if (TransportStatus.isRequest(status)) {
@@ -85,18 +83,15 @@ InboundMessage deserialize(BytesReference reference) throws IOException {
                     final String action = streamInput.readString();
 
                     if (remoteVersion.onOrAfter(TcpHeader.VERSION_WITH_HEADER_SIZE)) {
-                        streamInput = decompressingStream(status, streamInput);
-                        assertRemoteVersion(streamInput, remoteVersion);
+                        streamInput = decompressingStream(status, remoteVersion, streamInput);
                     }
-                    streamInput = namedWriteableStream(streamInput);
+                    streamInput = namedWriteableStream(streamInput, remoteVersion);
                     message = new Request(threadContext, remoteVersion, status, requestId, action, streamInput);
                 } else {
                     if (remoteVersion.onOrAfter(TcpHeader.VERSION_WITH_HEADER_SIZE)) {
-                        streamInput = decompressingStream(status, streamInput);
-                        assertRemoteVersion(streamInput, remoteVersion);
+                        streamInput = decompressingStream(status, remoteVersion, streamInput);
                     }
-                    streamInput = namedWriteableStream(streamInput);
-                    assertRemoteVersion(streamInput, remoteVersion);
+                    streamInput = namedWriteableStream(streamInput, remoteVersion);
                     message = new Response(threadContext, remoteVersion, status, requestId, streamInput);
                 }
                 success = true;
@@ -108,10 +103,12 @@ InboundMessage deserialize(BytesReference reference) throws IOException {
             }
         }
 
-        static StreamInput decompressingStream(byte status, StreamInput streamInput) throws IOException {
+        static StreamInput decompressingStream(byte status, Version remoteVersion, StreamInput streamInput) throws IOException {
             if (TransportStatus.isCompress(status) && streamInput.available() > 0) {
                 try {
-                    return CompressorFactory.COMPRESSOR.streamInput(streamInput);
+                    StreamInput decompressor = CompressorFactory.COMPRESSOR.streamInput(streamInput);
+                    decompressor.setVersion(remoteVersion);
+                    return decompressor;
                 } catch (IllegalArgumentException e) {
                     throw new IllegalStateException("stream marked as compressed, but is missing deflate header");
                 }
@@ -120,12 +117,10 @@ static StreamInput decompressingStream(byte status, StreamInput streamInput) thr
             }
         }
 
-        private StreamInput namedWriteableStream(StreamInput delegate) {
-            return new NamedWriteableAwareStreamInput(delegate, namedWriteableRegistry);
-        }
-
-        static void assertRemoteVersion(StreamInput in, Version version) {
-            assert version.equals(in.getVersion()) : "Stream version [" + in.getVersion() + "] does not match version [" + version + "]";
+        private StreamInput namedWriteableStream(StreamInput delegate, Version remoteVersion) {
+            NamedWriteableAwareStreamInput streamInput = new NamedWriteableAwareStreamInput(delegate, namedWriteableRegistry);
+            streamInput.setVersion(remoteVersion);
+            return streamInput;
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java b/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java
index bae4cb0d982d5..16ba7a15a539c 100644
--- a/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java
+++ b/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java
@@ -55,8 +55,7 @@ BytesReference serialize(BytesStreamOutput bytesStream) throws IOException {
         }
 
         try (CompressibleBytesOutputStream stream = new CompressibleBytesOutputStream(bytesStream, TransportStatus.isCompress(status))) {
-            assert stream.getVersion().equals(version) :
-                "Stream version [" + stream.getVersion() + "] does not match version [" + version + "]";
+            stream.setVersion(version);
             if (variableHeaderLength == -1) {
                 writeVariableHeader(stream);
             }
diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
index dc721bf6cde26..c1f79722d29db 100644
--- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
@@ -157,7 +157,9 @@ protected boolean shouldOpenMoreConnections() {
     protected boolean strategyMustBeRebuilt(Settings newSettings) {
         String address = PROXY_ADDRESS.getConcreteSettingForNamespace(clusterAlias).get(newSettings);
         int numOfSockets = REMOTE_SOCKET_CONNECTIONS.getConcreteSettingForNamespace(clusterAlias).get(newSettings);
-        return numOfSockets != maxNumConnections || configuredAddress.equals(address) == false;
+        String serverName = SERVER_NAME.getConcreteSettingForNamespace(clusterAlias).get(newSettings);
+        return numOfSockets != maxNumConnections || configuredAddress.equals(address) == false ||
+            Objects.equals(serverName, configuredServerName) == false;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
index 1d6b6a12af0c0..745496b98461d 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -36,6 +37,7 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
@@ -62,7 +64,7 @@
  */
 public final class RemoteClusterService extends RemoteClusterAware implements Closeable {
 
-    private static final Logger logger = LogManager.getLogger(RemoteClusterService.class);
+    private final Logger logger = LogManager.getLogger(RemoteClusterService.class);
 
     /**
      * The initial connect timeout for remote cluster connections
@@ -82,17 +84,6 @@ public final class RemoteClusterService extends RemoteClusterAware implements Cl
     public static final Setting<String> REMOTE_NODE_ATTRIBUTE =
         Setting.simpleString("cluster.remote.node.attr", Setting.Property.NodeScope);
 
-    /**
-     * If <code>true</code> connecting to remote clusters is supported on this node. If <code>false</code> this node will not establish
-     * connections to any remote clusters configured. Search requests executed against this node (where this node is the coordinating node)
-     * will fail if remote cluster syntax is used as an index pattern. The default is <code>true</code>
-     */
-    public static final Setting<Boolean> ENABLE_REMOTE_CLUSTERS =
-        Setting.boolSetting(
-            "cluster.remote.connect",
-            true,
-            Setting.Property.NodeScope);
-
     public static final Setting.AffixSetting<Boolean> REMOTE_CLUSTER_SKIP_UNAVAILABLE =
         Setting.affixKeySetting(
             "cluster.remote.",
@@ -116,11 +107,18 @@ public final class RemoteClusterService extends RemoteClusterAware implements Cl
         (ns, key) -> boolSetting(key, TransportSettings.TRANSPORT_COMPRESS,
             new RemoteConnectionEnabled<>(ns, key), Setting.Property.Dynamic, Setting.Property.NodeScope));
 
+    private final boolean enabled;
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
     private final TransportService transportService;
     private final Map<String, RemoteClusterConnection> remoteClusters = ConcurrentCollections.newConcurrentMap();
 
     RemoteClusterService(Settings settings, TransportService transportService) {
         super(settings);
+        this.enabled = Node.NODE_REMOTE_CLUSTER_CLIENT.get(settings);
         this.transportService = transportService;
     }
 
@@ -200,6 +198,10 @@ public Transport.Connection getConnection(String cluster) {
     }
 
     RemoteClusterConnection getRemoteClusterConnection(String cluster) {
+        if (enabled == false) {
+            throw new IllegalArgumentException(
+                "this node does not have the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role");
+        }
         RemoteClusterConnection connection = remoteClusters.get(cluster);
         if (connection == null) {
             throw new NoSuchRemoteClusterException(cluster);
@@ -336,6 +338,10 @@ public Stream<RemoteConnectionInfo> getRemoteConnectionInfos() {
      * function on success.
      */
     public void collectNodes(Set<String> clusters, ActionListener<BiFunction<String, String, DiscoveryNode>> listener) {
+        if (enabled == false) {
+            throw new IllegalArgumentException(
+                "this node does not have the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role");
+        }
         Map<String, RemoteClusterConnection> remoteClusters = this.remoteClusters;
         for (String cluster : clusters) {
             if (remoteClusters.containsKey(cluster) == false) {
@@ -379,6 +385,10 @@ public void onFailure(Exception e) {
      * @throws IllegalArgumentException if the given clusterAlias doesn't exist
      */
     public Client getRemoteClusterClient(ThreadPool threadPool, String clusterAlias) {
+        if (transportService.getRemoteClusterService().isEnabled() == false) {
+            throw new IllegalArgumentException(
+                "this node does not have the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role");
+        }
         if (transportService.getRemoteClusterService().getRemoteClusterNames().contains(clusterAlias) == false) {
             throw new NoSuchRemoteClusterException(clusterAlias);
         }
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportLogger.java b/server/src/main/java/org/elasticsearch/transport/TransportLogger.java
index 7543ecfa304d4..310819969b718 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportLogger.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportLogger.java
@@ -28,8 +28,6 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.transport.InboundMessage.Reader.assertRemoteVersion;
-
 public final class TransportLogger {
 
     private static final Logger logger = LogManager.getLogger(TransportLogger.class);
@@ -77,8 +75,7 @@ private static String format(TcpChannel channel, BytesReference message, String
                 final byte status = streamInput.readByte();
                 final boolean isRequest = TransportStatus.isRequest(status);
                 final String type = isRequest ? "request" : "response";
-                final Version version = Version.fromId(streamInput.readInt());
-                streamInput.setVersion(version);
+                Version version = Version.fromId(streamInput.readInt());
                 sb.append(" [length: ").append(messageLengthWithHeader);
                 sb.append(", request id: ").append(requestId);
                 sb.append(", type: ").append(type);
@@ -87,18 +84,11 @@ private static String format(TcpChannel channel, BytesReference message, String
                 if (version.onOrAfter(TcpHeader.VERSION_WITH_HEADER_SIZE)) {
                     sb.append(", header size: ").append(streamInput.readInt()).append('B');
                 } else {
-                    streamInput = InboundMessage.Reader.decompressingStream(status, streamInput);
-                    assertRemoteVersion(streamInput, version);
+                    streamInput = InboundMessage.Reader.decompressingStream(status, version, streamInput);
                 }
 
-                // TODO (jaymode) Need a better way to deal with this. In one aspect,
-                // changes were made to ThreadContext to allocate less internally, yet we have this
-                // ugliness needed to move past the threadcontext data in the stream and discard it
-                // Could we have an alternative that essentially just seeks through the stream with
-                // minimal allocation?
-                // read and discard thread context data
+                // read and discard headers
                 ThreadContext.readHeadersFromStream(streamInput);
-                ThreadContext.readAllowedSystemIndices(streamInput);
 
                 if (isRequest) {
                     if (version.before(Version.V_8_0_0)) {
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java
index dcf9b3595c80e..4f962a334b8a1 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
@@ -81,7 +82,7 @@ public class TransportService extends AbstractLifecycleComponent implements Tran
     protected final TaskManager taskManager;
     private final TransportInterceptor.AsyncSender asyncSender;
     private final Function<BoundTransportAddress, DiscoveryNode> localNodeFactory;
-    private final boolean connectToRemoteCluster;
+    private final boolean remoteClusterClient;
     private final Transport.ResponseHandlers responseHandlers;
     private final TransportInterceptor interceptor;
 
@@ -161,13 +162,13 @@ public TransportService(Settings settings, Transport transport, ThreadPool threa
         taskManager = createTaskManager(settings, threadPool, taskHeaders);
         this.interceptor = transportInterceptor;
         this.asyncSender = interceptor.interceptSender(this::sendRequestInternal);
-        this.connectToRemoteCluster = RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings);
+        this.remoteClusterClient = Node.NODE_REMOTE_CLUSTER_CLIENT.get(settings);
         remoteClusterService = new RemoteClusterService(settings, this);
         responseHandlers = transport.getResponseHandlers();
         if (clusterSettings != null) {
             clusterSettings.addSettingsUpdateConsumer(TransportSettings.TRACE_LOG_INCLUDE_SETTING, this::setTracerLogInclude);
             clusterSettings.addSettingsUpdateConsumer(TransportSettings.TRACE_LOG_EXCLUDE_SETTING, this::setTracerLogExclude);
-            if (connectToRemoteCluster) {
+            if (remoteClusterClient) {
                 remoteClusterService.listenForUpdates(clusterSettings);
             }
         }
@@ -227,7 +228,7 @@ protected void doStart() {
         localNode = localNodeFactory.apply(transport.boundAddress());
         transport.setLocalNode(localNode);
 
-        if (connectToRemoteCluster) {
+        if (remoteClusterClient) {
             // here we start to connect to the remote clusters
             remoteClusterService.initializeRemoteClusters();
         }
diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
index 61118d31b3a2a..114abe0b2dcb7 100644
--- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -109,7 +109,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() {
         UsageService usageService = new UsageService();
         ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
                 settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
-                null, usageService);
+                null, usageService, null);
         actionModule.initRestHandlers(null);
         // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
         Exception e = expectThrows(IllegalArgumentException.class, () ->
@@ -141,7 +141,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
             UsageService usageService = new UsageService();
             ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
                     settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
-                    singletonList(dupsMainAction), null, null, usageService);
+                    singletonList(dupsMainAction), null, null, usageService, null);
             Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
             assertThat(e.getMessage(), startsWith("Cannot replace existing handler for [/] for method: GET"));
         } finally {
@@ -175,7 +175,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
             UsageService usageService = new UsageService();
             ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
                     settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
-                    singletonList(registersFakeHandler), null, null, usageService);
+                    singletonList(registersFakeHandler), null, null, usageService, null);
             actionModule.initRestHandlers(null);
             // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
             Exception e = expectThrows(IllegalArgumentException.class, () ->
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestTests.java
index 7e94454102480..e0506c664e709 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestTests.java
@@ -23,7 +23,13 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.in;
+import static org.hamcrest.Matchers.not;
 
 /**
  * Granular tests for the {@link NodesInfoRequest} class. Higher-level tests
@@ -34,82 +40,94 @@ public class NodesInfoRequestTests extends ESTestCase {
     /**
      * Make sure that we can set, serialize, and deserialize arbitrary sets
      * of metrics.
-     *
-     * TODO: Once we can set values by string, use a collection rather than
-     *   checking each and every setter in the public API
      */
-    public void testMetricsSetters() throws Exception {
+    public void testAddMetricsSet() throws Exception {
+        NodesInfoRequest request = new NodesInfoRequest(randomAlphaOfLength(8));
+        randomSubsetOf(NodesInfoRequest.Metric.allMetrics()).forEach(request::addMetric);
+        NodesInfoRequest deserializedRequest = roundTripRequest(request);
+        assertThat(request.requestedMetrics(), equalTo(deserializedRequest.requestedMetrics()));
+    }
+
+    /**
+     * Check that we can add a metric.
+     */
+    public void testAddSingleMetric() throws Exception {
+        NodesInfoRequest request = new NodesInfoRequest(randomAlphaOfLength(8));
+        request.addMetric(randomFrom(NodesInfoRequest.Metric.allMetrics()));
+        NodesInfoRequest deserializedRequest = roundTripRequest(request);
+        assertThat(request.requestedMetrics(), equalTo(deserializedRequest.requestedMetrics()));
+    }
+
+    /**
+     * Check that we can remove a metric.
+     */
+    public void testRemoveSingleMetric() throws Exception {
         NodesInfoRequest request = new NodesInfoRequest(randomAlphaOfLength(8));
-        request.settings(randomBoolean());
-        request.os(randomBoolean());
-        request.process(randomBoolean());
-        request.jvm(randomBoolean());
-        request.threadPool(randomBoolean());
-        request.transport(randomBoolean());
-        request.http(randomBoolean());
-        request.plugins(randomBoolean());
-        request.ingest(randomBoolean());
-        request.indices(randomBoolean());
+        request.all();
+        String metric = randomFrom(NodesInfoRequest.Metric.allMetrics());
+        request.removeMetric(metric);
+
         NodesInfoRequest deserializedRequest = roundTripRequest(request);
-        assertRequestsEqual(request, deserializedRequest);
+        assertThat(request.requestedMetrics(), equalTo(deserializedRequest.requestedMetrics()));
+        assertThat(metric, not(in(request.requestedMetrics())));
     }
 
     /**
      * Test that a newly constructed NodesInfoRequestObject requests all of the
-     * possible metrics defined in {@link NodesInfoRequest.Metrics}.
+     * possible metrics defined in {@link NodesInfoRequest.Metric}.
      */
     public void testNodesInfoRequestDefaults() {
         NodesInfoRequest defaultNodesInfoRequest = new NodesInfoRequest(randomAlphaOfLength(8));
         NodesInfoRequest allMetricsNodesInfoRequest = new NodesInfoRequest(randomAlphaOfLength(8));
         allMetricsNodesInfoRequest.all();
 
-        assertRequestsEqual(defaultNodesInfoRequest, allMetricsNodesInfoRequest);
+        assertThat(defaultNodesInfoRequest.requestedMetrics(), equalTo(allMetricsNodesInfoRequest.requestedMetrics()));
     }
 
     /**
-     * Test that the {@link NodesInfoRequest#all()} method sets all of the
-     * metrics to {@code true}.
-     *
-     * TODO: Once we can check values by string, use a collection rather than
-     *   checking each and every getter in the public API
+     * Test that the {@link NodesInfoRequest#all()} method enables all metrics.
      */
     public void testNodesInfoRequestAll() throws Exception {
         NodesInfoRequest request = new NodesInfoRequest("node");
         request.all();
 
-        assertTrue(request.settings());
-        assertTrue(request.os());
-        assertTrue(request.process());
-        assertTrue(request.jvm());
-        assertTrue(request.threadPool());
-        assertTrue(request.transport());
-        assertTrue(request.http());
-        assertTrue(request.plugins());
-        assertTrue(request.ingest());
-        assertTrue(request.indices());
+        assertThat(request.requestedMetrics(), equalTo(NodesInfoRequest.Metric.allMetrics()));
     }
 
     /**
-     * Test that the {@link NodesInfoRequest#clear()} method sets all of the
-     * metrics to {@code false}.
-     *
-     * TODO: Once we can check values by string, use a collection rather than
-     *   checking each and every getter in the public API
+     * Test that the {@link NodesInfoRequest#clear()} method disables all metrics.
      */
     public void testNodesInfoRequestClear() throws Exception {
         NodesInfoRequest request = new NodesInfoRequest("node");
         request.clear();
 
-        assertFalse(request.settings());
-        assertFalse(request.os());
-        assertFalse(request.process());
-        assertFalse(request.jvm());
-        assertFalse(request.threadPool());
-        assertFalse(request.transport());
-        assertFalse(request.http());
-        assertFalse(request.plugins());
-        assertFalse(request.ingest());
-        assertFalse(request.indices());
+        assertThat(request.requestedMetrics(), empty());
+    }
+
+    /**
+     * Test that (for now) we can only add metrics from a set of known metrics.
+     */
+    public void testUnknownMetricsRejected() {
+        String unknownMetric1 = "unknown_metric1";
+        String unknownMetric2 = "unknown_metric2";
+        Set<String> unknownMetrics = new HashSet<>();
+        unknownMetrics.add(unknownMetric1);
+        unknownMetrics.addAll(randomSubsetOf(NodesInfoRequest.Metric.allMetrics()));
+
+        NodesInfoRequest request = new NodesInfoRequest();
+
+        IllegalStateException exception = expectThrows(IllegalStateException.class, () -> request.addMetric(unknownMetric1));
+        assertThat(exception.getMessage(), equalTo("Used an illegal metric: " + unknownMetric1));
+
+        exception = expectThrows(IllegalStateException.class, () -> request.removeMetric(unknownMetric1));
+        assertThat(exception.getMessage(), equalTo("Used an illegal metric: " + unknownMetric1));
+
+        exception = expectThrows(IllegalStateException.class, () -> request.addMetrics(unknownMetrics.toArray(String[]::new)));
+        assertThat(exception.getMessage(), equalTo("Used illegal metric: [" + unknownMetric1 + "]"));
+
+        unknownMetrics.add(unknownMetric2);
+        exception = expectThrows(IllegalStateException.class, () -> request.addMetrics(unknownMetrics.toArray(String[]::new)));
+        assertThat(exception.getMessage(), equalTo("Used illegal metrics: [" + unknownMetric1 + ", " + unknownMetric2 + "]"));
     }
 
     /**
@@ -125,20 +143,4 @@ private static NodesInfoRequest roundTripRequest(NodesInfoRequest request) throw
             }
         }
     }
-
-    private static void assertRequestsEqual(NodesInfoRequest request1, NodesInfoRequest request2) {
-
-        // TODO: Once we can check values by string, use a collection rather than
-        //   checking each and every getter in the public API
-        assertThat(request1.settings(), equalTo(request2.settings()));
-        assertThat(request1.os(), equalTo(request2.os()));
-        assertThat(request1.process(), equalTo(request2.process()));
-        assertThat(request1.jvm(), equalTo(request2.jvm()));
-        assertThat(request1.threadPool(), equalTo(request2.threadPool()));
-        assertThat(request1.transport(), equalTo(request2.transport()));
-        assertThat(request1.http(), equalTo(request2.http()));
-        assertThat(request1.plugins(), equalTo(request2.plugins()));
-        assertThat(request1.ingest(), equalTo(request2.ingest()));
-        assertThat(request1.indices(), equalTo(request2.indices()));
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
index 1999a18a92b31..0a9083fb22b30 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
@@ -84,6 +84,7 @@ public void testToXContent() throws IOException {
                     "    },\n" +
                     "    \"metadata\" : {\n" +
                     "      \"cluster_uuid\" : \"_na_\",\n" +
+                    "      \"cluster_uuid_committed\" : false,\n" +
                     "      \"cluster_coordination\" : {\n" +
                     "        \"term\" : 0,\n" +
                     "        \"last_committed_config\" : [ ],\n" +
@@ -93,6 +94,11 @@ public void testToXContent() throws IOException {
                     "      \"templates\" : { },\n" +
                     "      \"indices\" : {\n" +
                     "        \"index\" : {\n" +
+                    "          \"version\" : 1,\n" +
+                    "          \"mapping_version\" : 1,\n" +
+                    "          \"settings_version\" : 1,\n" +
+                    "          \"aliases_version\" : 1,\n" +
+                    "          \"routing_num_shards\" : 1,\n" +
                     "          \"state\" : \"open\",\n" +
                     "          \"settings\" : {\n" +
                     "            \"index\" : {\n" +
@@ -114,7 +120,8 @@ public void testToXContent() throws IOException {
                     "          },\n" +
                     "          \"in_sync_allocations\" : {\n" +
                     "            \"0\" : [ ]\n" +
-                    "          }\n" +
+                    "          },\n" +
+                    "          \"rollover_info\" : { }\n" +
                     "        }\n" +
                     "      },\n" +
                     "      \"index-graveyard\" : {\n" +
@@ -179,6 +186,7 @@ public void testToXContent() throws IOException {
                     "    \"cluster_uuid\" : \"_na_\",\n" +
                     "    \"metadata\" : {\n" +
                     "      \"cluster_uuid\" : \"_na_\",\n" +
+                    "      \"cluster_uuid_committed\" : false,\n" +
                     "      \"cluster_coordination\" : {\n" +
                     "        \"term\" : 0,\n" +
                     "        \"last_committed_config\" : [ ],\n" +
@@ -188,6 +196,11 @@ public void testToXContent() throws IOException {
                     "      \"templates\" : { },\n" +
                     "      \"indices\" : {\n" +
                     "        \"index\" : {\n" +
+                    "          \"version\" : 1,\n" +
+                    "          \"mapping_version\" : 1,\n" +
+                    "          \"settings_version\" : 1,\n" +
+                    "          \"aliases_version\" : 1,\n" +
+                    "          \"routing_num_shards\" : 1,\n" +
                     "          \"state\" : \"open\",\n" +
                     "          \"settings\" : {\n" +
                     "            \"index\" : {\n" +
@@ -204,7 +217,8 @@ public void testToXContent() throws IOException {
                     "          },\n" +
                     "          \"in_sync_allocations\" : {\n" +
                     "            \"0\" : [ ]\n" +
-                    "          }\n" +
+                    "          },\n" +
+                    "          \"rollover_info\" : { }\n" +
                     "        }\n" +
                     "      },\n" +
                     "      \"index-graveyard\" : {\n" +
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
index 2ca10f9b34519..e5126203b988d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
@@ -66,6 +66,7 @@ public void testNodeCounts() {
         expectedCounts.put(DiscoveryNodeRole.DATA_ROLE.roleName(), 1);
         expectedCounts.put(DiscoveryNodeRole.MASTER_ROLE.roleName(), 1);
         expectedCounts.put(DiscoveryNodeRole.INGEST_ROLE.roleName(), 1);
+        expectedCounts.put(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName(), 1);
         expectedCounts.put(ClusterStatsNodes.Counts.COORDINATING_ONLY, 0);
         int numNodes = randomIntBetween(1, 5);
 
@@ -76,9 +77,13 @@ public void testNodeCounts() {
             boolean isDataNode = randomBoolean();
             boolean isMasterNode = randomBoolean();
             boolean isIngestNode = randomBoolean();
-            Settings settings = Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), isDataNode)
-                    .put(Node.NODE_MASTER_SETTING.getKey(), isMasterNode).put(Node.NODE_INGEST_SETTING.getKey(), isIngestNode)
-                    .build();
+            boolean isRemoteClusterClientNode = randomBoolean();
+            Settings settings = Settings.builder()
+                .put(Node.NODE_DATA_SETTING.getKey(), isDataNode)
+                .put(Node.NODE_MASTER_SETTING.getKey(), isMasterNode)
+                .put(Node.NODE_INGEST_SETTING.getKey(), isIngestNode)
+                .put(Node.NODE_REMOTE_CLUSTER_CLIENT.getKey(), isRemoteClusterClientNode)
+                .build();
             internalCluster().startNode(settings);
             total++;
             waitForNodes(total);
@@ -92,7 +97,10 @@ public void testNodeCounts() {
             if (isIngestNode) {
                 incrementCountForRole(DiscoveryNodeRole.INGEST_ROLE.roleName(), expectedCounts);
             }
-            if (!isDataNode && !isMasterNode && !isIngestNode) {
+            if (isRemoteClusterClientNode) {
+                incrementCountForRole(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName(), expectedCounts);
+            }
+            if (!isDataNode && !isMasterNode && !isIngestNode && !isRemoteClusterClientNode) {
                 incrementCountForRole(ClusterStatsNodes.Counts.COORDINATING_ONLY, expectedCounts);
             }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java
index d6a846c205fb3..6945811c8df23 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java
@@ -18,10 +18,21 @@
  */
 package org.elasticsearch.action.admin.indices.datastream;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction.Request;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
 public class CreateDataStreamRequestTests extends AbstractWireSerializingTestCase<Request> {
 
     @Override
@@ -35,4 +46,40 @@ protected Request createTestInstance() {
         request.setTimestampFieldName(randomAlphaOfLength(8));
         return request;
     }
+
+    public void testValidateRequest() {
+        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request("my-data-stream");
+        req.setTimestampFieldName("my-timestamp-field");
+        ActionRequestValidationException e = req.validate();
+        assertNull(e);
+    }
+
+    public void testValidateRequestWithoutTimestampField() {
+        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request("my-data-stream");
+        ActionRequestValidationException e = req.validate();
+        assertNotNull(e);
+        assertThat(e.validationErrors().size(), equalTo(1));
+        assertThat(e.validationErrors().get(0), containsString("timestamp field name is missing"));
+    }
+
+    public void testCreateDataStream() {
+        final String dataStreamName = "my-data-stream";
+        ClusterState cs = ClusterState.builder(new ClusterName("_name")).build();
+        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(dataStreamName);
+        ClusterState newState = CreateDataStreamAction.TransportAction.createDataStream(cs, req);
+        assertThat(newState.metaData().dataStreams().size(), equalTo(1));
+        assertThat(newState.metaData().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName));
+    }
+
+    public void testCreateDuplicateDataStream() {
+        final String dataStreamName = "my-data-stream";
+        DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", Collections.emptyList());
+        ClusterState cs = ClusterState.builder(new ClusterName("_name"))
+            .metaData(MetaData.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build();
+        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(dataStreamName);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CreateDataStreamAction.TransportAction.createDataStream(cs, req));
+        assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] already exists"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java
index f460065699795..42a961d828f4e 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java
@@ -18,10 +18,22 @@
  */
 package org.elasticsearch.action.admin.indices.datastream;
 
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction.Request;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
 public class DeleteDataStreamRequestTests extends AbstractWireSerializingTestCase<Request> {
 
     @Override
@@ -33,4 +45,38 @@ protected Writeable.Reader<Request> instanceReader() {
     protected Request createTestInstance() {
         return new Request(randomAlphaOfLength(8));
     }
+
+    public void testValidateRequest() {
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request("my-data-stream");
+        ActionRequestValidationException e = req.validate();
+        assertNull(e);
+    }
+
+    public void testValidateRequestWithoutName() {
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request("");
+        ActionRequestValidationException e = req.validate();
+        assertNotNull(e);
+        assertThat(e.validationErrors().size(), equalTo(1));
+        assertThat(e.validationErrors().get(0), containsString("name is missing"));
+    }
+
+    public void testDeleteDataStream() {
+        final String dataStreamName = "my-data-stream";
+        DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", Collections.emptyList());
+        ClusterState cs = ClusterState.builder(new ClusterName("_name"))
+            .metaData(MetaData.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build();
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(dataStreamName);
+
+        ClusterState newState = DeleteDataStreamAction.TransportAction.removeDataStream(cs, req);
+        assertThat(newState.metaData().dataStreams().size(), equalTo(0));
+    }
+
+    public void testDeleteNonexistentDataStream() {
+        final String dataStreamName = "my-data-stream";
+        ClusterState cs = ClusterState.builder(new ClusterName("_name")).build();
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(dataStreamName);
+        ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class,
+            () -> DeleteDataStreamAction.TransportAction.removeDataStream(cs, req));
+        assertThat(e.getMessage(), containsString("data_streams matching [" + dataStreamName + "] not found"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java
index 062bdef629cc9..1caa8ce568883 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java
@@ -18,10 +18,21 @@
  */
 package org.elasticsearch.action.admin.indices.datastream;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction.Request;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
 public class GetDataStreamsRequestTests extends AbstractWireSerializingTestCase<Request> {
 
     @Override
@@ -33,4 +44,29 @@ protected Writeable.Reader<Request> instanceReader() {
     protected Request createTestInstance() {
         return new Request(generateRandomStringArray(8, 8, false));
     }
+
+    public void testValidateRequest() {
+        GetDataStreamsAction.Request req = new GetDataStreamsAction.Request(new String[]{});
+        ActionRequestValidationException e = req.validate();
+        assertNull(e);
+    }
+
+    public void testGetDataStreams() {
+        final String dataStreamName = "my-data-stream";
+        DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", Collections.emptyList());
+        ClusterState cs = ClusterState.builder(new ClusterName("_name"))
+            .metaData(MetaData.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build();
+        GetDataStreamsAction.Request req = new GetDataStreamsAction.Request(new String[]{dataStreamName});
+        List<DataStream> dataStreams = GetDataStreamsAction.TransportAction.getDataStreams(cs, req);
+        assertThat(dataStreams.size(), equalTo(1));
+        assertThat(dataStreams.get(0).getName(), equalTo(dataStreamName));
+    }
+
+    public void testGetNonexistentDataStream() {
+        final String dataStreamName = "my-data-stream";
+        ClusterState cs = ClusterState.builder(new ClusterName("_name")).build();
+        GetDataStreamsAction.Request req = new GetDataStreamsAction.Request(new String[]{dataStreamName});
+        List<DataStream> dataStreams = GetDataStreamsAction.TransportAction.getDataStreams(cs, req);
+        assertThat(dataStreams.size(), equalTo(0));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java
index 5c74c515634cc..24bd432ce4229 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java
@@ -29,6 +29,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.elasticsearch.cluster.metadata.DataStreamTests.randomIndexInstances;
+
 public class GetDataStreamsResponseTests extends AbstractSerializingTestCase<Response> {
 
     @Override
@@ -52,8 +54,7 @@ protected Response createTestInstance() {
         int numDataStreams = randomIntBetween(0, 8);
         List<DataStream> dataStreams = new ArrayList<>();
         for (int i = 0; i < numDataStreams; i++) {
-            dataStreams.add(new DataStream(randomAlphaOfLength(4), randomAlphaOfLength(4),
-                List.of(generateRandomStringArray(8, 4, false))));
+            dataStreams.add(new DataStream(randomAlphaOfLength(4), randomAlphaOfLength(4), randomIndexInstances()));
         }
         return new Response(dataStreams);
     }
diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
index b8957b7fef272..c2a58d0688ff8 100644
--- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
@@ -163,12 +163,12 @@ public void testBuildShardSearchTransportRequest() {
 
     public void testBuildSearchResponse() {
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean());
+        ArraySearchPhaseResults<SearchPhaseResult> phaseResults = new ArraySearchPhaseResults<>(10);
         AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(searchRequest,
-            new ArraySearchPhaseResults<>(10), null, false, new AtomicLong());
-        String scrollId = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10);
+            phaseResults, null, false, new AtomicLong());
         InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
-        SearchResponse searchResponse = action.buildSearchResponse(internalSearchResponse, scrollId, action.buildShardFailures());
-        assertEquals(scrollId, searchResponse.getScrollId());
+        SearchResponse searchResponse = action.buildSearchResponse(
+            internalSearchResponse, phaseResults.results, action.buildShardFailures());
         assertSame(searchResponse.getAggregations(), internalSearchResponse.aggregations());
         assertSame(searchResponse.getSuggest(), internalSearchResponse.suggest());
         assertSame(searchResponse.getProfileResults(), internalSearchResponse.profile());
@@ -177,14 +177,13 @@ public void testBuildSearchResponse() {
 
     public void testBuildSearchResponseAllowPartialFailures() {
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
-        AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(searchRequest,
-            new ArraySearchPhaseResults<>(10), null, false, new AtomicLong());
+        final ArraySearchPhaseResults<SearchPhaseResult> queryResult = new ArraySearchPhaseResults<>(10);
+        AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(searchRequest, queryResult, null, false, new AtomicLong());
         action.onShardFailure(0, new SearchShardTarget("node", new ShardId("index", "index-uuid", 0), null, OriginalIndices.NONE),
             new IllegalArgumentException());
-        String scrollId = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10);
         InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
-        SearchResponse searchResponse = action.buildSearchResponse(internalSearchResponse, scrollId, action.buildShardFailures());
-        assertEquals(scrollId, searchResponse.getScrollId());
+        SearchResponse searchResponse = action.buildSearchResponse(
+            internalSearchResponse, queryResult.results, action.buildShardFailures());
         assertSame(searchResponse.getAggregations(), internalSearchResponse.aggregations());
         assertSame(searchResponse.getSuggest(), internalSearchResponse.suggest());
         assertSame(searchResponse.getProfileResults(), internalSearchResponse.profile());
@@ -207,7 +206,7 @@ public void testSendSearchResponseDisallowPartialFailures() {
             action.onShardFailure(i, new SearchShardTarget(failureNodeId, failureShardId, failureClusterAlias, OriginalIndices.NONE),
                 new IllegalArgumentException());
         }
-        action.sendSearchResponse(InternalSearchResponse.empty(), randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10));
+        action.sendSearchResponse(InternalSearchResponse.empty(), phaseResults.results);
         assertThat(exception.get(), instanceOf(SearchPhaseExecutionException.class));
         SearchPhaseExecutionException searchPhaseExecutionException = (SearchPhaseExecutionException)exception.get();
         assertEquals(0, searchPhaseExecutionException.getSuppressed().length);
diff --git a/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/ClearContextControllerTests.java
similarity index 95%
rename from server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
rename to server/src/test/java/org/elasticsearch/action/search/ClearContextControllerTests.java
index 616b382baf5ca..e81164ca2a2f4 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ClearContextControllerTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.transport.NodeNotConnectedException;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportResponse;
@@ -43,7 +44,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 
-public class ClearScrollControllerTests extends ESTestCase {
+public class ClearContextControllerTests extends ESTestCase {
 
     public void testClearAll() throws InterruptedException {
         DiscoveryNode node1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
@@ -79,8 +80,8 @@ Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
         };
         ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
         clearScrollRequest.scrollIds(Arrays.asList("_all"));
-        ClearScrollController controller = new ClearScrollController(clearScrollRequest, listener,
-            nodes, logger, searchTransportService);
+        ClearContextController controller = new ClearContextController(
+            clearScrollRequest, listener, nodes, logger, searchTransportService);
         controller.run();
         latch.await();
         assertEquals(3, nodesInvoked.size());
@@ -107,7 +108,7 @@ public void testClearScrollIds() throws IOException, InterruptedException {
         array.setOnce(1, testSearchPhaseResult2);
         array.setOnce(2, testSearchPhaseResult3);
         AtomicInteger numFreed = new AtomicInteger(0);
-        String scrollId = TransportSearchHelper.buildScrollId(array, randomBoolean());
+        String scrollId = TransportSearchHelper.buildScrollId(array, VersionUtils.randomVersion(random()));
         DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
         CountDownLatch latch = new CountDownLatch(1);
         ActionListener<ClearScrollResponse> listener = new LatchedActionListener<>(new ActionListener<ClearScrollResponse>() {
@@ -144,7 +145,7 @@ Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
         };
         ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
         clearScrollRequest.scrollIds(Arrays.asList(scrollId));
-        ClearScrollController controller = new ClearScrollController(clearScrollRequest, listener,
+        ClearContextController controller = new ClearContextController(clearScrollRequest, listener,
             nodes, logger, searchTransportService);
         controller.run();
         latch.await();
@@ -173,7 +174,7 @@ public void testClearScrollIdsWithFailure() throws IOException, InterruptedExcep
         AtomicInteger numFreed = new AtomicInteger(0);
         AtomicInteger numFailures = new AtomicInteger(0);
         AtomicInteger numConnectionFailures = new AtomicInteger(0);
-        String scrollId = TransportSearchHelper.buildScrollId(array, randomBoolean());
+        String scrollId = TransportSearchHelper.buildScrollId(array, VersionUtils.randomVersion(random()));
         DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
         CountDownLatch latch = new CountDownLatch(1);
 
@@ -228,7 +229,7 @@ Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
         };
         ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
         clearScrollRequest.scrollIds(Arrays.asList(scrollId));
-        ClearScrollController controller = new ClearScrollController(clearScrollRequest, listener,
+        ClearContextController controller = new ClearContextController(clearScrollRequest, listener,
             nodes, logger, searchTransportService);
         controller.run();
         latch.await();
diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
index 5cef855e46697..055defdcdd129 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
@@ -137,7 +137,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
                 InternalSearchResponse internalSearchResponse = new InternalSearchResponse(collapsedHits,
                     null, null, null, false, null, 1);
                 SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, 1, 1, 0, 0,
-                    ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                    ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
                 listener.onResponse(new MultiSearchResponse(
                     new MultiSearchResponse.Item[]{
                             new MultiSearchResponse.Item(null, new RuntimeException("boom")),
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index 1849ed8f5690d..783a48e2f6170 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.index.shard.ShardId;
@@ -71,7 +70,7 @@ public void testShortcutQueryAndFetchOptimization() {
             numHits = 0;
         }
 
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
             @Override
             public void run() {
@@ -129,7 +128,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -187,7 +186,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
 
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -239,7 +238,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
             }
         };
         CountDownLatch latch = new CountDownLatch(1);
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -310,7 +309,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
@@ -363,7 +362,7 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext, ClusterState.EMPTY_STATE,
+        FetchSearchPhase phase = new FetchSearchPhase(results, controller, null, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
                 public void run() {
diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
index cd060c971a4f4..ed5b6f41023c1 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
@@ -20,8 +20,11 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.SearchContextId;
@@ -83,9 +86,12 @@ public SearchRequest getRequest() {
     }
 
     @Override
-    public void sendSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId) {
+    public void sendSearchResponse(InternalSearchResponse internalSearchResponse, AtomicArray<SearchPhaseResult> queryResults) {
+        String scrollId = getRequest().scroll() != null ? TransportSearchHelper.buildScrollId(queryResults, Version.CURRENT) : null;
+        String readerContextId = getRequest().source() != null && getRequest().source().reader() != null ?
+            TransportSearchHelper.buildScrollId(queryResults, Version.CURRENT) : null;
         searchResponse.set(new SearchResponse(internalSearchResponse, scrollId, numShards, numSuccess.get(), 0, 0,
-            failures.toArray(ShardSearchFailure.EMPTY_ARRAY), SearchResponse.Clusters.EMPTY));
+            failures.toArray(ShardSearchFailure.EMPTY_ARRAY), SearchResponse.Clusters.EMPTY, readerContextId));
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java
index 19b53e2f8d380..7b9c1eb6aa1b6 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java
@@ -156,7 +156,7 @@ public void search(final SearchRequest request, final ActionListener<SearchRespo
                 commonExecutor.execute(() -> {
                     counter.decrementAndGet();
                     listener.onResponse(new SearchResponse(InternalSearchResponse.empty(), null, 0, 0, 0, 0L,
-                        ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY));
+                        ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null));
                 });
             }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java
index d91a4eaf02288..fecf47bd29c7d 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java
@@ -49,7 +49,7 @@ protected MultiSearchResponse createTestInstance() {
             SearchResponse.Clusters clusters = SearchResponseTests.randomClusters();
             InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
             SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, totalShards,
-                    successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters);
+                    successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters, null);
             items[i] = new MultiSearchResponse.Item(searchResponse, null);
         }
         return new MultiSearchResponse(items, randomNonNegativeLong());
@@ -68,7 +68,7 @@ private static MultiSearchResponse createTestInstanceWithFailures() {
                 SearchResponse.Clusters clusters = SearchResponseTests.randomClusters();
                 InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
                 SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, totalShards,
-                        successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters);
+                        successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters, null);
                 items[i] = new MultiSearchResponse.Item(searchResponse, null);
             } else {
                 items[i] = new MultiSearchResponse.Item(null, new ElasticsearchException("an error"));
@@ -81,7 +81,7 @@ private static MultiSearchResponse createTestInstanceWithFailures() {
     protected MultiSearchResponse doParseInstance(XContentParser parser) throws IOException {
         return MultiSearchResponse.fromXContext(parser);
     }
-    
+
     @Override
     protected void assertEqualInstances(MultiSearchResponse expected, MultiSearchResponse actual) {
         assertThat(actual.getTook(), equalTo(expected.getTook()));
@@ -106,7 +106,7 @@ protected boolean supportsUnknownFields() {
 
     protected Predicate<String> getRandomFieldsExcludeFilterWhenResultHasErrors() {
         return field -> field.startsWith("responses");
-    }     
+    }
 
     /**
      * Test parsing {@link MultiSearchResponse} with inner failures as they don't support asserting on xcontent equivalence, given that
@@ -123,6 +123,6 @@ public void testFromXContentWithFailures() throws IOException {
         AbstractXContentTestCase.testFromXContent(NUMBER_OF_TEST_RUNS, instanceSupplier, supportsUnknownFields, Strings.EMPTY_ARRAY,
                 getRandomFieldsExcludeFilterWhenResultHasErrors(), this::createParser, this::doParseInstance,
                 this::assertEqualInstances, assertToXContentEquivalence, ToXContent.EMPTY_PARAMS);
-    }       
+    }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index 249426057c4ac..e7524fcb3b74b 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -511,7 +511,7 @@ public static class TestSearchResponse extends SearchResponse {
         final Set<ShardId> queried = new HashSet<>();
 
         TestSearchResponse() {
-            super(InternalSearchResponse.empty(), null, 0, 0, 0, 0L, ShardSearchFailure.EMPTY_ARRAY, Clusters.EMPTY);
+            super(InternalSearchResponse.empty(), null, 0, 0, 0, 0L, ShardSearchFailure.EMPTY_ARRAY, Clusters.EMPTY, null);
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index d0c98562328fb..d78af9ba31cb2 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -96,7 +96,8 @@ public void setup() {
             @Override
             public ReduceContext forPartialReduction() {
                 reductions.add(false);
-                return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null);
+                return InternalAggregation.ReduceContext.forPartialReduction(
+                        BigArrays.NON_RECYCLING_INSTANCE, null, () -> PipelineTree.EMPTY);
             }
 
             public ReduceContext forFinalReduction() {
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
index e93c986bd8eb3..bae4b2da99447 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
@@ -30,14 +30,11 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContextId;
@@ -45,6 +42,7 @@
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
 import org.elasticsearch.transport.Transport;
 
 import java.util.Collections;
@@ -125,17 +123,7 @@ public void sendExecuteQuery(Transport.Connection connection, ShardSearchRequest
             searchRequest.source().trackTotalHitsUpTo(2);
         }
         searchRequest.allowPartialSearchResults(false);
-        SearchPhaseController controller = new SearchPhaseController((b) -> new InternalAggregation.ReduceContextBuilder() {
-            @Override
-            public InternalAggregation.ReduceContext forPartialReduction() {
-                return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null);
-            }
-
-            public InternalAggregation.ReduceContext forFinalReduction() {
-                return InternalAggregation.ReduceContext.forFinalReduction(
-                    BigArrays.NON_RECYCLING_INSTANCE, null, b -> {}, PipelineAggregator.PipelineTree.EMPTY);
-            };
-        });
+        SearchPhaseController controller = new SearchPhaseController(r -> InternalAggregationTestCase.emptyReduceContextBuilder());
         SearchTask task = new SearchTask(0, "n/a", "n/a", "test", null, Collections.emptyMap());
         SearchQueryThenFetchAsyncAction action = new SearchQueryThenFetchAsyncAction(logger,
             searchTransportService, (clusterAlias, node) -> lookup.get(node),
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java
index df04549ddee88..6e8f487f31171 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java
@@ -100,7 +100,7 @@ public void testMergeTookInMillis() throws InterruptedException {
             SearchContext.TRACK_TOTAL_HITS_ACCURATE, timeProvider, emptyReduceContextBuilder());
         for (int i = 0; i < numResponses; i++) {
             SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), null, 1, 1, 0, randomLong(),
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponseTests.randomClusters());
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponseTests.randomClusters(), null);
             addResponse(merger, searchResponse);
         }
         awaitResponsesAdded();
@@ -136,7 +136,7 @@ public void testMergeShardFailures() throws InterruptedException {
                 priorityQueue.add(Tuple.tuple(searchShardTarget, failure));
             }
             SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), null,
-                1, 1, 0, 100L, shardSearchFailures, SearchResponse.Clusters.EMPTY);
+                1, 1, 0, 100L, shardSearchFailures, SearchResponse.Clusters.EMPTY, null);
             addResponse(merger, searchResponse);
         }
         awaitResponsesAdded();
@@ -174,7 +174,7 @@ public void testMergeShardFailuresNullShardTarget() throws InterruptedException
                 priorityQueue.add(Tuple.tuple(shardId, failure));
             }
             SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), null,
-                1, 1, 0, 100L, shardSearchFailures, SearchResponse.Clusters.EMPTY);
+                1, 1, 0, 100L, shardSearchFailures, SearchResponse.Clusters.EMPTY, null);
             addResponse(merger, searchResponse);
         }
         awaitResponsesAdded();
@@ -208,7 +208,7 @@ public void testMergeShardFailuresNullShardId() throws InterruptedException {
                 expectedFailures.add(shardSearchFailure);
             }
             SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), null,
-                1, 1, 0, 100L, shardSearchFailures, SearchResponse.Clusters.EMPTY);
+                1, 1, 0, 100L, shardSearchFailures, SearchResponse.Clusters.EMPTY, null);
             addResponse(merger, searchResponse);
         }
         awaitResponsesAdded();
@@ -228,7 +228,7 @@ public void testMergeProfileResults() throws InterruptedException {
             SearchHits searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN);
             InternalSearchResponse internalSearchResponse = new InternalSearchResponse(searchHits, null, null, profile, false, null, 1);
             SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, 1, 1, 0, 100L,
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
             addResponse(merger, searchResponse);
         }
         awaitResponsesAdded();
@@ -270,7 +270,7 @@ public void testMergeCompletionSuggestions()  throws InterruptedException {
             SearchHits searchHits = new SearchHits(new SearchHit[0], null, Float.NaN);
             InternalSearchResponse internalSearchResponse = new InternalSearchResponse(searchHits, null, suggest, null, false, null, 1);
             SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, 1, 1, 0, randomLong(),
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
             addResponse(searchResponseMerger, searchResponse);
         }
         awaitResponsesAdded();
@@ -318,7 +318,7 @@ public void testMergeCompletionSuggestionsTieBreak()  throws InterruptedExceptio
             SearchHits searchHits = new SearchHits(new SearchHit[0], null, Float.NaN);
             InternalSearchResponse internalSearchResponse = new InternalSearchResponse(searchHits, null, suggest, null, false, null, 1);
             SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, 1, 1, 0, randomLong(),
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
             addResponse(searchResponseMerger, searchResponse);
         }
         awaitResponsesAdded();
@@ -373,7 +373,7 @@ public void testMergeAggs() throws InterruptedException {
             SearchHits searchHits = new SearchHits(new SearchHit[0], null, Float.NaN);
             InternalSearchResponse internalSearchResponse = new InternalSearchResponse(searchHits, aggs, null, null, false, null, 1);
             SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, 1, 1, 0, randomLong(),
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
             addResponse(searchResponseMerger, searchResponse);
         }
         awaitResponsesAdded();
@@ -494,7 +494,7 @@ public void testMergeSearchHits() throws InterruptedException {
                 searchHits, null, null, null, timedOut, terminatedEarly, numReducePhases);
 
             SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, total, successful, skipped,
-                randomLong(), ShardSearchFailure.EMPTY_ARRAY, SearchResponseTests.randomClusters());
+                randomLong(), ShardSearchFailure.EMPTY_ARRAY, SearchResponseTests.randomClusters(), null);
 
             addResponse(searchResponseMerger, searchResponse);
         }
@@ -600,14 +600,14 @@ public void testMergeEmptySearchHitsWithNonEmpty() {
             SearchHits searchHits = new SearchHits(hits, new TotalHits(10, TotalHits.Relation.EQUAL_TO), Float.NaN, sortFields, null, null);
             InternalSearchResponse response = new InternalSearchResponse(searchHits, null, null, null, false, false, 1);
             SearchResponse searchResponse = new SearchResponse(response, null, 1, 1, 0, 1L,
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
             merger.add(searchResponse);
         }
         {
             SearchHits empty = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN, null, null, null);
             InternalSearchResponse response = new InternalSearchResponse(empty, null, null, null, false, false, 1);
             SearchResponse searchResponse = new SearchResponse(response, null, 1, 1, 0, 1L,
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
             merger.add(searchResponse);
         }
         assertEquals(2, merger.numResponses());
@@ -642,7 +642,7 @@ public void testMergeOnlyEmptyHits() {
             SearchHits empty = new SearchHits(new SearchHit[0], totalHits, Float.NaN, null, null, null);
             InternalSearchResponse response = new InternalSearchResponse(empty, null, null, null, false, false, 1);
             SearchResponse searchResponse = new SearchResponse(response, null, 1, 1, 0, 1L,
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
             merger.add(searchResponse);
         }
         SearchResponse mergedResponse = merger.getMergedResponse(clusters);
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
index d47213930b162..9908d360db4f9 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
@@ -123,7 +123,7 @@ private SearchResponse createTestItem(boolean minimal, ShardSearchFailure... sha
         }
 
         return new SearchResponse(internalSearchResponse, null, totalShards, successfulShards, skippedShards, tookInMillis,
-            shardSearchFailures, randomBoolean() ? randomClusters() : SearchResponse.Clusters.EMPTY);
+            shardSearchFailures, randomBoolean() ? randomClusters() : SearchResponse.Clusters.EMPTY, null);
     }
 
     static SearchResponse.Clusters randomClusters() {
@@ -216,7 +216,7 @@ public void testToXContent() {
                         null, false, null, 1),
                         null, 0
             , 0, 0, 0,
-                    ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                    ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
             StringBuilder expectedString = new StringBuilder();
             expectedString.append("{");
             {
@@ -245,7 +245,7 @@ public void testToXContent() {
                         new SearchHits(hits, new TotalHits(100, TotalHits.Relation.EQUAL_TO), 1.5f), null, null, null, false, null, 1
                     ),
                 null, 0, 0, 0, 0, ShardSearchFailure.EMPTY_ARRAY,
-                new SearchResponse.Clusters(5, 3, 2));
+                new SearchResponse.Clusters(5, 3, 2), null);
             StringBuilder expectedString = new StringBuilder();
             expectedString.append("{");
             {
@@ -295,7 +295,7 @@ public void testSerialization() throws IOException {
 
     public void testToXContentEmptyClusters() throws IOException {
         SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), null, 1, 1, 0, 1,
-            ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+            ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
         SearchResponse deserialized = copyWriteable(searchResponse, namedWriteableRegistry, SearchResponse::new, Version.CURRENT);
         XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
         deserialized.getClusters().toXContent(builder, ToXContent.EMPTY_PARAMS);
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
index 49eb7673592bf..1bc18dddde16b 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
@@ -50,11 +50,11 @@ public class SearchScrollAsyncActionTests extends ESTestCase {
     public void testSendRequestsToNodes() throws InterruptedException {
 
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 1)),
-            new ScrollIdForNode(null, "node2", new SearchContextId(UUIDs.randomBase64UUID(), 2)),
-            new ScrollIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 17)),
-            new ScrollIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 0)),
-            new ScrollIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 0)));
+            new ReaderIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 1)),
+            new ReaderIdForNode(null, "node2", new SearchContextId(UUIDs.randomBase64UUID(), 2)),
+            new ReaderIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 17)),
+            new ReaderIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 0)),
+            new ReaderIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -108,7 +108,7 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         latch.await();
         ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
         assertEquals(0, shardSearchFailures.length);
-        ScrollIdForNode[] context = scrollId.getContext();
+        ReaderIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             assertNotNull(results.get(i));
             assertEquals(context[i].getContextId(), results.get(i).getContextId());
@@ -119,11 +119,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
     public void testFailNextPhase() throws InterruptedException {
 
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ScrollIdForNode(null, "node2", new SearchContextId("a", 2)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("b", 17)),
-            new ScrollIdForNode(null, "node1", new SearchContextId("c", 0)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("d", 0)));
+            new ReaderIdForNode(null, "node1", new SearchContextId("", 1)),
+            new ReaderIdForNode(null, "node2", new SearchContextId("a", 2)),
+            new ReaderIdForNode(null, "node3", new SearchContextId("b", 17)),
+            new ReaderIdForNode(null, "node1", new SearchContextId("c", 0)),
+            new ReaderIdForNode(null, "node3", new SearchContextId("d", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -199,7 +199,7 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         latch.await();
         ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
         assertEquals(0, shardSearchFailures.length);
-        ScrollIdForNode[] context = scrollId.getContext();
+        ReaderIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             assertNotNull(results.get(i));
             assertEquals(context[i].getContextId(), results.get(i).getContextId());
@@ -209,11 +209,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testNodeNotAvailable() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ScrollIdForNode(null, "node2", new SearchContextId("", 2)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("", 17)),
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 0)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("", 0)));
+            new ReaderIdForNode(null, "node1", new SearchContextId("", 1)),
+            new ReaderIdForNode(null, "node2", new SearchContextId("", 2)),
+            new ReaderIdForNode(null, "node3", new SearchContextId("", 17)),
+            new ReaderIdForNode(null, "node1", new SearchContextId("", 0)),
+            new ReaderIdForNode(null, "node3", new SearchContextId("", 0)));
         // node2 is not available
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -275,7 +275,7 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         // .reason() returns the full stack trace
         assertThat(shardSearchFailures[0].reason(), startsWith("java.lang.IllegalStateException: node [node2] is not available"));
 
-        ScrollIdForNode[] context = scrollId.getContext();
+        ReaderIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             if (context[i].getNode().equals("node2")) {
                 assertNull(results.get(i));
@@ -289,11 +289,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testShardFailures() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ScrollIdForNode(null, "node2", new SearchContextId("", 2)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("",17)),
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 0)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("", 0)));
+            new ReaderIdForNode(null, "node1", new SearchContextId("", 1)),
+            new ReaderIdForNode(null, "node2", new SearchContextId("", 2)),
+            new ReaderIdForNode(null, "node3", new SearchContextId("",17)),
+            new ReaderIdForNode(null, "node1", new SearchContextId("", 0)),
+            new ReaderIdForNode(null, "node3", new SearchContextId("", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -353,7 +353,7 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         assertEquals(1, shardSearchFailures.length);
         assertThat(shardSearchFailures[0].reason(), containsString("IllegalArgumentException: BOOM on shard"));
 
-        ScrollIdForNode[] context = scrollId.getContext();
+        ReaderIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             if (context[i].getContextId().getId() == 17) {
                 assertNull(results.get(i));
@@ -367,11 +367,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testAllShardsFailed() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ScrollIdForNode(null, "node2", new SearchContextId("", 2)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("", 17)),
-            new ScrollIdForNode(null, "node1", new SearchContextId("", 0)),
-            new ScrollIdForNode(null, "node3", new SearchContextId("", 0)));
+            new ReaderIdForNode(null, "node1", new SearchContextId("", 1)),
+            new ReaderIdForNode(null, "node2", new SearchContextId("", 2)),
+            new ReaderIdForNode(null, "node3", new SearchContextId("", 17)),
+            new ReaderIdForNode(null, "node1", new SearchContextId("", 0)),
+            new ReaderIdForNode(null, "node3", new SearchContextId("", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -433,7 +433,7 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
         action.run();
         latch.await();
-        ScrollIdForNode[] context = scrollId.getContext();
+        ReaderIdForNode[] context = scrollId.getContext();
 
         ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
         assertEquals(context.length, shardSearchFailures.length);
@@ -444,10 +444,10 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         }
     }
 
-    private static ParsedScrollId getParsedScrollId(ScrollIdForNode... idsForNodes) {
-        List<ScrollIdForNode> scrollIdForNodes = Arrays.asList(idsForNodes);
-        Collections.shuffle(scrollIdForNodes, random());
-        return new ParsedScrollId("", "test", scrollIdForNodes.toArray(new ScrollIdForNode[0]));
+    private static ParsedScrollId getParsedScrollId(ReaderIdForNode... idsForNodes) {
+        List<ReaderIdForNode> readerIdForNodes = Arrays.asList(idsForNodes);
+        Collections.shuffle(readerIdForNodes, random());
+        return new ParsedScrollId("", "test", readerIdForNodes.toArray(new ReaderIdForNode[0]));
     }
 
     private ActionListener<SearchResponse> dummyListener() {
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
index cde432de98bff..391b7f847355d 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
@@ -159,7 +159,7 @@ public void search(final SearchRequest request, final ActionListener<SearchRespo
                 executorService.execute(() -> {
                     counter.decrementAndGet();
                     listener.onResponse(new SearchResponse(InternalSearchResponse.empty(), null, 0, 0, 0, 0L,
-                        ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY));
+                        ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null));
                 });
             }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
index 32bca4d0dc18c..cb3d07fc5ff7e 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -19,6 +19,28 @@
 
 package org.elasticsearch.action.search;
 
+import static org.elasticsearch.test.InternalAggregationTestCase.emptyReduceContextBuilder;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.awaitLatch;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.startsWith;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
@@ -29,11 +51,13 @@
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.GroupShardsIteratorTests;
-import org.elasticsearch.cluster.routing.PlainShardIterator;
-import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
@@ -44,6 +68,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
@@ -52,12 +77,12 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -74,29 +99,6 @@
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-
-import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.awaitLatch;
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.startsWith;
-
 public class TransportSearchActionTests extends ESTestCase {
 
     private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
@@ -140,7 +142,7 @@ public void testMergeShardsIterators() {
         List<SearchShardIterator> expected = new ArrayList<>();
         String localClusterAlias = randomAlphaOfLengthBetween(5, 10);
         OriginalIndices localIndices = OriginalIndicesTests.randomOriginalIndices();
-        List<ShardIterator> localShardIterators = new ArrayList<>();
+        List<SearchShardIterator> localShardIterators = new ArrayList<>();
         List<SearchShardIterator> remoteShardIterators = new ArrayList<>();
         int numShards = randomIntBetween(0, 10);
         for (int i = 0; i < numShards; i++) {
@@ -150,7 +152,7 @@ public void testMergeShardsIterators() {
                 boolean localIndex = randomBoolean();
                 if (localIndex) {
                     SearchShardIterator localIterator = createSearchShardIterator(i, index, localIndices, localClusterAlias);
-                    localShardIterators.add(new PlainShardIterator(localIterator.shardId(), localIterator.getShardRoutings()));
+                    localShardIterators.add(localIterator);
                     if (rarely()) {
                         String remoteClusterAlias = randomFrom(remoteClusters);
                         //simulate scenario where the local cluster is also registered as a remote one
@@ -187,11 +189,12 @@ public void testMergeShardsIterators() {
             }
         }
 
+
         Collections.shuffle(localShardIterators, random());
         Collections.shuffle(remoteShardIterators, random());
 
-        GroupShardsIterator<SearchShardIterator> groupShardsIterator = TransportSearchAction.mergeShardsIterators(
-            new GroupShardsIterator<>(localShardIterators), localIndices, localClusterAlias, remoteShardIterators);
+        GroupShardsIterator<SearchShardIterator> groupShardsIterator =
+            TransportSearchAction.mergeShardsIterators(localShardIterators, remoteShardIterators);
         List<SearchShardIterator> result = new ArrayList<>();
         for (SearchShardIterator searchShardIterator : groupShardsIterator) {
             result.add(searchShardIterator);
@@ -363,7 +366,7 @@ private MockTransportService[] startTransport(int numClusters, DiscoveryNode[] n
     private static SearchResponse emptySearchResponse() {
         InternalSearchResponse response = new InternalSearchResponse(new SearchHits(new SearchHit[0],
             new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), InternalAggregations.EMPTY, null, null, false, null, 1);
-        return new SearchResponse(response, null, 1, 1, 0, 100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+        return new SearchResponse(response, null, 1, 1, 0, 100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
     }
 
     public void testCCSRemoteReduceMergeFails() throws Exception {
@@ -389,7 +392,7 @@ public void testCCSRemoteReduceMergeFails() throws Exception {
             LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                 ActionListener.wrap(r -> fail("no response expected"), failure::set), latch);
             TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
-                    aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                    emptyReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
             if (localIndices == null) {
                 assertNull(setOnce.get());
             } else {
@@ -434,7 +437,7 @@ public void testCCSRemoteReduce() throws Exception {
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(response::set, e -> fail("no failures expected")), latch);
                 TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
-                        aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                        emptyReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
                 } else {
@@ -460,7 +463,7 @@ public void testCCSRemoteReduce() throws Exception {
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(r -> fail("no response expected"), failure::set), latch);
                 TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
-                        aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                        emptyReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
                 } else {
@@ -507,7 +510,7 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(r -> fail("no response expected"), failure::set), latch);
                 TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
-                        aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                        emptyReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
                 } else {
@@ -535,8 +538,8 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
                 AtomicReference<SearchResponse> response = new AtomicReference<>();
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(response::set, e -> fail("no failures expected")), latch);
-                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, 
-                        aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
+                        emptyReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
                 } else {
@@ -576,7 +579,7 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
                 LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
                     ActionListener.wrap(response::set, e -> fail("no failures expected")), latch);
                 TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider,
-                        aggReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
+                        emptyReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
                 if (localIndices == null) {
                     assertNull(setOnce.get());
                 } else {
@@ -757,7 +760,8 @@ public void testCreateSearchResponseMerger() {
             assertEquals(-1, source.size());
             assertEquals(-1, source.from());
             assertNull(source.trackTotalHitsUpTo());
-            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(source, timeProvider, aggReduceContextBuilder());
+            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(
+                    source, timeProvider, emptyReduceContextBuilder());
             assertEquals(0, merger.from);
             assertEquals(10, merger.size);
             assertEquals(SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO, merger.trackTotalHitsUpTo);
@@ -766,7 +770,7 @@ public void testCreateSearchResponseMerger() {
             assertNull(source.trackTotalHitsUpTo());
         }
         {
-            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(null, timeProvider, aggReduceContextBuilder());
+            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(null, timeProvider, emptyReduceContextBuilder());
             assertEquals(0, merger.from);
             assertEquals(10, merger.size);
             assertEquals(SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO, merger.trackTotalHitsUpTo);
@@ -779,7 +783,8 @@ public void testCreateSearchResponseMerger() {
             source.size(originalSize);
             int trackTotalHitsUpTo = randomIntBetween(0, Integer.MAX_VALUE);
             source.trackTotalHitsUpTo(trackTotalHitsUpTo);
-            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(source, timeProvider, aggReduceContextBuilder());
+            SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(
+                    source, timeProvider, emptyReduceContextBuilder());
             assertEquals(0, source.from());
             assertEquals(originalFrom + originalSize, source.size());
             assertEquals(trackTotalHitsUpTo, (int)source.trackTotalHitsUpTo());
@@ -838,17 +843,95 @@ public void testShouldMinimizeRoundtrips() throws Exception {
         }
     }
 
-    private InternalAggregation.ReduceContextBuilder aggReduceContextBuilder() {
-        return new InternalAggregation.ReduceContextBuilder() {
-            @Override
-            public InternalAggregation.ReduceContext forPartialReduction() {
-                return InternalAggregation.ReduceContext.forPartialReduction(null, null);
-            };
+    public void testShouldPreFilterSearchShards() {
+        int numIndices = randomIntBetween(2, 10);
+        String[] indices = new String[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            indices[i] = randomAlphaOfLengthBetween(5, 10);
+        }
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).build();
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(2, 127)));
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery("timestamp")));
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(2, 127)));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().sort(SortBuilders.fieldSort("timestamp")));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(2, 127)));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().sort(SortBuilders.fieldSort("timestamp")))
+                .scroll("5m");
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(2, 127)));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+    }
 
-            @Override
-            public InternalAggregation.ReduceContext forFinalReduction() {
-                return InternalAggregation.ReduceContext.forFinalReduction(null, null, b -> {}, new PipelineTree(emptyMap(), emptyList()));
+    public void testShouldPreFilterSearchShardsWithReadOnly() {
+        int numIndices = randomIntBetween(2, 10);
+        int numReadOnly = randomIntBetween(1, numIndices);
+        String[] indices = new String[numIndices];
+        ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder();
+        for (int i = 0; i < numIndices; i++) {
+            indices[i] = randomAlphaOfLengthBetween(5, 10);;
+            if (--numReadOnly >= 0) {
+                if (randomBoolean()) {
+                    blocksBuilder.addIndexBlock(indices[i], IndexMetaData.INDEX_WRITE_BLOCK);
+                } else {
+                    blocksBuilder.addIndexBlock(indices[i], IndexMetaData.INDEX_READ_ONLY_BLOCK);
+                }
             }
-        };
+        }
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).blocks(blocksBuilder).build();
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(2, 127)));
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery("timestamp")));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(2, 127)));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery("timestamp")));
+            searchRequest.scroll("5s");
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(2, 127)));
+            assertTrue(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery("timestamp")));
+            searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(2, 127)));
+            assertFalse(TransportSearchAction.shouldPreFilterSearchShards(clusterState, searchRequest,
+                indices, randomIntBetween(127, 10000)));
+        }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
index c9d58c7dc9090..f18284daedd8e 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
@@ -26,40 +26,48 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 
-import java.io.IOException;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.nullValue;
 
 public class TransportSearchHelperTests extends ESTestCase {
 
-    public void testParseScrollId() throws IOException {
+    AtomicArray<SearchPhaseResult> generateQueryResults() {
         AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
         DiscoveryNode node1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("x", 1), node1);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("a", 1), node1);
         testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), "cluster_x", null));
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("y", 12), node2);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("b", 12), node2);
         testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), "cluster_y", null));
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("z", 42), node3);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("c", 42), node3);
         testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
         array.setOnce(0, testSearchPhaseResult1);
         array.setOnce(1, testSearchPhaseResult2);
         array.setOnce(2, testSearchPhaseResult3);
+        return array;
+    }
 
-        boolean includeUUID = randomBoolean();
-        String scrollId = TransportSearchHelper.buildScrollId(array, includeUUID);
+    public void testParseScrollId()  {
+        final Version version = VersionUtils.randomVersion(random());
+        boolean includeUUID = version.onOrAfter(Version.V_7_7_0);
+        final AtomicArray<SearchPhaseResult> queryResults = generateQueryResults();
+        String scrollId = TransportSearchHelper.buildScrollId(queryResults, version);
         ParsedScrollId parseScrollId = TransportSearchHelper.parseScrollId(scrollId);
         assertEquals(3, parseScrollId.getContext().length);
         assertEquals("node_1", parseScrollId.getContext()[0].getNode());
         assertEquals("cluster_x", parseScrollId.getContext()[0].getClusterAlias());
         assertEquals(1, parseScrollId.getContext()[0].getContextId().getId());
         if (includeUUID) {
-            assertThat(parseScrollId.getContext()[0].getContextId().getReaderId(), equalTo("x"));
+            assertThat(parseScrollId.getContext()[0].getContextId().getReaderId(), equalTo("a"));
         } else {
             assertThat(parseScrollId.getContext()[0].getContextId().getReaderId(), equalTo(""));
         }
@@ -68,7 +76,7 @@ public void testParseScrollId() throws IOException {
         assertEquals("cluster_y", parseScrollId.getContext()[1].getClusterAlias());
         assertEquals(12, parseScrollId.getContext()[1].getContextId().getId());
         if (includeUUID) {
-            assertThat(parseScrollId.getContext()[1].getContextId().getReaderId(), equalTo("y"));
+            assertThat(parseScrollId.getContext()[1].getContextId().getReaderId(), equalTo("b"));
         } else {
             assertThat(parseScrollId.getContext()[1].getContextId().getReaderId(), equalTo(""));
         }
@@ -77,9 +85,35 @@ public void testParseScrollId() throws IOException {
         assertNull(parseScrollId.getContext()[2].getClusterAlias());
         assertEquals(42, parseScrollId.getContext()[2].getContextId().getId());
         if (includeUUID) {
-            assertThat(parseScrollId.getContext()[2].getContextId().getReaderId(), equalTo("z"));
+            assertThat(parseScrollId.getContext()[2].getContextId().getReaderId(), equalTo("c"));
         } else {
             assertThat(parseScrollId.getContext()[2].getContextId().getReaderId(), equalTo(""));
         }
     }
+
+    public void testEncodeDecodeReaderId() {
+        final AtomicArray<SearchPhaseResult> queryResults = generateQueryResults();
+        final Version version = VersionUtils.randomVersion(random());
+        final String readerId = TransportSearchHelper.encodeReaderIds(queryResults, version);
+        final Map<ShardId, ReaderIdForNode> contextIds = TransportSearchHelper.decodeReaderIds(readerId);
+        assertThat(contextIds.keySet(), hasSize(3));
+
+        ReaderIdForNode node1 = contextIds.get(new ShardId("idx", "uuid1", 2));
+        assertThat(node1.getClusterAlias(), equalTo("cluster_x"));
+        assertThat(node1.getNode(), equalTo("node_1"));
+        assertThat(node1.getContextId().getId(), equalTo(1L));
+        assertThat(node1.getContextId().getReaderId(), equalTo("a"));
+
+        ReaderIdForNode node2 = contextIds.get(new ShardId("idy", "uuid2", 42));
+        assertThat(node2.getClusterAlias(), equalTo("cluster_y"));
+        assertThat(node2.getNode(), equalTo("node_2"));
+        assertThat(node2.getContextId().getId(), equalTo(12L));
+        assertThat(node2.getContextId().getReaderId(), equalTo("b"));
+
+        ReaderIdForNode node3 = contextIds.get(new ShardId("idy", "uuid2", 43));
+        assertThat(node3.getClusterAlias(), nullValue());
+        assertThat(node3.getNode(), equalTo("node_3"));
+        assertThat(node3.getContextId().getId(), equalTo(42L));
+        assertThat(node3.getContextId().getReaderId(), equalTo("c"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java
index 1d264a31fbdb3..63cb6a45e1a72 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java
@@ -303,7 +303,8 @@ public void testHealthOnMasterFailover() throws Exception {
         // Run a few health requests concurrent to master fail-overs against a data-node to make sure master failover is handled
         // without exceptions
         for (int i = 0; i < 20; ++i) {
-            responseFutures.add(client(node).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).execute());
+            responseFutures.add(client(node).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID)
+                .setWaitForGreenStatus().execute());
             internalCluster().restartNode(internalCluster().getMasterName(), InternalTestCluster.EMPTY_CALLBACK);
         }
         for (ActionFuture<ClusterHealthResponse> responseFuture : responseFutures) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
index 86d3eec9c9c76..9b83752dd0e17 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
@@ -19,14 +19,48 @@
 package org.elasticsearch.cluster;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
+import org.elasticsearch.cluster.block.ClusterBlock;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.coordination.CoordinationMetaData;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestCustomMetaData;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -73,4 +107,741 @@ public void testBuilderRejectsNullInCustoms() {
         final ImmutableOpenMap<String, ClusterState.Custom> map = mapBuilder.build();
         assertThat(expectThrows(NullPointerException.class, () -> builder.customs(map)).getMessage(), containsString(key));
     }
+
+    public void testToXContent() throws IOException {
+        final ClusterState clusterState = buildClusterState();
+
+        IndexRoutingTable index = clusterState.getRoutingTable().getIndicesRouting().get("index");
+
+        String ephemeralId = clusterState.getNodes().get("nodeId1").getEphemeralId();
+        String allocationId = index.getShards().get(1).getAllAllocationIds().iterator().next();
+
+        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
+        builder.startObject();
+        clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+
+        assertEquals("{\n" +
+            "  \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "  \"version\" : 0,\n" +
+            "  \"state_uuid\" : \"stateUUID\",\n" +
+            "  \"master_node\" : \"masterNodeId\",\n" +
+            "  \"blocks\" : {\n" +
+            "    \"global\" : {\n" +
+            "      \"1\" : {\n" +
+            "        \"description\" : \"description\",\n" +
+            "        \"retryable\" : true,\n" +
+            "        \"disable_state_persistence\" : true,\n" +
+            "        \"levels\" : [\n" +
+            "          \"read\",\n" +
+            "          \"write\",\n" +
+            "          \"metadata_read\",\n" +
+            "          \"metadata_write\"\n" +
+            "        ]\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"2\" : {\n" +
+            "          \"description\" : \"description2\",\n" +
+            "          \"retryable\" : false,\n" +
+            "          \"levels\" : [\n" +
+            "            \"read\",\n" +
+            "            \"write\",\n" +
+            "            \"metadata_read\",\n" +
+            "            \"metadata_write\"\n" +
+            "          ]\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"nodes\" : {\n" +
+            "    \"nodeId1\" : {\n" +
+            "      \"name\" : \"\",\n" +
+            "      \"ephemeral_id\" : \"" + ephemeralId + "\",\n" +
+            "      \"transport_address\" : \"127.0.0.1:111\",\n" +
+            "      \"attributes\" : { }\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"metadata\" : {\n" +
+            "    \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "    \"cluster_uuid_committed\" : false,\n" +
+            "    \"cluster_coordination\" : {\n" +
+            "      \"term\" : 1,\n" +
+            "      \"last_committed_config\" : [\n" +
+            "        \"commitedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"last_accepted_config\" : [\n" +
+            "        \"acceptedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"voting_config_exclusions\" : [\n" +
+            "        {\n" +
+            "          \"node_id\" : \"exlucdedNodeId\",\n" +
+            "          \"node_name\" : \"excludedNodeName\"\n" +
+            "        }\n" +
+            "      ]\n" +
+            "    },\n" +
+            "    \"templates\" : {\n" +
+            "      \"template\" : {\n" +
+            "        \"order\" : 0,\n" +
+            "        \"index_patterns\" : [\n" +
+            "          \"pattern1\",\n" +
+            "          \"pattern2\"\n" +
+            "        ],\n" +
+            "        \"settings\" : {\n" +
+            "          \"index\" : {\n" +
+            "            \"version\" : {\n" +
+            "              \"created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"key1\" : { }\n" +
+            "        },\n" +
+            "        \"aliases\" : { }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"version\" : 1,\n" +
+            "        \"mapping_version\" : 1,\n" +
+            "        \"settings_version\" : 1,\n" +
+            "        \"aliases_version\" : 1,\n" +
+            "        \"routing_num_shards\" : 1,\n" +
+            "        \"state\" : \"open\",\n" +
+            "        \"settings\" : {\n" +
+            "          \"index\" : {\n" +
+            "            \"number_of_shards\" : \"1\",\n" +
+            "            \"number_of_replicas\" : \"2\",\n" +
+            "            \"version\" : {\n" +
+            "              \"created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"type\" : {\n" +
+            "            \"type1\" : {\n" +
+            "              \"key\" : \"value\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"aliases\" : [\n" +
+            "          \"alias\"\n" +
+            "        ],\n" +
+            "        \"primary_terms\" : {\n" +
+            "          \"0\" : 1\n" +
+            "        },\n" +
+            "        \"in_sync_allocations\" : {\n" +
+            "          \"0\" : [\n" +
+            "            \"allocationId\"\n" +
+            "          ]\n" +
+            "        },\n" +
+            "        \"rollover_info\" : {\n" +
+            "          \"rolloveAlias\" : {\n" +
+            "            \"met_conditions\" : { },\n" +
+            "            \"time\" : 1\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"index-graveyard\" : {\n" +
+            "      \"tombstones\" : [ ]\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"routing_table\" : {\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"shards\" : {\n" +
+            "          \"1\" : [\n" +
+            "            {\n" +
+            "              \"state\" : \"STARTED\",\n" +
+            "              \"primary\" : true,\n" +
+            "              \"node\" : \"nodeId2\",\n" +
+            "              \"relocating_node\" : null,\n" +
+            "              \"shard\" : 1,\n" +
+            "              \"index\" : \"index\",\n" +
+            "              \"allocation_id\" : {\n" +
+            "                \"id\" : \"" + allocationId + "\"\n" +
+            "              }\n" +
+            "            }\n" +
+            "          ]\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"routing_nodes\" : {\n" +
+            "    \"unassigned\" : [ ],\n" +
+            "    \"nodes\" : {\n" +
+            "      \"nodeId2\" : [\n" +
+            "        {\n" +
+            "          \"state\" : \"STARTED\",\n" +
+            "          \"primary\" : true,\n" +
+            "          \"node\" : \"nodeId2\",\n" +
+            "          \"relocating_node\" : null,\n" +
+            "          \"shard\" : 1,\n" +
+            "          \"index\" : \"index\",\n" +
+            "          \"allocation_id\" : {\n" +
+            "            \"id\" : \"" + allocationId + "\"\n" +
+            "          }\n" +
+            "        }\n" +
+            "      ],\n" +
+            "      \"nodeId1\" : [ ]\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", Strings.toString(builder));
+
+    }
+
+    public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOException {
+        Map<String, String> mapParams = new HashMap<>(){{
+            put("flat_settings", "true");
+            put("reduce_mappings", "false");
+        }};
+
+        final ClusterState clusterState = buildClusterState();
+        IndexRoutingTable index = clusterState.getRoutingTable().getIndicesRouting().get("index");
+
+        String ephemeralId = clusterState.getNodes().get("nodeId1").getEphemeralId();
+        String allocationId = index.getShards().get(1).getAllAllocationIds().iterator().next();
+
+        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
+        builder.startObject();
+        clusterState.toXContent(builder, new ToXContent.MapParams(mapParams));
+        builder.endObject();
+
+        assertEquals("{\n" +
+            "  \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "  \"version\" : 0,\n" +
+            "  \"state_uuid\" : \"stateUUID\",\n" +
+            "  \"master_node\" : \"masterNodeId\",\n" +
+            "  \"blocks\" : {\n" +
+            "    \"global\" : {\n" +
+            "      \"1\" : {\n" +
+            "        \"description\" : \"description\",\n" +
+            "        \"retryable\" : true,\n" +
+            "        \"disable_state_persistence\" : true,\n" +
+            "        \"levels\" : [\n" +
+            "          \"read\",\n" +
+            "          \"write\",\n" +
+            "          \"metadata_read\",\n" +
+            "          \"metadata_write\"\n" +
+            "        ]\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"2\" : {\n" +
+            "          \"description\" : \"description2\",\n" +
+            "          \"retryable\" : false,\n" +
+            "          \"levels\" : [\n" +
+            "            \"read\",\n" +
+            "            \"write\",\n" +
+            "            \"metadata_read\",\n" +
+            "            \"metadata_write\"\n" +
+            "          ]\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"nodes\" : {\n" +
+            "    \"nodeId1\" : {\n" +
+            "      \"name\" : \"\",\n" +
+            "      \"ephemeral_id\" : \"" + ephemeralId + "\",\n" +
+            "      \"transport_address\" : \"127.0.0.1:111\",\n" +
+            "      \"attributes\" : { }\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"metadata\" : {\n" +
+            "    \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "    \"cluster_uuid_committed\" : false,\n" +
+            "    \"cluster_coordination\" : {\n" +
+            "      \"term\" : 1,\n" +
+            "      \"last_committed_config\" : [\n" +
+            "        \"commitedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"last_accepted_config\" : [\n" +
+            "        \"acceptedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"voting_config_exclusions\" : [\n" +
+            "        {\n" +
+            "          \"node_id\" : \"exlucdedNodeId\",\n" +
+            "          \"node_name\" : \"excludedNodeName\"\n" +
+            "        }\n" +
+            "      ]\n" +
+            "    },\n" +
+            "    \"templates\" : {\n" +
+            "      \"template\" : {\n" +
+            "        \"order\" : 0,\n" +
+            "        \"index_patterns\" : [\n" +
+            "          \"pattern1\",\n" +
+            "          \"pattern2\"\n" +
+            "        ],\n" +
+            "        \"settings\" : {\n" +
+            "          \"index.version.created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"key1\" : { }\n" +
+            "        },\n" +
+            "        \"aliases\" : { }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"version\" : 1,\n" +
+            "        \"mapping_version\" : 1,\n" +
+            "        \"settings_version\" : 1,\n" +
+            "        \"aliases_version\" : 1,\n" +
+            "        \"routing_num_shards\" : 1,\n" +
+            "        \"state\" : \"open\",\n" +
+            "        \"settings\" : {\n" +
+            "          \"index.number_of_replicas\" : \"2\",\n" +
+            "          \"index.number_of_shards\" : \"1\",\n" +
+            "          \"index.version.created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"type\" : {\n" +
+            "            \"type1\" : {\n" +
+            "              \"key\" : \"value\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"aliases\" : [\n" +
+            "          \"alias\"\n" +
+            "        ],\n" +
+            "        \"primary_terms\" : {\n" +
+            "          \"0\" : 1\n" +
+            "        },\n" +
+            "        \"in_sync_allocations\" : {\n" +
+            "          \"0\" : [\n" +
+            "            \"allocationId\"\n" +
+            "          ]\n" +
+            "        },\n" +
+            "        \"rollover_info\" : {\n" +
+            "          \"rolloveAlias\" : {\n" +
+            "            \"met_conditions\" : { },\n" +
+            "            \"time\" : 1\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"index-graveyard\" : {\n" +
+            "      \"tombstones\" : [ ]\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"routing_table\" : {\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"shards\" : {\n" +
+            "          \"1\" : [\n" +
+            "            {\n" +
+            "              \"state\" : \"STARTED\",\n" +
+            "              \"primary\" : true,\n" +
+            "              \"node\" : \"nodeId2\",\n" +
+            "              \"relocating_node\" : null,\n" +
+            "              \"shard\" : 1,\n" +
+            "              \"index\" : \"index\",\n" +
+            "              \"allocation_id\" : {\n" +
+            "                \"id\" : \"" + allocationId + "\"\n" +
+            "              }\n" +
+            "            }\n" +
+            "          ]\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"routing_nodes\" : {\n" +
+            "    \"unassigned\" : [ ],\n" +
+            "    \"nodes\" : {\n" +
+            "      \"nodeId2\" : [\n" +
+            "        {\n" +
+            "          \"state\" : \"STARTED\",\n" +
+            "          \"primary\" : true,\n" +
+            "          \"node\" : \"nodeId2\",\n" +
+            "          \"relocating_node\" : null,\n" +
+            "          \"shard\" : 1,\n" +
+            "          \"index\" : \"index\",\n" +
+            "          \"allocation_id\" : {\n" +
+            "            \"id\" : \"" + allocationId + "\"\n" +
+            "          }\n" +
+            "        }\n" +
+            "      ],\n" +
+            "      \"nodeId1\" : [ ]\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", Strings.toString(builder));
+
+    }
+
+    public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOException {
+        Map<String, String> mapParams = new HashMap<>(){{
+            put("flat_settings", "false");
+            put("reduce_mappings", "true");
+        }};
+
+        final ClusterState clusterState = buildClusterState();
+
+        IndexRoutingTable index = clusterState.getRoutingTable().getIndicesRouting().get("index");
+
+        String ephemeralId = clusterState.getNodes().get("nodeId1").getEphemeralId();
+        String allocationId = index.getShards().get(1).getAllAllocationIds().iterator().next();
+
+        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
+        builder.startObject();
+        clusterState.toXContent(builder, new ToXContent.MapParams(mapParams));
+        builder.endObject();
+
+        assertEquals("{\n" +
+            "  \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "  \"version\" : 0,\n" +
+            "  \"state_uuid\" : \"stateUUID\",\n" +
+            "  \"master_node\" : \"masterNodeId\",\n" +
+            "  \"blocks\" : {\n" +
+            "    \"global\" : {\n" +
+            "      \"1\" : {\n" +
+            "        \"description\" : \"description\",\n" +
+            "        \"retryable\" : true,\n" +
+            "        \"disable_state_persistence\" : true,\n" +
+            "        \"levels\" : [\n" +
+            "          \"read\",\n" +
+            "          \"write\",\n" +
+            "          \"metadata_read\",\n" +
+            "          \"metadata_write\"\n" +
+            "        ]\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"2\" : {\n" +
+            "          \"description\" : \"description2\",\n" +
+            "          \"retryable\" : false,\n" +
+            "          \"levels\" : [\n" +
+            "            \"read\",\n" +
+            "            \"write\",\n" +
+            "            \"metadata_read\",\n" +
+            "            \"metadata_write\"\n" +
+            "          ]\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"nodes\" : {\n" +
+            "    \"nodeId1\" : {\n" +
+            "      \"name\" : \"\",\n" +
+            "      \"ephemeral_id\" : \"" + ephemeralId + "\",\n" +
+            "      \"transport_address\" : \"127.0.0.1:111\",\n" +
+            "      \"attributes\" : { }\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"metadata\" : {\n" +
+            "    \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "    \"cluster_uuid_committed\" : false,\n" +
+            "    \"cluster_coordination\" : {\n" +
+            "      \"term\" : 1,\n" +
+            "      \"last_committed_config\" : [\n" +
+            "        \"commitedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"last_accepted_config\" : [\n" +
+            "        \"acceptedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"voting_config_exclusions\" : [\n" +
+            "        {\n" +
+            "          \"node_id\" : \"exlucdedNodeId\",\n" +
+            "          \"node_name\" : \"excludedNodeName\"\n" +
+            "        }\n" +
+            "      ]\n" +
+            "    },\n" +
+            "    \"templates\" : {\n" +
+            "      \"template\" : {\n" +
+            "        \"order\" : 0,\n" +
+            "        \"index_patterns\" : [\n" +
+            "          \"pattern1\",\n" +
+            "          \"pattern2\"\n" +
+            "        ],\n" +
+            "        \"settings\" : {\n" +
+            "          \"index\" : {\n" +
+            "            \"version\" : {\n" +
+            "              \"created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"mappings\" : { },\n" +
+            "        \"aliases\" : { }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"version\" : 1,\n" +
+            "        \"mapping_version\" : 1,\n" +
+            "        \"settings_version\" : 1,\n" +
+            "        \"aliases_version\" : 1,\n" +
+            "        \"routing_num_shards\" : 1,\n" +
+            "        \"state\" : \"open\",\n" +
+            "        \"settings\" : {\n" +
+            "          \"index\" : {\n" +
+            "            \"number_of_shards\" : \"1\",\n" +
+            "            \"number_of_replicas\" : \"2\",\n" +
+            "            \"version\" : {\n" +
+            "              \"created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"type\" : {\n" +
+            "            \"type1\" : {\n" +
+            "              \"key\" : \"value\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"aliases\" : [\n" +
+            "          \"alias\"\n" +
+            "        ],\n" +
+            "        \"primary_terms\" : {\n" +
+            "          \"0\" : 1\n" +
+            "        },\n" +
+            "        \"in_sync_allocations\" : {\n" +
+            "          \"0\" : [\n" +
+            "            \"allocationId\"\n" +
+            "          ]\n" +
+            "        },\n" +
+            "        \"rollover_info\" : {\n" +
+            "          \"rolloveAlias\" : {\n" +
+            "            \"met_conditions\" : { },\n" +
+            "            \"time\" : 1\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"index-graveyard\" : {\n" +
+            "      \"tombstones\" : [ ]\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"routing_table\" : {\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"shards\" : {\n" +
+            "          \"1\" : [\n" +
+            "            {\n" +
+            "              \"state\" : \"STARTED\",\n" +
+            "              \"primary\" : true,\n" +
+            "              \"node\" : \"nodeId2\",\n" +
+            "              \"relocating_node\" : null,\n" +
+            "              \"shard\" : 1,\n" +
+            "              \"index\" : \"index\",\n" +
+            "              \"allocation_id\" : {\n" +
+            "                \"id\" : \"" + allocationId + "\"\n" +
+            "              }\n" +
+            "            }\n" +
+            "          ]\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"routing_nodes\" : {\n" +
+            "    \"unassigned\" : [ ],\n" +
+            "    \"nodes\" : {\n" +
+            "      \"nodeId2\" : [\n" +
+            "        {\n" +
+            "          \"state\" : \"STARTED\",\n" +
+            "          \"primary\" : true,\n" +
+            "          \"node\" : \"nodeId2\",\n" +
+            "          \"relocating_node\" : null,\n" +
+            "          \"shard\" : 1,\n" +
+            "          \"index\" : \"index\",\n" +
+            "          \"allocation_id\" : {\n" +
+            "            \"id\" : \"" + allocationId + "\"\n" +
+            "          }\n" +
+            "        }\n" +
+            "      ],\n" +
+            "      \"nodeId1\" : [ ]\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", Strings.toString(builder));
+
+    }
+
+    public void testToXContentSameTypeName() throws IOException {
+        final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+                                                    .stateUUID("stateUUID")
+                                                    .metaData(MetaData.builder()
+                                                        .clusterUUID("clusterUUID")
+                                                        .coordinationMetaData(CoordinationMetaData.builder()
+                                                            .build())
+                                                        .put(IndexMetaData.builder("index")
+                                                            .state(IndexMetaData.State.OPEN)
+                                                            .settings(Settings.builder()
+                                                                .put(SETTING_VERSION_CREATED, Version.CURRENT.id))
+                                                            .putMapping(new MappingMetaData("type",
+                                                                // the type name is the root value,
+                                                                // the original logic in ClusterState.toXContent will reduce
+                                                                new HashMap<>(){{
+                                                                    put("type", new HashMap<String, Object>(){{
+                                                                        put("key", "value");
+                                                                    }});
+                                                                }}))
+                                                            .numberOfShards(1)
+                                                            .primaryTerm(0, 1L)
+                                                            .numberOfReplicas(2)))
+                                                    .build();
+
+        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
+        builder.startObject();
+        clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+
+        assertEquals("{\n" +
+            "  \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "  \"version\" : 0,\n" +
+            "  \"state_uuid\" : \"stateUUID\",\n" +
+            "  \"master_node\" : null,\n" +
+            "  \"blocks\" : { },\n" +
+            "  \"nodes\" : { },\n" +
+            "  \"metadata\" : {\n" +
+            "    \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "    \"cluster_uuid_committed\" : false,\n" +
+            "    \"cluster_coordination\" : {\n" +
+            "      \"term\" : 0,\n" +
+            "      \"last_committed_config\" : [ ],\n" +
+            "      \"last_accepted_config\" : [ ],\n" +
+            "      \"voting_config_exclusions\" : [ ]\n" +
+            "    },\n" +
+            "    \"templates\" : { },\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"version\" : 2,\n" +
+            "        \"mapping_version\" : 1,\n" +
+            "        \"settings_version\" : 1,\n" +
+            "        \"aliases_version\" : 1,\n" +
+            "        \"routing_num_shards\" : 1,\n" +
+            "        \"state\" : \"open\",\n" +
+            "        \"settings\" : {\n" +
+            "          \"index\" : {\n" +
+            "            \"number_of_shards\" : \"1\",\n" +
+            "            \"number_of_replicas\" : \"2\",\n" +
+            "            \"version\" : {\n" +
+            "              \"created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"type\" : {\n" +
+            "            \"key\" : \"value\"\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"aliases\" : [ ],\n" +
+            "        \"primary_terms\" : {\n" +
+            "          \"0\" : 1\n" +
+            "        },\n" +
+            "        \"in_sync_allocations\" : {\n" +
+            "          \"0\" : [ ]\n" +
+            "        },\n" +
+            "        \"rollover_info\" : { }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"index-graveyard\" : {\n" +
+            "      \"tombstones\" : [ ]\n" +
+            "    }\n" +
+            "  },\n" +
+            "  \"routing_table\" : {\n" +
+            "    \"indices\" : { }\n" +
+            "  },\n" +
+            "  \"routing_nodes\" : {\n" +
+            "    \"unassigned\" : [ ],\n" +
+            "    \"nodes\" : { }\n" +
+            "  }\n" +
+            "}", Strings.toString(builder));
+    }
+
+    private ClusterState buildClusterState() throws IOException {
+        IndexMetaData indexMetaData = IndexMetaData.builder("index")
+            .state(IndexMetaData.State.OPEN)
+            .settings(Settings.builder()
+                .put(SETTING_VERSION_CREATED, Version.CURRENT.id))
+            .putMapping(new MappingMetaData("type",
+                new HashMap<>() {{
+                    put("type1", new HashMap<String, Object>(){{
+                        put("key", "value");
+                    }});
+                }}))
+            .putAlias(AliasMetaData.builder("alias")
+                .indexRouting("indexRouting")
+                .build())
+            .numberOfShards(1)
+            .primaryTerm(0, 1L)
+            .putInSyncAllocationIds(0, new HashSet<>() {{
+                add("allocationId");
+            }})
+            .numberOfReplicas(2)
+            .putRolloverInfo(new RolloverInfo("rolloveAlias", new ArrayList<>(), 1L))
+            .build();
+
+        return ClusterState.builder(ClusterName.DEFAULT)
+            .stateUUID("stateUUID")
+            .nodes(DiscoveryNodes.builder()
+                .masterNodeId("masterNodeId")
+                .add(new DiscoveryNode("nodeId1", new TransportAddress(InetAddress.getByName("127.0.0.1"), 111),
+                                                                Version.CURRENT))
+                .build())
+            .blocks(ClusterBlocks.builder()
+                .addGlobalBlock(
+                    new ClusterBlock(1, "description", true, true, true, RestStatus.ACCEPTED, EnumSet.allOf((ClusterBlockLevel.class))))
+                .addBlocks(indexMetaData)
+                .addIndexBlock("index",
+                    new ClusterBlock(2, "description2", false, false, false, RestStatus.ACCEPTED, EnumSet.allOf((ClusterBlockLevel.class))))
+                .build())
+            .metaData(MetaData.builder()
+                .clusterUUID("clusterUUID")
+                .coordinationMetaData(CoordinationMetaData.builder()
+                    .term(1)
+                    .lastCommittedConfiguration(new CoordinationMetaData.VotingConfiguration(new HashSet<>(){{
+                        add("commitedConfigurationNodeId");
+                    }}))
+                    .lastAcceptedConfiguration(new CoordinationMetaData.VotingConfiguration(new HashSet<>(){{
+                        add("acceptedConfigurationNodeId");
+                    }}))
+                    .addVotingConfigExclusion(new CoordinationMetaData.VotingConfigExclusion("exlucdedNodeId", "excludedNodeName"))
+                    .build())
+                .persistentSettings(Settings.builder()
+                    .put(SETTING_VERSION_CREATED, Version.CURRENT.id).build())
+                .transientSettings(Settings.builder()
+                    .put(SETTING_VERSION_CREATED, Version.CURRENT.id).build())
+                .put(indexMetaData, false)
+                .put(IndexTemplateMetaData.builder("template")
+                    .patterns(List.of("pattern1", "pattern2"))
+                    .order(0)
+                    .settings(Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT.id))
+                    .putMapping("type", "{ \"key1\": {} }")
+                    .build()))
+            .routingTable(RoutingTable.builder()
+                .add(IndexRoutingTable.builder(new Index("index", "indexUUID"))
+                    .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId("index", "_na_", 1))
+                        .addShard(TestShardRouting.newShardRouting(
+                            new ShardId("index", "_na_", 1), "nodeId2", true, ShardRoutingState.STARTED))
+                        .build())
+                    .build())
+                .build())
+            .build();
+    }
+
+    public static class CustomMetaData extends TestCustomMetaData {
+        public static final String TYPE = "custom_md";
+
+        CustomMetaData(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<MetaData.XContentContext> context() {
+            return EnumSet.of(MetaData.XContentContext.GATEWAY, MetaData.XContentContext.SNAPSHOT);
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandTests.java
new file mode 100644
index 0000000000000..9ef39de9c5b26
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandTests.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.metadata.IndexGraveyard;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
+
+public class ElasticsearchNodeCommandTests extends ESTestCase {
+
+    public void testLoadStateWithoutMissingCustoms() throws IOException {
+        runLoadStateTest(false, false);
+    }
+
+    public void testLoadStateWithoutMissingCustomsButPreserved() throws IOException {
+        runLoadStateTest(false, true);
+    }
+
+    public void testLoadStateWithMissingCustomsButPreserved() throws IOException {
+        runLoadStateTest(true, true);
+    }
+
+    public void testLoadStateWithMissingCustomsAndNotPreserved() throws IOException {
+        runLoadStateTest(true, false);
+    }
+
+    private void runLoadStateTest(boolean hasMissingCustoms, boolean preserveUnknownCustoms) throws IOException {
+        final MetaData latestMetaData = randomMeta();
+        final XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        MetaData.FORMAT.toXContent(builder, latestMetaData);
+        builder.endObject();
+
+        MetaData loadedMetaData;
+        try (XContentParser parser = createParser(hasMissingCustoms ? ElasticsearchNodeCommand.namedXContentRegistry : xContentRegistry(),
+            JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
+            loadedMetaData = MetaData.fromXContent(parser);
+        }
+        assertThat(loadedMetaData.clusterUUID(), not(equalTo("_na_")));
+        assertThat(loadedMetaData.clusterUUID(), equalTo(latestMetaData.clusterUUID()));
+
+        // make sure the index tombstones are the same too
+        if (hasMissingCustoms) {
+            assertNotNull(loadedMetaData.custom(IndexGraveyard.TYPE));
+            assertThat(loadedMetaData.custom(IndexGraveyard.TYPE), instanceOf(ElasticsearchNodeCommand.UnknownMetaDataCustom.class));
+
+            if (preserveUnknownCustoms) {
+                // check that we reserialize unknown metadata correctly again
+                final Path tempdir = createTempDir();
+                MetaData.FORMAT.write(loadedMetaData, tempdir);
+                final MetaData reloadedMetaData = MetaData.FORMAT.loadLatestState(logger, xContentRegistry(), tempdir);
+                assertThat(reloadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard()));
+            }
+        }  else {
+            assertThat(loadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard()));
+        }
+    }
+
+    private MetaData randomMeta() {
+        MetaData.Builder mdBuilder = MetaData.builder();
+        mdBuilder.generateClusterUuidIfNeeded();
+        int numDelIndices = randomIntBetween(0, 5);
+        final IndexGraveyard.Builder graveyard = IndexGraveyard.builder();
+        for (int i = 0; i < numDelIndices; i++) {
+            graveyard.addTombstone(new Index(randomAlphaOfLength(10) + "del-idx-" + i, UUIDs.randomBase64UUID()));
+        }
+        mdBuilder.indexGraveyard(graveyard.build());
+        return mdBuilder.build();
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(Stream.of(ClusterModule.getNamedXWriteables().stream(), IndicesModule.getNamedXContents().stream())
+            .flatMap(Function.identity())
+            .collect(Collectors.toList()));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
index 470d573bb5212..82976d348f4e3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
@@ -85,7 +85,7 @@ public static ComponentTemplate randomInstance() {
         return new ComponentTemplate(template, randomBoolean() ? null : randomNonNegativeLong(), meta);
     }
 
-    private static Map<String, AliasMetaData> randomAliases() {
+    public static Map<String, AliasMetaData> randomAliases() {
         String aliasName = randomAlphaOfLength(5);
         AliasMetaData aliasMeta = AliasMetaData.builder(aliasName)
             .filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)))
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java
new file mode 100644
index 0000000000000..7ef4aad1a1802
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.AbstractNamedWriteableTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class DataStreamMetadataTests extends AbstractNamedWriteableTestCase<DataStreamMetadata> {
+
+    @Override
+    protected DataStreamMetadata createTestInstance() {
+        if (randomBoolean()) {
+            return new DataStreamMetadata(Collections.emptyMap());
+        }
+        Map<String, DataStream> dataStreams = new HashMap<>();
+        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+            dataStreams.put(randomAlphaOfLength(5), DataStreamTests.randomInstance());
+        }
+        return new DataStreamMetadata(dataStreams);
+    }
+
+    @Override
+    protected DataStreamMetadata mutateInstance(DataStreamMetadata instance) throws IOException {
+        return randomValueOtherThan(instance, this::createTestInstance);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Collections.singletonList(new NamedWriteableRegistry.Entry(DataStreamMetadata.class,
+            DataStreamMetadata.TYPE, DataStreamMetadata::new)));
+    }
+
+    @Override
+    protected Class<DataStreamMetadata> categoryClass() {
+        return DataStreamMetadata.class;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
index 072165ab098ef..e944f6b1de1b1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
@@ -18,8 +18,10 @@
  */
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 
 import java.io.IOException;
@@ -28,6 +30,19 @@
 
 public class DataStreamTests extends AbstractSerializingTestCase<DataStream> {
 
+    public static List<Index> randomIndexInstances() {
+        int numIndices = randomIntBetween(0, 128);
+        List<Index> indices = new ArrayList<>(numIndices);
+        for (int i = 0; i < numIndices; i++) {
+            indices.add(new Index(randomAlphaOfLength(10), UUIDs.randomBase64UUID(random())));
+        }
+        return indices;
+    }
+
+    public static DataStream randomInstance() {
+        return new DataStream(randomAlphaOfLength(10), randomAlphaOfLength(10), randomIndexInstances());
+    }
+
     @Override
     protected DataStream doParseInstance(XContentParser parser) throws IOException {
         return DataStream.fromXContent(parser);
@@ -40,11 +55,7 @@ protected Writeable.Reader<DataStream> instanceReader() {
 
     @Override
     protected DataStream createTestInstance() {
-        int numIndices = randomIntBetween(0, 128);
-        List<String> indices = new ArrayList<>(numIndices);
-        for (int i = 0; i < numIndices; i++) {
-            indices.add(randomAlphaOfLength(10));
-        }
-        return new DataStream(randomAlphaOfLength(10), randomAlphaOfLength(10), indices);
+        return randomInstance();
     }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
index e4fc30dad676d..8c8155e54c96d 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -95,7 +94,7 @@ public void testIndexMetaDataSerialization() throws IOException {
 
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        metaData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        IndexMetaData.FORMAT.toXContent(builder, metaData);
         builder.endObject();
         XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder));
         final IndexMetaData fromXContentMeta = IndexMetaData.fromXContent(parser);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java
index 94d37a52b89be..4e9afabaa0266 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java
@@ -19,11 +19,13 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import com.fasterxml.jackson.core.JsonParseException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutRequest;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -37,6 +39,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -197,21 +200,41 @@ public void testPutGlobalTemplateWithIndexHiddenSetting() throws Exception {
         assertThat(errors.get(0).getMessage(), containsString("global templates may not specify the setting index.hidden"));
     }
 
-    public void testAddComponentTemplate() {
+    public void testAddComponentTemplate() throws Exception{
+        MetaDataIndexTemplateService metaDataIndexTemplateService = getMetaDataIndexTemplateService();
         ClusterState state = ClusterState.EMPTY_STATE;
-        ComponentTemplate template = ComponentTemplateTests.randomInstance();
-        state = MetaDataIndexTemplateService.addComponentTemplate(state, false, "foo", template);
+        Template template = new Template(Settings.builder().build(), null, ComponentTemplateTests.randomAliases());
+        ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>());
+        state = metaDataIndexTemplateService.addComponentTemplate(state, false, "foo", componentTemplate);
 
         assertNotNull(state.metaData().componentTemplates().get("foo"));
-        assertThat(state.metaData().componentTemplates().get("foo"), equalTo(template));
+        assertThat(state.metaData().componentTemplates().get("foo"), equalTo(componentTemplate));
 
         final ClusterState throwState = ClusterState.builder(state).build();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> MetaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo", template));
+            () -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo", componentTemplate));
         assertThat(e.getMessage(), containsString("component template [foo] already exists"));
 
-        state = MetaDataIndexTemplateService.addComponentTemplate(state, randomBoolean(), "bar", template);
+        state = metaDataIndexTemplateService.addComponentTemplate(state, randomBoolean(), "bar", componentTemplate);
         assertNotNull(state.metaData().componentTemplates().get("bar"));
+
+        template = new Template(Settings.builder().build(), new CompressedXContent("{\"invalid\"}"),
+            ComponentTemplateTests.randomAliases());
+        ComponentTemplate componentTemplate2 = new ComponentTemplate(template, 1L, new HashMap<>());
+        expectThrows(JsonParseException.class,
+            () -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate2));
+
+        template = new Template(Settings.builder().build(), new CompressedXContent("{\"invalid\":\"invalid\"}"),
+            ComponentTemplateTests.randomAliases());
+        ComponentTemplate componentTemplate3 = new ComponentTemplate(template, 1L, new HashMap<>());
+        expectThrows(MapperParsingException.class,
+            () -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate3));
+
+        template = new Template(Settings.builder().put("invalid", "invalid").build(), new CompressedXContent("{}"),
+            ComponentTemplateTests.randomAliases());
+        ComponentTemplate componentTemplate4 = new ComponentTemplate(template, 1L, new HashMap<>());
+        expectThrows(IllegalArgumentException.class,
+            () -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate4));
     }
 
     private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
@@ -247,23 +270,7 @@ public void onFailure(Exception e) {
     }
 
     private List<Throwable> putTemplateDetail(PutRequest request) throws Exception {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
-        MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
-                Settings.EMPTY,
-                clusterService,
-                indicesService,
-                null,
-                null,
-                new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null),
-                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
-                null,
-                xContentRegistry(),
-                Collections.emptyList(),
-                true);
-        MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(
-                clusterService, createIndexService, new AliasValidator(), indicesService,
-                new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry());
+        MetaDataIndexTemplateService service = getMetaDataIndexTemplateService();
 
         final List<Throwable> throwables = new ArrayList<>();
         final CountDownLatch latch = new CountDownLatch(1);
@@ -282,4 +289,24 @@ public void onFailure(Exception e) {
         latch.await();
         return throwables;
     }
+
+    private MetaDataIndexTemplateService getMetaDataIndexTemplateService() {
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+        MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
+                Settings.EMPTY,
+                clusterService,
+                indicesService,
+                null,
+                null,
+                new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null),
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+                null,
+                xContentRegistry(),
+                Collections.emptyList(),
+                true);
+        return new MetaDataIndexTemplateService(
+                clusterService, createIndexService, new AliasValidator(), indicesService,
+                new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index bd2e48a20c2e4..c18fab31bdb75 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -399,7 +398,7 @@ public void testUnknownFieldClusterMetaData() throws IOException {
                 .endObject()
             .endObject());
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, metadata)) {
-            MetaData.Builder.fromXContent(parser, randomBoolean());
+            MetaData.Builder.fromXContent(parser);
             fail();
         } catch (IllegalArgumentException e) {
             assertEquals("Unexpected field [random]", e.getMessage());
@@ -438,7 +437,7 @@ public void testXContentWithIndexGraveyard() throws IOException {
         final MetaData originalMeta = MetaData.builder().indexGraveyard(graveyard).build();
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        originalMeta.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        MetaData.FORMAT.toXContent(builder, originalMeta);
         builder.endObject();
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
             final MetaData fromXContentMeta = MetaData.fromXContent(parser);
@@ -451,7 +450,7 @@ public void testXContentClusterUUID() throws IOException {
             .clusterUUIDCommitted(randomBoolean()).build();
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        originalMeta.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        MetaData.FORMAT.toXContent(builder, originalMeta);
         builder.endObject();
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
             final MetaData fromXContentMeta = MetaData.fromXContent(parser);
@@ -504,7 +503,7 @@ public void testXContentWithCoordinationMetaData() throws IOException {
 
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        metaData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        MetaData.FORMAT.toXContent(builder, metaData);
         builder.endObject();
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
@@ -939,6 +938,7 @@ public static MetaData randomMetaData() {
             .version(randomNonNegativeLong())
             .put("component_template_" + randomAlphaOfLength(3), ComponentTemplateTests.randomInstance())
             .put("index_template_v2_" + randomAlphaOfLength(3), IndexTemplateV2Tests.randomInstance())
+            .put(DataStreamTests.randomInstance())
             .build();
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
index d5e76c97e1ad4..fbbd33dcef51b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
@@ -20,95 +20,38 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
+import org.elasticsearch.cluster.coordination.CoordinationMetaData;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestCustomMetaData;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.cluster.metadata.AliasMetaData.newAliasMetaDataBuilder;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
+import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_API;
+import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
 
 public class ToAndFromJsonMetaDataTests extends ESTestCase {
 
     public void testSimpleJsonFromAndTo() throws IOException {
         MetaData metaData = MetaData.builder()
-                .put(IndexMetaData.builder("test1")
-                        .settings(settings(Version.CURRENT))
-                        .numberOfShards(1)
-                        .numberOfReplicas(2)
-                        .primaryTerm(0, 1))
-                .put(IndexMetaData.builder("test2")
-                        .settings(settings(Version.CURRENT).put("setting1", "value1").put("setting2", "value2"))
-                        .numberOfShards(2)
-                        .numberOfReplicas(3)
-                        .primaryTerm(0, 2)
-                        .primaryTerm(1, 2))
-                .put(IndexMetaData.builder("test3")
-                        .settings(settings(Version.CURRENT))
-                        .numberOfShards(1)
-                        .numberOfReplicas(2)
-                        .putMapping(MAPPING_SOURCE1))
-                .put(IndexMetaData.builder("test4")
-                        .settings(settings(Version.CURRENT))
-                        .numberOfShards(1)
-                        .numberOfReplicas(2)
-                        .creationDate(2L))
-                .put(IndexMetaData.builder("test5")
-                        .settings(settings(Version.CURRENT).put("setting1", "value1").put("setting2", "value2"))
-                        .numberOfShards(1)
-                        .numberOfReplicas(2)
-                        .putMapping(MAPPING_SOURCE2))
-                .put(IndexMetaData.builder("test6")
-                        .settings(settings(Version.CURRENT).put("setting1", "value1").put("setting2", "value2"))
-                        .numberOfShards(1)
-                        .numberOfReplicas(2)
-                        .creationDate(2L))
-                .put(IndexMetaData.builder("test7")
-                        .settings(settings(Version.CURRENT))
-                        .numberOfShards(1)
-                        .numberOfReplicas(2)
-                        .creationDate(2L)
-                        .putMapping(MAPPING_SOURCE2))
-                .put(IndexMetaData.builder("test8")
-                        .settings(settings(Version.CURRENT).put("setting1", "value1").put("setting2", "value2"))
-                        .numberOfShards(1)
-                        .numberOfReplicas(2)
-                        .putMapping(MAPPING_SOURCE1)
-                        .putAlias(newAliasMetaDataBuilder("alias1"))
-                        .putAlias(newAliasMetaDataBuilder("alias2")))
-                .put(IndexMetaData.builder("test9")
-                        .settings(settings(Version.CURRENT).put("setting1", "value1").put("setting2", "value2"))
-                        .creationDate(2L)
-                        .numberOfShards(1)
-                        .numberOfReplicas(2)
-                        .putMapping(MAPPING_SOURCE1)
-                        .putAlias(newAliasMetaDataBuilder("alias1"))
-                        .putAlias(newAliasMetaDataBuilder("alias2")))
-                .put(IndexMetaData.builder("test10")
-                        .settings(settings(Version.CURRENT)
-                                .put("setting1", "value1")
-                                .put("setting2", "value2"))
-                        .numberOfShards(1)
-                        .numberOfReplicas(2)
-                        .putMapping(MAPPING_SOURCE1)
-                        .putAlias(newAliasMetaDataBuilder("alias1"))
-                        .putAlias(newAliasMetaDataBuilder("alias2")))
-                .put(IndexMetaData.builder("test11")
-                        .settings(settings(Version.CURRENT)
-                                .put("setting1", "value1")
-                                .put("setting2", "value2"))
-                        .numberOfShards(1)
-                        .numberOfReplicas(2)
-                        .putMapping(MAPPING_SOURCE1)
-                        .putAlias(newAliasMetaDataBuilder("alias1").filter(ALIAS_FILTER1))
-                        .putAlias(newAliasMetaDataBuilder("alias2").writeIndex(randomBoolean() ? null : randomBoolean()))
-                        .putAlias(newAliasMetaDataBuilder("alias4").filter(ALIAS_FILTER2)))
                 .put(IndexTemplateMetaData.builder("foo")
                         .patterns(Collections.singletonList("bar"))
                         .order(1)
@@ -151,141 +94,16 @@ public void testSimpleJsonFromAndTo() throws IOException {
                         .putAlias(newAliasMetaDataBuilder("alias-bar1"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
                         .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
+                .put(new DataStream("data-stream1", "@timestamp", Collections.emptyList()))
+                .put(new DataStream("data-stream2", "@timestamp2", Collections.emptyList()))
                 .build();
 
-        String metaDataSource = MetaData.Builder.toXContent(metaData);
-
-        MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource), false);
-
-        IndexMetaData indexMetaData = parsedMetaData.index("test1");
-        assertThat(indexMetaData.primaryTerm(0), equalTo(1L));
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(3));
-        assertNull(indexMetaData.mapping());
-
-        indexMetaData = parsedMetaData.index("test2");
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(2));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(3));
-        assertThat(indexMetaData.primaryTerm(0), equalTo(2L));
-        assertThat(indexMetaData.primaryTerm(1), equalTo(2L));
-        assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(5));
-        assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
-        assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
-        assertNull(indexMetaData.mapping());
-
-        indexMetaData = parsedMetaData.index("test3");
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(3));
-        assertThat(indexMetaData.mapping().source().string(), equalTo(MAPPING_SOURCE1));
-
-        indexMetaData = parsedMetaData.index("test4");
-        assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getSettings().size(), equalTo(4));
-        assertNull(indexMetaData.mapping());
-
-        indexMetaData = parsedMetaData.index("test5");
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(5));
-        assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
-        assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
-        assertThat(indexMetaData.mapping().source().string(), equalTo(MAPPING_SOURCE2));
-
-        indexMetaData = parsedMetaData.index("test6");
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(6));
-        assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
-        assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
-        assertNull(indexMetaData.mapping());
-
-        indexMetaData = parsedMetaData.index("test7");
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(4));
-        assertThat(indexMetaData.mapping().source().string(), equalTo(MAPPING_SOURCE2));
-
-        indexMetaData = parsedMetaData.index("test8");
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(5));
-        assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
-        assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
-        assertThat(indexMetaData.mapping().source().string(), equalTo(MAPPING_SOURCE1));
-        assertThat(indexMetaData.getAliases().size(), equalTo(2));
-        assertThat(indexMetaData.getAliases().get("alias1").alias(), equalTo("alias1"));
-        assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2"));
-
-        indexMetaData = parsedMetaData.index("test9");
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(6));
-        assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
-        assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
-        assertThat(indexMetaData.mapping().source().string(), equalTo(MAPPING_SOURCE1));
-        assertThat(indexMetaData.getAliases().size(), equalTo(2));
-        assertThat(indexMetaData.getAliases().get("alias1").alias(), equalTo("alias1"));
-        assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2"));
-
-        indexMetaData = parsedMetaData.index("test10");
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(5));
-        assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
-        assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
-        assertThat(indexMetaData.mapping().source().string(), equalTo(MAPPING_SOURCE1));
-        assertThat(indexMetaData.getAliases().size(), equalTo(2));
-        assertThat(indexMetaData.getAliases().get("alias1").alias(), equalTo("alias1"));
-        assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2"));
-
-        indexMetaData = parsedMetaData.index("test11");
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(5));
-        assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
-        assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
-        assertThat(indexMetaData.mapping().source().string(), equalTo(MAPPING_SOURCE1));
-        assertThat(indexMetaData.getAliases().size(), equalTo(3));
-        assertThat(indexMetaData.getAliases().get("alias1").alias(), equalTo("alias1"));
-        assertThat(indexMetaData.getAliases().get("alias1").filter().string(), equalTo(ALIAS_FILTER1));
-        assertThat(indexMetaData.getAliases().get("alias2").alias(), equalTo("alias2"));
-        assertThat(indexMetaData.getAliases().get("alias2").filter(), nullValue());
-        assertThat(indexMetaData.getAliases().get("alias2").writeIndex(),
-            equalTo(metaData.index("test11").getAliases().get("alias2").writeIndex()));
-        assertThat(indexMetaData.getAliases().get("alias4").alias(), equalTo("alias4"));
-        assertThat(indexMetaData.getAliases().get("alias4").filter().string(), equalTo(ALIAS_FILTER2));
-
-        indexMetaData = parsedMetaData.index("test12");
-        assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
-        assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().size(), equalTo(6));
-        assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
-        assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
-        assertThat(indexMetaData.mapping().source().string(), equalTo(MAPPING_SOURCE1));
-        assertThat(indexMetaData.getAliases().size(), equalTo(3));
-        assertThat(indexMetaData.getAliases().get("alias1").alias(), equalTo("alias1"));
-        assertThat(indexMetaData.getAliases().get("alias1").filter().string(), equalTo(ALIAS_FILTER1));
-        assertThat(indexMetaData.getAliases().get("alias3").alias(), equalTo("alias3"));
-        assertThat(indexMetaData.getAliases().get("alias3").filter(), nullValue());
-        assertThat(indexMetaData.getAliases().get("alias3").writeIndex(),
-            equalTo(metaData.index("test12").getAliases().get("alias3").writeIndex()));
-        assertThat(indexMetaData.getAliases().get("alias4").alias(), equalTo("alias4"));
-        assertThat(indexMetaData.getAliases().get("alias4").filter().string(), equalTo(ALIAS_FILTER2));
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        MetaData.FORMAT.toXContent(builder, metaData);
+        builder.endObject();
+
+        MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(builder));
 
         // templates
         assertThat(parsedMetaData.templates().get("foo").name(), is("foo"));
@@ -323,10 +141,486 @@ public void testSimpleJsonFromAndTo() throws IOException {
             equalTo(new Template(Settings.builder().put("setting", "value").build(),
                 new CompressedXContent("{\"baz\":\"eggplant\"}"),
                 Collections.singletonMap("alias", AliasMetaData.builder("alias").build()))));
+
+        // data streams
+        assertNotNull(parsedMetaData.dataStreams().get("data-stream1"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream1").getName(), is("data-stream1"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream1").getTimeStampField(), is("@timestamp"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream1").getIndices(), is(Collections.emptyList()));
+        assertNotNull(parsedMetaData.dataStreams().get("data-stream2"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream2").getName(), is("data-stream2"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream2").getTimeStampField(), is("@timestamp2"));
+        assertThat(parsedMetaData.dataStreams().get("data-stream2").getIndices(), is(Collections.emptyList()));
     }
 
     private static final String MAPPING_SOURCE1 = "{\"mapping1\":{\"text1\":{\"type\":\"string\"}}}";
     private static final String MAPPING_SOURCE2 = "{\"mapping2\":{\"text2\":{\"type\":\"string\"}}}";
     private static final String ALIAS_FILTER1 = "{\"field1\":\"value1\"}";
     private static final String ALIAS_FILTER2 = "{\"field2\":\"value2\"}";
+
+    public void testToXContentGateway_FlatSettingTrue_ReduceMappingFalse() throws IOException {
+        Map<String, String> mapParams = new HashMap<>(){{
+            put(MetaData.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY);
+            put("flat_settings", "true");
+            put("reduce_mappings", "false");
+        }};
+
+        MetaData metaData = buildMetaData();
+        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
+        builder.startObject();
+        metaData.toXContent(builder, new ToXContent.MapParams(mapParams));
+        builder.endObject();
+
+        assertEquals("{\n" +
+            "  \"meta-data\" : {\n" +
+            "    \"version\" : 0,\n" +
+            "    \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "    \"cluster_uuid_committed\" : false,\n" +
+            "    \"cluster_coordination\" : {\n" +
+            "      \"term\" : 1,\n" +
+            "      \"last_committed_config\" : [\n" +
+            "        \"commitedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"last_accepted_config\" : [\n" +
+            "        \"acceptedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"voting_config_exclusions\" : [\n" +
+            "        {\n" +
+            "          \"node_id\" : \"exlucdedNodeId\",\n" +
+            "          \"node_name\" : \"excludedNodeName\"\n" +
+            "        }\n" +
+            "      ]\n" +
+            "    },\n" +
+            "    \"settings\" : {\n" +
+            "      \"index.version.created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "    },\n" +
+            "    \"templates\" : {\n" +
+            "      \"template\" : {\n" +
+            "        \"order\" : 0,\n" +
+            "        \"index_patterns\" : [\n" +
+            "          \"pattern1\",\n" +
+            "          \"pattern2\"\n" +
+            "        ],\n" +
+            "        \"settings\" : {\n" +
+            "          \"index.version.created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"key1\" : { }\n" +
+            "        },\n" +
+            "        \"aliases\" : { }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"index-graveyard\" : {\n" +
+            "      \"tombstones\" : [ ]\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", Strings.toString(builder));
+    }
+
+    public void testToXContentAPI_SameTypeName() throws IOException {
+        Map<String, String> mapParams = new HashMap<>(){{
+            put(MetaData.CONTEXT_MODE_PARAM, CONTEXT_MODE_API);
+        }};
+
+        MetaData metaData = MetaData.builder()
+            .clusterUUID("clusterUUID")
+            .coordinationMetaData(CoordinationMetaData.builder()
+                .build())
+            .put(IndexMetaData.builder("index")
+                .state(IndexMetaData.State.OPEN)
+                .settings(Settings.builder()
+                    .put(SETTING_VERSION_CREATED, Version.CURRENT.id))
+                .putMapping(new MappingMetaData("type",
+                    // the type name is the root value,
+                    // the original logic in ClusterState.toXContent will reduce
+                    new HashMap<>(){{
+                        put("type", new HashMap<String, Object>(){{
+                            put("key", "value");
+                        }});
+                    }}))
+                .numberOfShards(1)
+                .primaryTerm(0, 1L)
+                .numberOfReplicas(2))
+            .build();
+        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
+        builder.startObject();
+        metaData.toXContent(builder, new ToXContent.MapParams(mapParams));
+        builder.endObject();
+
+        assertEquals("{\n" +
+            "  \"metadata\" : {\n" +
+            "    \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "    \"cluster_uuid_committed\" : false,\n" +
+            "    \"cluster_coordination\" : {\n" +
+            "      \"term\" : 0,\n" +
+            "      \"last_committed_config\" : [ ],\n" +
+            "      \"last_accepted_config\" : [ ],\n" +
+            "      \"voting_config_exclusions\" : [ ]\n" +
+            "    },\n" +
+            "    \"templates\" : { },\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"version\" : 2,\n" +
+            "        \"mapping_version\" : 1,\n" +
+            "        \"settings_version\" : 1,\n" +
+            "        \"aliases_version\" : 1,\n" +
+            "        \"routing_num_shards\" : 1,\n" +
+            "        \"state\" : \"open\",\n" +
+            "        \"settings\" : {\n" +
+            "          \"index\" : {\n" +
+            "            \"number_of_shards\" : \"1\",\n" +
+            "            \"number_of_replicas\" : \"2\",\n" +
+            "            \"version\" : {\n" +
+            "              \"created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"type\" : {\n" +
+            "            \"key\" : \"value\"\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"aliases\" : [ ],\n" +
+            "        \"primary_terms\" : {\n" +
+            "          \"0\" : 1\n" +
+            "        },\n" +
+            "        \"in_sync_allocations\" : {\n" +
+            "          \"0\" : [ ]\n" +
+            "        },\n" +
+            "        \"rollover_info\" : { }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"index-graveyard\" : {\n" +
+            "      \"tombstones\" : [ ]\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", Strings.toString(builder));
+    }
+
+    public void testToXContentGateway_FlatSettingFalse_ReduceMappingTrue() throws IOException {
+        Map<String, String> mapParams = new HashMap<>(){{
+            put(MetaData.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY);
+            put("flat_settings", "false");
+            put("reduce_mappings", "true");
+        }};
+
+        MetaData metaData = buildMetaData();
+        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
+        builder.startObject();
+        metaData.toXContent(builder, new ToXContent.MapParams(mapParams));
+        builder.endObject();
+
+        assertEquals("{\n" +
+            "  \"meta-data\" : {\n" +
+            "    \"version\" : 0,\n" +
+            "    \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "    \"cluster_uuid_committed\" : false,\n" +
+            "    \"cluster_coordination\" : {\n" +
+            "      \"term\" : 1,\n" +
+            "      \"last_committed_config\" : [\n" +
+            "        \"commitedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"last_accepted_config\" : [\n" +
+            "        \"acceptedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"voting_config_exclusions\" : [\n" +
+            "        {\n" +
+            "          \"node_id\" : \"exlucdedNodeId\",\n" +
+            "          \"node_name\" : \"excludedNodeName\"\n" +
+            "        }\n" +
+            "      ]\n" +
+            "    },\n" +
+            "    \"settings\" : {\n" +
+            "      \"index.version.created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "    },\n" +
+            "    \"templates\" : {\n" +
+            "      \"template\" : {\n" +
+            "        \"order\" : 0,\n" +
+            "        \"index_patterns\" : [\n" +
+            "          \"pattern1\",\n" +
+            "          \"pattern2\"\n" +
+            "        ],\n" +
+            "        \"settings\" : {\n" +
+            "          \"index\" : {\n" +
+            "            \"version\" : {\n" +
+            "              \"created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"mappings\" : { },\n" +
+            "        \"aliases\" : { }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"index-graveyard\" : {\n" +
+            "      \"tombstones\" : [ ]\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", Strings.toString(builder));
+    }
+
+    public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOException {
+        Map<String, String> mapParams = new HashMap<>(){{
+            put(MetaData.CONTEXT_MODE_PARAM, CONTEXT_MODE_API);
+            put("flat_settings", "true");
+            put("reduce_mappings", "false");
+        }};
+
+        final MetaData metaData = buildMetaData();
+
+        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
+        builder.startObject();
+        metaData.toXContent(builder, new ToXContent.MapParams(mapParams));
+        builder.endObject();
+
+        assertEquals("{\n" +
+            "  \"metadata\" : {\n" +
+            "    \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "    \"cluster_uuid_committed\" : false,\n" +
+            "    \"cluster_coordination\" : {\n" +
+            "      \"term\" : 1,\n" +
+            "      \"last_committed_config\" : [\n" +
+            "        \"commitedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"last_accepted_config\" : [\n" +
+            "        \"acceptedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"voting_config_exclusions\" : [\n" +
+            "        {\n" +
+            "          \"node_id\" : \"exlucdedNodeId\",\n" +
+            "          \"node_name\" : \"excludedNodeName\"\n" +
+            "        }\n" +
+            "      ]\n" +
+            "    },\n" +
+            "    \"templates\" : {\n" +
+            "      \"template\" : {\n" +
+            "        \"order\" : 0,\n" +
+            "        \"index_patterns\" : [\n" +
+            "          \"pattern1\",\n" +
+            "          \"pattern2\"\n" +
+            "        ],\n" +
+            "        \"settings\" : {\n" +
+            "          \"index.version.created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"key1\" : { }\n" +
+            "        },\n" +
+            "        \"aliases\" : { }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"version\" : 2,\n" +
+            "        \"mapping_version\" : 1,\n" +
+            "        \"settings_version\" : 1,\n" +
+            "        \"aliases_version\" : 1,\n" +
+            "        \"routing_num_shards\" : 1,\n" +
+            "        \"state\" : \"open\",\n" +
+            "        \"settings\" : {\n" +
+            "          \"index.number_of_replicas\" : \"2\",\n" +
+            "          \"index.number_of_shards\" : \"1\",\n" +
+            "          \"index.version.created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"type\" : {\n" +
+            "            \"type1\" : {\n" +
+            "              \"key\" : \"value\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"aliases\" : [\n" +
+            "          \"alias\"\n" +
+            "        ],\n" +
+            "        \"primary_terms\" : {\n" +
+            "          \"0\" : 1\n" +
+            "        },\n" +
+            "        \"in_sync_allocations\" : {\n" +
+            "          \"0\" : [\n" +
+            "            \"allocationId\"\n" +
+            "          ]\n" +
+            "        },\n" +
+            "        \"rollover_info\" : {\n" +
+            "          \"rolloveAlias\" : {\n" +
+            "            \"met_conditions\" : { },\n" +
+            "            \"time\" : 1\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"index-graveyard\" : {\n" +
+            "      \"tombstones\" : [ ]\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", Strings.toString(builder));
+    }
+
+    public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOException {
+        Map<String, String> mapParams = new HashMap<>(){{
+            put(MetaData.CONTEXT_MODE_PARAM, CONTEXT_MODE_API);
+            put("flat_settings", "false");
+            put("reduce_mappings", "true");
+        }};
+
+        final MetaData metaData = buildMetaData();
+
+        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
+        builder.startObject();
+        metaData.toXContent(builder, new ToXContent.MapParams(mapParams));
+        builder.endObject();
+
+        assertEquals("{\n" +
+            "  \"metadata\" : {\n" +
+            "    \"cluster_uuid\" : \"clusterUUID\",\n" +
+            "    \"cluster_uuid_committed\" : false,\n" +
+            "    \"cluster_coordination\" : {\n" +
+            "      \"term\" : 1,\n" +
+            "      \"last_committed_config\" : [\n" +
+            "        \"commitedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"last_accepted_config\" : [\n" +
+            "        \"acceptedConfigurationNodeId\"\n" +
+            "      ],\n" +
+            "      \"voting_config_exclusions\" : [\n" +
+            "        {\n" +
+            "          \"node_id\" : \"exlucdedNodeId\",\n" +
+            "          \"node_name\" : \"excludedNodeName\"\n" +
+            "        }\n" +
+            "      ]\n" +
+            "    },\n" +
+            "    \"templates\" : {\n" +
+            "      \"template\" : {\n" +
+            "        \"order\" : 0,\n" +
+            "        \"index_patterns\" : [\n" +
+            "          \"pattern1\",\n" +
+            "          \"pattern2\"\n" +
+            "        ],\n" +
+            "        \"settings\" : {\n" +
+            "          \"index\" : {\n" +
+            "            \"version\" : {\n" +
+            "              \"created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"mappings\" : { },\n" +
+            "        \"aliases\" : { }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"indices\" : {\n" +
+            "      \"index\" : {\n" +
+            "        \"version\" : 2,\n" +
+            "        \"mapping_version\" : 1,\n" +
+            "        \"settings_version\" : 1,\n" +
+            "        \"aliases_version\" : 1,\n" +
+            "        \"routing_num_shards\" : 1,\n" +
+            "        \"state\" : \"open\",\n" +
+            "        \"settings\" : {\n" +
+            "          \"index\" : {\n" +
+            "            \"number_of_shards\" : \"1\",\n" +
+            "            \"number_of_replicas\" : \"2\",\n" +
+            "            \"version\" : {\n" +
+            "              \"created\" : \"" + Version.CURRENT.id + "\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"mappings\" : {\n" +
+            "          \"type\" : {\n" +
+            "            \"type1\" : {\n" +
+            "              \"key\" : \"value\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"aliases\" : [\n" +
+            "          \"alias\"\n" +
+            "        ],\n" +
+            "        \"primary_terms\" : {\n" +
+            "          \"0\" : 1\n" +
+            "        },\n" +
+            "        \"in_sync_allocations\" : {\n" +
+            "          \"0\" : [\n" +
+            "            \"allocationId\"\n" +
+            "          ]\n" +
+            "        },\n" +
+            "        \"rollover_info\" : {\n" +
+            "          \"rolloveAlias\" : {\n" +
+            "            \"met_conditions\" : { },\n" +
+            "            \"time\" : 1\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    },\n" +
+            "    \"index-graveyard\" : {\n" +
+            "      \"tombstones\" : [ ]\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", Strings.toString(builder));
+    }
+
+
+    private MetaData buildMetaData() throws IOException {
+        return MetaData.builder()
+            .clusterUUID("clusterUUID")
+            .coordinationMetaData(CoordinationMetaData.builder()
+                .term(1)
+                .lastCommittedConfiguration(new CoordinationMetaData.VotingConfiguration(new HashSet<>(){{
+                    add("commitedConfigurationNodeId");
+                }}))
+                .lastAcceptedConfiguration(new CoordinationMetaData.VotingConfiguration(new HashSet<>(){{
+                    add("acceptedConfigurationNodeId");
+                }}))
+                .addVotingConfigExclusion(new CoordinationMetaData.VotingConfigExclusion("exlucdedNodeId", "excludedNodeName"))
+                .build())
+            .persistentSettings(Settings.builder()
+                .put(SETTING_VERSION_CREATED, Version.CURRENT.id).build())
+            .transientSettings(Settings.builder()
+                .put(SETTING_VERSION_CREATED, Version.CURRENT.id).build())
+            .put(IndexMetaData.builder("index")
+                .state(IndexMetaData.State.OPEN)
+                .settings(Settings.builder()
+                    .put(SETTING_VERSION_CREATED, Version.CURRENT.id))
+                .putMapping(new MappingMetaData("type",
+                    new HashMap<>(){{
+                        put("type1", new HashMap<String, Object>(){{
+                            put("key", "value");
+                        }});
+                    }}))
+                .putAlias(AliasMetaData.builder("alias")
+                    .indexRouting("indexRouting")
+                    .build())
+                .numberOfShards(1)
+                .primaryTerm(0, 1L)
+                .putInSyncAllocationIds(0, new HashSet<>(){{
+                    add("allocationId");
+                }})
+                .numberOfReplicas(2)
+                .putRolloverInfo(new RolloverInfo("rolloveAlias", new ArrayList<>(), 1L)))
+            .put(IndexTemplateMetaData.builder("template")
+                .patterns(List.of("pattern1", "pattern2"))
+                .order(0)
+                .settings(Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT.id))
+                .putMapping("type", "{ \"key1\": {} }")
+                .build())
+            .build();
+    }
+
+    public static class CustomMetaData extends TestCustomMetaData {
+        public static final String TYPE = "custom_md";
+
+        CustomMetaData(String data) {
+            super(data);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.CURRENT;
+        }
+
+        @Override
+        public EnumSet<MetaData.XContentContext> context() {
+            return EnumSet.of(MetaData.XContentContext.GATEWAY, MetaData.XContentContext.SNAPSHOT);
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
index b020c7e03f107..26ea2c49fb980 100644
--- a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
@@ -22,13 +22,18 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 
 import java.net.InetAddress;
+import java.util.HashSet;
+import java.util.Set;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.hamcrest.Matchers.equalTo;
 
 public class DiscoveryNodeTests extends ESTestCase {
 
@@ -58,4 +63,29 @@ public void testDiscoveryNodeSerializationKeepsHost() throws Exception {
         assertEquals(transportAddress.getAddress(), serialized.getAddress().getAddress());
         assertEquals(transportAddress.getPort(), serialized.getAddress().getPort());
     }
+
+    public void testDiscoveryNodeIsRemoteClusterClientDefault() {
+        runTestDiscoveryNodeIsRemoteClusterClient(Settings.EMPTY, true);
+    }
+
+    public void testDiscoveryNodeIsRemoteClusterClientSet() {
+        runTestDiscoveryNodeIsRemoteClusterClient(Settings.builder().put(Node.NODE_REMOTE_CLUSTER_CLIENT.getKey(), true).build(), true);
+    }
+
+    public void testDiscoveryNodeIsRemoteClusterClientUnset() {
+        runTestDiscoveryNodeIsRemoteClusterClient(Settings.builder().put(Node.NODE_REMOTE_CLUSTER_CLIENT.getKey(), false).build(), false);
+    }
+
+    private void runTestDiscoveryNodeIsRemoteClusterClient(final Settings settings, final boolean expected) {
+        final DiscoveryNode node = DiscoveryNode.createLocal(settings, new TransportAddress(TransportAddress.META_ADDRESS, 9200), "node");
+        assertThat(node.isRemoteClusterClient(), equalTo(expected));
+        final Set<DiscoveryNodeRole> expectedRoles = new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES);
+        if (expected) {
+            assertThat(node.getRoles(), equalTo(expectedRoles));
+        } else {
+            expectedRoles.remove(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE);
+            assertThat(node.getRoles(), equalTo(expectedRoles));
+        }
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java b/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java
index c02b4e460d28d..33d11aa23d890 100644
--- a/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java
+++ b/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java
@@ -21,13 +21,11 @@
 
 import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.TestUtil;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -390,12 +388,9 @@ private void doTest(byte bytes[]) throws IOException {
         StreamInput rawIn = new ByteBufferStreamInput(bb);
         Compressor c = compressor;
 
-        final Version version = VersionUtils.randomVersion(random());
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
         OutputStreamStreamOutput rawOs = new OutputStreamStreamOutput(bos);
-        rawOs.setVersion(version);
         StreamOutput os = c.streamOutput(rawOs);
-        assertEquals(version, os.getVersion());
 
         Random r = random();
         int bufferSize = r.nextBoolean() ? 65535 : TestUtil.nextInt(random(), 1, 70000);
@@ -415,9 +410,7 @@ private void doTest(byte bytes[]) throws IOException {
         byte compressed[] = bos.toByteArray();
         ByteBuffer bb2 = ByteBuffer.wrap(compressed);
         StreamInput compressedIn = new ByteBufferStreamInput(bb2);
-        compressedIn.setVersion(version);
         StreamInput in = c.streamInput(compressedIn);
-        assertEquals(version, in.getVersion());
 
         // randomize constants again
         bufferSize = r.nextBoolean() ? 65535 : TestUtil.nextInt(random(), 1, 70000);
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
index 94e2801df8cee..195abb85939df 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.io.stream;
 
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -49,12 +50,14 @@
 import java.util.stream.IntStream;
 
 import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
 
 /**
  * Tests for {@link BytesStreamOutput} paging behaviour.
@@ -850,4 +853,28 @@ public void testTimeValueSerialize() throws Exception {
         assertEqualityAfterSerialize(timeValue, 1 + out.bytes().length());
     }
 
+    public void testWriteCircularReferenceException() throws IOException {
+        IOException rootEx = new IOException("disk broken");
+        AlreadyClosedException ace = new AlreadyClosedException("closed", rootEx);
+        rootEx.addSuppressed(ace); // circular reference
+
+        BytesStreamOutput testOut = new BytesStreamOutput();
+        AssertionError error = expectThrows(AssertionError.class, () -> testOut.writeException(rootEx));
+        assertThat(error.getMessage(), containsString("too many nested exceptions"));
+        assertThat(error.getCause(), equalTo(rootEx));
+
+        BytesStreamOutput prodOut = new BytesStreamOutput() {
+            @Override
+            boolean failOnTooManyNestedExceptions(Throwable throwable) {
+                assertThat(throwable, sameInstance(rootEx));
+                return true;
+            }
+        };
+        prodOut.writeException(rootEx);
+        StreamInput in = prodOut.bytes().streamInput();
+        Exception newEx = in.readException();
+        assertThat(newEx, instanceOf(IOException.class));
+        assertThat(newEx.getMessage(), equalTo("disk broken"));
+        assertArrayEquals(newEx.getStackTrace(), rootEx.getStackTrace());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
new file mode 100644
index 0000000000000..d12a15c1a7207
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.io.stream;
+
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.equalTo;
+
+import java.io.IOException;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+public class DelayableWriteableTests extends ESTestCase {
+    // NOTE: we don't use AbstractWireSerializingTestCase because we don't implement equals and hashCode.
+    private static class Example implements NamedWriteable {
+        private final String s;
+
+        Example(String s) {
+            this.s = s;
+        }
+
+        Example(StreamInput in) throws IOException {
+            s = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(s);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return "example";
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || getClass() != obj.getClass()) {
+                return false;
+            }
+            Example other = (Example) obj;
+            return s.equals(other.s);
+        }
+
+        @Override
+        public int hashCode() {
+            return s.hashCode();
+        }
+    }
+
+    private static class NamedHolder implements Writeable {
+        private final Example e;
+
+        NamedHolder(Example e) {
+            this.e = e;
+        }
+
+        NamedHolder(StreamInput in) throws IOException {
+            e = in.readNamedWriteable(Example.class);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeNamedWriteable(e);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || getClass() != obj.getClass()) {
+                return false;
+            }
+            NamedHolder other = (NamedHolder) obj;
+            return e.equals(other.e);
+        }
+
+        @Override
+        public int hashCode() {
+            return e.hashCode();
+        }
+    }
+
+    private static class SneakOtherSideVersionOnWire implements Writeable {
+        private final Version version;
+
+        SneakOtherSideVersionOnWire() {
+            version = Version.CURRENT;
+        }
+
+        SneakOtherSideVersionOnWire(StreamInput in) throws IOException {
+            version = Version.readVersion(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            Version.writeVersion(out.getVersion(), out);
+        }
+    }
+
+    public void testRoundTripFromReferencing() throws IOException {
+        Example e = new Example(randomAlphaOfLength(5));
+        DelayableWriteable<Example> original = DelayableWriteable.referencing(e);
+        assertFalse(original.isDelayed());
+        roundTripTestCase(original, Example::new);
+    }
+
+    public void testRoundTripFromReferencingWithNamedWriteable() throws IOException {
+        NamedHolder n = new NamedHolder(new Example(randomAlphaOfLength(5)));
+        DelayableWriteable<NamedHolder> original = DelayableWriteable.referencing(n);
+        assertFalse(original.isDelayed());
+        roundTripTestCase(original, NamedHolder::new);
+    }
+
+    public void testRoundTripFromDelayed() throws IOException {
+        Example e = new Example(randomAlphaOfLength(5));
+        DelayableWriteable<Example> original = roundTrip(DelayableWriteable.referencing(e), Example::new, Version.CURRENT);
+        assertTrue(original.isDelayed());
+        roundTripTestCase(original, Example::new);
+    }
+
+    public void testRoundTripFromDelayedWithNamedWriteable() throws IOException {
+        NamedHolder n = new NamedHolder(new Example(randomAlphaOfLength(5)));
+        DelayableWriteable<NamedHolder> original = roundTrip(DelayableWriteable.referencing(n), NamedHolder::new, Version.CURRENT);
+        assertTrue(original.isDelayed());
+        roundTripTestCase(original, NamedHolder::new);
+    }
+
+    public void testRoundTripFromDelayedFromOldVersion() throws IOException {
+        Example e = new Example(randomAlphaOfLength(5));
+        DelayableWriteable<Example> original = roundTrip(DelayableWriteable.referencing(e), Example::new, randomOldVersion());
+        assertTrue(original.isDelayed());
+        roundTripTestCase(original, Example::new);
+    }
+
+    public void testRoundTripFromDelayedFromOldVersionWithNamedWriteable() throws IOException {
+        NamedHolder n = new NamedHolder(new Example(randomAlphaOfLength(5)));
+        DelayableWriteable<NamedHolder> original = roundTrip(DelayableWriteable.referencing(n), NamedHolder::new, randomOldVersion());
+        assertTrue(original.isDelayed());
+        roundTripTestCase(original, NamedHolder::new);
+    }
+
+    public void testSerializesWithRemoteVersion() throws IOException {
+        Version remoteVersion = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+        DelayableWriteable<SneakOtherSideVersionOnWire> original = DelayableWriteable.referencing(new SneakOtherSideVersionOnWire());
+        assertThat(roundTrip(original, SneakOtherSideVersionOnWire::new, remoteVersion).get().version, equalTo(remoteVersion));
+    }
+
+    private <T extends Writeable> void roundTripTestCase(DelayableWriteable<T> original, Writeable.Reader<T> reader) throws IOException {
+        DelayableWriteable<T> roundTripped = roundTrip(original, reader, Version.CURRENT);
+        assertTrue(roundTripped.isDelayed());
+        assertThat(roundTripped.get(), equalTo(original.get()));
+    }
+
+    private <T extends Writeable> DelayableWriteable<T> roundTrip(DelayableWriteable<T> original,
+            Writeable.Reader<T> reader, Version version) throws IOException {
+        return copyInstance(original, writableRegistry(), (out, d) -> d.writeTo(out),
+            in -> DelayableWriteable.delayed(reader, in), version);
+    }
+
+    @Override
+    protected NamedWriteableRegistry writableRegistry() {
+        return new NamedWriteableRegistry(singletonList(
+                new NamedWriteableRegistry.Entry(Example.class, "example", Example::new)));
+    }
+
+    private static Version randomOldVersion() {
+        return randomValueOtherThanMany(Version.CURRENT::before, () -> VersionUtils.randomCompatibleVersion(random(), Version.CURRENT));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index 5305b80c9828b..a2a456916e4d6 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -29,13 +29,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
-
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class ThreadContextTests extends ESTestCase {
@@ -258,49 +254,6 @@ public void testSerialize() throws IOException {
             threadContext.addResponseHeader("Warning", "123456");
         }
         threadContext.addResponseHeader("Warning", "234567");
-        threadContext.disallowSystemIndexAccess();
-
-        BytesStreamOutput out = new BytesStreamOutput();
-        threadContext.writeTo(out);
-        try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
-            assertNull(threadContext.getHeader("foo"));
-            assertNull(threadContext.getTransient("ctx.foo"));
-            assertTrue(threadContext.getResponseHeaders().isEmpty());
-            assertEquals("1", threadContext.getHeader("default"));
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
-
-            threadContext.readFrom(out.bytes().streamInput());
-            assertEquals("bar", threadContext.getHeader("foo"));
-            assertNull(threadContext.getTransient("ctx.foo"));
-
-            final Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
-            final List<String> warnings = responseHeaders.get("Warning");
-
-            assertThat(responseHeaders.keySet(), hasSize(1));
-            assertThat(warnings, hasSize(2));
-            assertThat(warnings, hasItem(equalTo("123456")));
-            assertThat(warnings, hasItem(equalTo("234567")));
-
-            assertFalse(threadContext.isSystemIndexAccessAllowed());
-        }
-        assertEquals("bar", threadContext.getHeader("foo"));
-        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
-        assertEquals("1", threadContext.getHeader("default"));
-        assertFalse(threadContext.isSystemIndexAccessAllowed());
-    }
-
-    public void testSerializeWithAllowedSystemIndexPatterns() throws IOException {
-        Settings build = Settings.builder().put("request.headers.default", "1").build();
-        ThreadContext threadContext = new ThreadContext(build);
-        threadContext.putHeader("foo", "bar");
-        threadContext.putTransient("ctx.foo", 1);
-        threadContext.addResponseHeader("Warning", "123456");
-        if (rarely()) {
-            threadContext.addResponseHeader("Warning", "123456");
-        }
-        threadContext.addResponseHeader("Warning", "234567");
-        final List<String> allowed = randomList(1, 8, () -> randomAlphaOfLengthBetween(2, 8));
-        threadContext.allowSystemIndexAccess(allowed);
 
         BytesStreamOutput out = new BytesStreamOutput();
         threadContext.writeTo(out);
@@ -309,9 +262,8 @@ public void testSerializeWithAllowedSystemIndexPatterns() throws IOException {
             assertNull(threadContext.getTransient("ctx.foo"));
             assertTrue(threadContext.getResponseHeaders().isEmpty());
             assertEquals("1", threadContext.getHeader("default"));
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
 
-            threadContext.readFrom(out.bytes().streamInput());
+            threadContext.readHeaders(out.bytes().streamInput());
             assertEquals("bar", threadContext.getHeader("foo"));
             assertNull(threadContext.getTransient("ctx.foo"));
 
@@ -322,15 +274,10 @@ public void testSerializeWithAllowedSystemIndexPatterns() throws IOException {
             assertThat(warnings, hasSize(2));
             assertThat(warnings, hasItem(equalTo("123456")));
             assertThat(warnings, hasItem(equalTo("234567")));
-
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
-            assertThat(threadContext.allowedSystemIndexPatterns(), equalTo(allowed));
         }
         assertEquals("bar", threadContext.getHeader("foo"));
         assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
-        assertTrue(threadContext.isSystemIndexAccessAllowed());
-        assertThat(threadContext.allowedSystemIndexPatterns(), equalTo(allowed));
     }
 
     public void testSerializeInDifferentContext() throws IOException {
@@ -345,63 +292,17 @@ public void testSerializeInDifferentContext() throws IOException {
                 threadContext.addResponseHeader("Warning", "123456");
             }
             threadContext.addResponseHeader("Warning", "234567");
-            threadContext.disallowSystemIndexAccess();
-
-            assertEquals("bar", threadContext.getHeader("foo"));
-            assertNotNull(threadContext.getTransient("ctx.foo"));
-            assertEquals("1", threadContext.getHeader("default"));
-            assertThat(threadContext.getResponseHeaders().keySet(), hasSize(1));
-            assertFalse(threadContext.isSystemIndexAccessAllowed());
-            threadContext.writeTo(out);
-        }
-        {
-            Settings otherSettings = Settings.builder().put("request.headers.default", "5").build();
-            ThreadContext otherThreadContext = new ThreadContext(otherSettings);
-            otherThreadContext.readFrom(out.bytes().streamInput());
-
-            assertEquals("bar", otherThreadContext.getHeader("foo"));
-            assertNull(otherThreadContext.getTransient("ctx.foo"));
-            assertEquals("1", otherThreadContext.getHeader("default"));
-
-            final Map<String, List<String>> responseHeaders = otherThreadContext.getResponseHeaders();
-            final List<String> warnings = responseHeaders.get("Warning");
-
-            assertThat(responseHeaders.keySet(), hasSize(1));
-            assertThat(warnings, hasSize(2));
-            assertThat(warnings, hasItem(equalTo("123456")));
-            assertThat(warnings, hasItem(equalTo("234567")));
-
-            assertFalse(otherThreadContext.isSystemIndexAccessAllowed());
-        }
-    }
-
-    public void testSerializeInDifferentContextWithAllowedSystemIndices() throws IOException {
-        final List<String> allowed = randomList(1, 8, () -> randomAlphaOfLengthBetween(2, 8));
-        BytesStreamOutput out = new BytesStreamOutput();
-        {
-            Settings build = Settings.builder().put("request.headers.default", "1").build();
-            ThreadContext threadContext = new ThreadContext(build);
-            threadContext.putHeader("foo", "bar");
-            threadContext.putTransient("ctx.foo", 1);
-            threadContext.addResponseHeader("Warning", "123456");
-            if (rarely()) {
-                threadContext.addResponseHeader("Warning", "123456");
-            }
-            threadContext.addResponseHeader("Warning", "234567");
-            threadContext.allowSystemIndexAccess(allowed);
 
             assertEquals("bar", threadContext.getHeader("foo"));
             assertNotNull(threadContext.getTransient("ctx.foo"));
             assertEquals("1", threadContext.getHeader("default"));
             assertThat(threadContext.getResponseHeaders().keySet(), hasSize(1));
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
-            assertThat(threadContext.allowedSystemIndexPatterns(), equalTo(allowed));
             threadContext.writeTo(out);
         }
         {
             Settings otherSettings = Settings.builder().put("request.headers.default", "5").build();
             ThreadContext otherThreadContext = new ThreadContext(otherSettings);
-            otherThreadContext.readFrom(out.bytes().streamInput());
+            otherThreadContext.readHeaders(out.bytes().streamInput());
 
             assertEquals("bar", otherThreadContext.getHeader("foo"));
             assertNull(otherThreadContext.getTransient("ctx.foo"));
@@ -414,9 +315,6 @@ public void testSerializeInDifferentContextWithAllowedSystemIndices() throws IOE
             assertThat(warnings, hasSize(2));
             assertThat(warnings, hasItem(equalTo("123456")));
             assertThat(warnings, hasItem(equalTo("234567")));
-
-            assertTrue(otherThreadContext.isSystemIndexAccessAllowed());
-            assertThat(otherThreadContext.allowedSystemIndexPatterns(), equalTo(allowed));
         }
     }
 
@@ -435,7 +333,7 @@ public void testSerializeInDifferentContextNoDefaults() throws IOException {
         {
             Settings otherSettings = Settings.builder().put("request.headers.default", "5").build();
             ThreadContext otherhreadContext = new ThreadContext(otherSettings);
-            otherhreadContext.readFrom(out.bytes().streamInput());
+            otherhreadContext.readHeaders(out.bytes().streamInput());
 
             assertEquals("bar", otherhreadContext.getHeader("foo"));
             assertNull(otherhreadContext.getTransient("ctx.foo"));
@@ -715,31 +613,6 @@ public void testPutHeaders() {
         assertEquals("value for key [foo] already present", e.getMessage());
     }
 
-    public void testSystemIndexAccessAllowed() {
-        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        assertTrue(threadContext.isSystemIndexAccessAllowed());
-        assertThat(threadContext.allowedSystemIndexPatterns(), empty());
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
-            threadContext.disallowSystemIndexAccess();
-            assertFalse(threadContext.isSystemIndexAccessAllowed());
-            assertThat(threadContext.allowedSystemIndexPatterns(), nullValue());
-        }
-        assertTrue(threadContext.isSystemIndexAccessAllowed());
-        assertThat(threadContext.allowedSystemIndexPatterns(), empty());
-
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-            threadContext.disallowSystemIndexAccess();
-            final List<String> allowed = randomList(1, 8, () -> randomAlphaOfLengthBetween(2, 8));
-            threadContext.allowSystemIndexAccess(allowed);
-            assertTrue(threadContext.isSystemIndexAccessAllowed());
-            assertThat(threadContext.allowedSystemIndexPatterns(), not(sameInstance(allowed)));
-            assertThat(threadContext.allowedSystemIndexPatterns(), equalTo(allowed));
-        }
-        assertTrue(threadContext.isSystemIndexAccessAllowed());
-        assertThat(threadContext.allowedSystemIndexPatterns(), empty());
-    }
-
     /**
      * Sometimes wraps a Runnable in an AbstractRunnable.
      */
diff --git a/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java b/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java
index 783d4131f83db..ae4ea8a3076d8 100644
--- a/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java
+++ b/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java
@@ -72,7 +72,7 @@ public void createNodePaths() throws IOException {
             final String nodeId = randomAlphaOfLength(10);
             try (PersistedClusterStateService.Writer writer = new PersistedClusterStateService(nodePaths, nodeId,
                 xContentRegistry(), BigArrays.NON_RECYCLING_INSTANCE,
-                new ClusterSettings(dataMasterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L, true).createWriter()) {
+                new ClusterSettings(dataMasterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L).createWriter()) {
                 writer.writeFullStateAndCommit(1L, ClusterState.EMPTY_STATE);
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/env/OverrideNodeVersionCommandTests.java b/server/src/test/java/org/elasticsearch/env/OverrideNodeVersionCommandTests.java
index 742c052a07b96..d7b8251051985 100644
--- a/server/src/test/java/org/elasticsearch/env/OverrideNodeVersionCommandTests.java
+++ b/server/src/test/java/org/elasticsearch/env/OverrideNodeVersionCommandTests.java
@@ -58,7 +58,7 @@ public void createNodePaths() throws IOException {
 
             try (PersistedClusterStateService.Writer writer = new PersistedClusterStateService(nodePaths, nodeId,
                 xContentRegistry(), BigArrays.NON_RECYCLING_INSTANCE,
-                new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L, true).createWriter()) {
+                new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L).createWriter()) {
                 writer.writeFullStateAndCommit(1L, ClusterState.builder(ClusterName.DEFAULT).metaData(MetaData.builder()
                     .persistentSettings(Settings.builder().put(MetaData.SETTING_READ_ONLY_SETTING.getKey(), true).build()).build())
                     .build());
@@ -70,7 +70,7 @@ public void createNodePaths() throws IOException {
     public void checkClusterStateIntact() throws IOException {
         assertTrue(MetaData.SETTING_READ_ONLY_SETTING.get(new PersistedClusterStateService(nodePaths, nodeId,
             xContentRegistry(), BigArrays.NON_RECYCLING_INSTANCE,
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L, true)
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L)
             .loadBestOnDiskState().metaData.persistentSettings()));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
index c2f04a8bd03b1..a7fcc99197d22 100644
--- a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
@@ -27,23 +27,12 @@
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterModule;
-import org.elasticsearch.cluster.metadata.IndexGraveyard;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -55,19 +44,12 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
 import java.util.stream.StreamSupport;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 
 @LuceneTestCase.SuppressFileSystems("ExtrasFS") // TODO: fix test to work with ExtrasFS
@@ -85,7 +67,7 @@ public void toXContent(XContentBuilder builder, MetaData state) {
 
             @Override
             public MetaData fromXContent(XContentParser parser) throws IOException {
-                return MetaData.Builder.fromXContent(parser, false);
+                return MetaData.Builder.fromXContent(parser);
             }
         };
         Path tmp = createTempDir();
@@ -238,99 +220,6 @@ public static void corruptFile(Path fileToCorrupt, Logger logger) throws IOExcep
         }
     }
 
-    public void testLoadStateWithoutMissingCustoms() throws IOException {
-        runLoadStateTest(false, false);
-    }
-
-    public void testLoadStateWithoutMissingCustomsButPreserved() throws IOException {
-        runLoadStateTest(false, true);
-    }
-
-    public void testLoadStateWithMissingCustomsButPreserved() throws IOException {
-        runLoadStateTest(true, true);
-    }
-
-    public void testLoadStateWithMissingCustomsAndNotPreserved() throws IOException {
-        runLoadStateTest(true, false);
-    }
-
-    private void runLoadStateTest(boolean hasMissingCustoms, boolean preserveUnknownCustoms) throws IOException {
-        final Path[] dirs = new Path[randomIntBetween(1, 5)];
-        int numStates = randomIntBetween(1, 5);
-        List<MetaData> meta = new ArrayList<>();
-        for (int i = 0; i < numStates; i++) {
-            meta.add(randomMeta());
-        }
-        Set<Path> corruptedFiles = new HashSet<>();
-        MetaDataStateFormat<MetaData> format = metaDataFormat(preserveUnknownCustoms);
-        for (int i = 0; i < dirs.length; i++) {
-            dirs[i] = createTempDir();
-            Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME));
-            for (int j = 0; j < numStates; j++) {
-                format.writeAndCleanup(meta.get(j), dirs[i]);
-                if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) {  // corrupt a file that we do not necessarily
-                                                                                            // need here....
-                    Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j + ".st");
-                    corruptedFiles.add(file);
-                    MetaDataStateFormatTests.corruptFile(file, logger);
-                }
-            }
-
-        }
-        List<Path> dirList = Arrays.asList(dirs);
-        Collections.shuffle(dirList, random());
-        MetaData loadedMetaData = format.loadLatestState(logger, hasMissingCustoms ?
-            xContentRegistryWithMissingCustoms() : xContentRegistry(), dirList.toArray(new Path[0]));
-        MetaData latestMetaData = meta.get(numStates-1);
-        assertThat(loadedMetaData.clusterUUID(), not(equalTo("_na_")));
-        assertThat(loadedMetaData.clusterUUID(), equalTo(latestMetaData.clusterUUID()));
-        ImmutableOpenMap<String, IndexMetaData> indices = loadedMetaData.indices();
-        assertThat(indices.size(), equalTo(latestMetaData.indices().size()));
-        for (IndexMetaData original : latestMetaData) {
-            IndexMetaData deserialized = indices.get(original.getIndex().getName());
-            assertThat(deserialized, notNullValue());
-            assertThat(deserialized.getVersion(), equalTo(original.getVersion()));
-            assertThat(deserialized.getMappingVersion(), equalTo(original.getMappingVersion()));
-            assertThat(deserialized.getSettingsVersion(), equalTo(original.getSettingsVersion()));
-            assertThat(deserialized.getNumberOfReplicas(), equalTo(original.getNumberOfReplicas()));
-            assertThat(deserialized.getNumberOfShards(), equalTo(original.getNumberOfShards()));
-        }
-
-        // make sure the index tombstones are the same too
-        if (hasMissingCustoms) {
-            if (preserveUnknownCustoms) {
-                assertNotNull(loadedMetaData.custom(IndexGraveyard.TYPE));
-                assertThat(loadedMetaData.custom(IndexGraveyard.TYPE), instanceOf(MetaData.UnknownGatewayOnlyCustom.class));
-
-                // check that we reserialize unknown metadata correctly again
-                final Path tempdir = createTempDir();
-                metaDataFormat(randomBoolean()).write(loadedMetaData, tempdir);
-                final MetaData reloadedMetaData = metaDataFormat(randomBoolean()).loadLatestState(logger, xContentRegistry(), tempdir);
-                assertThat(reloadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard()));
-            } else {
-                assertNotNull(loadedMetaData.indexGraveyard());
-                assertThat(loadedMetaData.indexGraveyard().getTombstones(), hasSize(0));
-            }
-        }  else {
-            assertThat(loadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard()));
-        }
-
-        // now corrupt all the latest ones and make sure we fail to load the state
-        for (int i = 0; i < dirs.length; i++) {
-            Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + (numStates-1) + ".st");
-            if (corruptedFiles.contains(file)) {
-                continue;
-            }
-            MetaDataStateFormatTests.corruptFile(file, logger);
-        }
-        try {
-            format.loadLatestState(logger, xContentRegistry(), dirList.toArray(new Path[0]));
-            fail("latest version can not be read");
-        } catch (ElasticsearchException ex) {
-            assertThat(ExceptionsHelper.unwrap(ex, CorruptStateException.class), notNullValue());
-        }
-    }
-
     private DummyState writeAndReadStateSuccessfully(Format format, Path... paths) throws IOException {
         format.noFailures();
         DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 100), randomInt(), randomLong(),
@@ -457,43 +346,6 @@ public void testFailRandomlyAndReadAnyState() throws IOException {
         writeAndReadStateSuccessfully(format, paths);
     }
 
-    private static MetaDataStateFormat<MetaData> metaDataFormat(boolean preserveUnknownCustoms) {
-        return new MetaDataStateFormat<MetaData>(MetaData.GLOBAL_STATE_FILE_PREFIX) {
-            @Override
-            public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
-                MetaData.Builder.toXContent(state, builder, ToXContent.EMPTY_PARAMS);
-            }
-
-            @Override
-            public MetaData fromXContent(XContentParser parser) throws IOException {
-                return MetaData.Builder.fromXContent(parser, preserveUnknownCustoms);
-            }
-        };
-    }
-
-    private MetaData randomMeta() throws IOException {
-        int numIndices = randomIntBetween(1, 10);
-        MetaData.Builder mdBuilder = MetaData.builder();
-        mdBuilder.generateClusterUuidIfNeeded();
-        for (int i = 0; i < numIndices; i++) {
-            mdBuilder.put(indexBuilder(randomAlphaOfLength(10) + "idx-"+i));
-        }
-        int numDelIndices = randomIntBetween(0, 5);
-        final IndexGraveyard.Builder graveyard = IndexGraveyard.builder();
-        for (int i = 0; i < numDelIndices; i++) {
-            graveyard.addTombstone(new Index(randomAlphaOfLength(10) + "del-idx-" + i, UUIDs.randomBase64UUID()));
-        }
-        mdBuilder.indexGraveyard(graveyard.build());
-        return mdBuilder.build();
-    }
-
-    private IndexMetaData.Builder indexBuilder(String index) throws IOException {
-        return IndexMetaData.builder(index)
-                .settings(settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10))
-                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 5)));
-    }
-
-
     private static class Format extends MetaDataStateFormat<DummyState> {
         private enum FailureMode {
             NO_FAILURES,
@@ -716,13 +568,4 @@ public long addDummyFiles(String prefix, Path... paths) throws IOException {
     protected final NamedXContentRegistry xContentRegistry() {
         return new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
     }
-
-    /**
-     * The {@link NamedXContentRegistry} to use for {@link XContentParser}s that should be
-     * missing all of the normal cluster state parsers.
-     */
-    protected NamedXContentRegistry xContentRegistryWithMissingCustoms() {
-        return new NamedXContentRegistry(Arrays.asList(
-                new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField("garbage"), RepositoriesMetaData::fromXContent)));
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
index ef4b5f6f7ec9e..19e60e9783dd9 100644
--- a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
@@ -229,7 +229,7 @@ public void testFailsOnMismatchedNodeIds() throws IOException {
         final String message = expectThrows(IllegalStateException.class,
             () -> new PersistedClusterStateService(combinedPaths, nodeIds[0], xContentRegistry(), BigArrays.NON_RECYCLING_INSTANCE,
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                () -> 0L, randomBoolean()).loadBestOnDiskState()).getMessage();
+                () -> 0L).loadBestOnDiskState()).getMessage();
         assertThat(message,
             allOf(containsString("unexpected node ID in metadata"), containsString(nodeIds[0]), containsString(nodeIds[1])));
         assertTrue("[" + message + "] should match " + Arrays.toString(dataPaths2),
diff --git a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
index 94596ffd6c58d..4253eaded3056 100644
--- a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
@@ -116,7 +116,7 @@ public void testNumeric() throws Exception {
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class,
                 () -> query.toQuery(context));
-        assertEquals("Can only use prefix queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+        assertEquals("Can only use prefix queries on keyword, text and wildcard fields - not on [mapped_int] which is of type [integer]",
                 e.getMessage());
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index 85453e774fe6e..7724e82a9e083 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -813,7 +813,7 @@ public void testPrefixNumeric() throws Exception {
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class,
                 () -> query.toQuery(context));
-        assertEquals("Can only use prefix queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+        assertEquals("Can only use prefix queries on keyword, text and wildcard fields - not on [mapped_int] which is of type [integer]",
                 e.getMessage());
         query.lenient(true);
         query.toQuery(context); // no exception
diff --git a/server/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java b/server/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java
index a76fced9772f5..770fc65621055 100644
--- a/server/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java
+++ b/server/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java
@@ -36,7 +36,6 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.closeTo;
@@ -152,7 +151,7 @@ public ScheduledCancellable schedule(Runnable command, TimeValue delay, String n
             }
         };
         try {
-            workerState.delayPrepareBulkRequest(threadPool, timeValueNanos(System.nanoTime()), batchSizeForMaxDelay,
+            workerState.delayPrepareBulkRequest(threadPool, System.nanoTime(), batchSizeForMaxDelay,
                 new AbstractRunnable() {
                     @Override
                     protected void doRun() throws Exception {
@@ -225,7 +224,7 @@ public boolean isCancelled() {
         };
         try {
             // Have the task use the thread pool to delay a task that does nothing
-            workerState.delayPrepareBulkRequest(threadPool, timeValueSeconds(0), 1, new AbstractRunnable() {
+            workerState.delayPrepareBulkRequest(threadPool, 0, 1, new AbstractRunnable() {
                 @Override
                 protected void doRun() throws Exception {
                 }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java
index c16e0262b3a3b..18c6e900a597a 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java
@@ -153,7 +153,7 @@ public void setup() throws IOException {
             final Path[] dataPaths = Arrays.stream(lock.getNodePaths()).filter(Objects::nonNull).map(p -> p.path).toArray(Path[]::new);
             try (PersistedClusterStateService.Writer writer = new PersistedClusterStateService(dataPaths, nodeId,
                 xContentRegistry(), BigArrays.NON_RECYCLING_INSTANCE,
-                new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L, true).createWriter()) {
+                new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L).createWriter()) {
                 writer.writeFullStateAndCommit(1L, clusterState);
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
index d06a7e1cecfa4..34996bca20525 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
+import org.elasticsearch.test.junit.annotations.TestIssueLogging;
 
 import java.time.ZoneId;
 import java.time.ZoneOffset;
@@ -97,7 +98,9 @@ public void testCacheAggs() throws Exception {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32827")
+    @TestIssueLogging(
+        value = "org.elasticsearch.indices.IndicesRequestCache:TRACE",
+        issueUrl = "https://github.com/elastic/elasticsearch/issues/32827")
     public void testQueryRewrite() throws Exception {
         Client client = client();
         assertAcked(client.admin().indices().prepareCreate("index").setMapping("s", "type=date")
@@ -128,7 +131,7 @@ public void testQueryRewrite() throws Exception {
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25"))
             // to ensure that query is executed even if it rewrites to match_no_docs
             .addAggregation(new GlobalAggregationBuilder("global"))
-            .setPreFilterShardSize(Integer.MAX_VALUE).get();
+            .get();
         ElasticsearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(7L));
         assertCacheState(client, "index", 0, 5);
@@ -136,7 +139,7 @@ public void testQueryRewrite() throws Exception {
         final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26"))
             .addAggregation(new GlobalAggregationBuilder("global"))
-            .setPreFilterShardSize(Integer.MAX_VALUE).get();
+            .get();
         ElasticsearchAssertions.assertAllSuccessful(r2);
         assertThat(r2.getHits().getTotalHits().value, equalTo(7L));
         assertCacheState(client, "index", 3, 7);
@@ -144,7 +147,6 @@ public void testQueryRewrite() throws Exception {
         final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27"))
             .addAggregation(new GlobalAggregationBuilder("global"))
-            .setPreFilterShardSize(Integer.MAX_VALUE)
             .get();
         ElasticsearchAssertions.assertAllSuccessful(r3);
         assertThat(r3.getHits().getTotalHits().value, equalTo(7L));
@@ -220,29 +222,35 @@ public void testQueryRewriteDates() throws Exception {
 
         assertCacheState(client, "index", 0, 0);
 
-        final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
-                .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
-                .get();
+        final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH)
+            .setSize(0)
+            .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
+            // to ensure that query is executed even if it rewrites to match_no_docs
+            .addAggregation(new GlobalAggregationBuilder("global"))
+            .get();
         ElasticsearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(9L));
         assertCacheState(client, "index", 0, 1);
 
-        final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
-                .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
-                .get();
+        final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH)
+            .setSize(0)
+            .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
+            .addAggregation(new GlobalAggregationBuilder("global"))
+            .get();
         ElasticsearchAssertions.assertAllSuccessful(r2);
         assertThat(r2.getHits().getTotalHits().value, equalTo(9L));
         assertCacheState(client, "index", 1, 1);
 
-        final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
-                .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
-                .get();
+        final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH)
+            .setSize(0)
+            .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
+            .addAggregation(new GlobalAggregationBuilder("global"))
+            .get();
         ElasticsearchAssertions.assertAllSuccessful(r3);
         assertThat(r3.getHits().getTotalHits().value, equalTo(9L));
         assertCacheState(client, "index", 2, 1);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32827")
     public void testQueryRewriteDatesWithNow() throws Exception {
         Client client = client();
         Settings settings = Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
@@ -280,8 +288,10 @@ public void testQueryRewriteDatesWithNow() throws Exception {
         assertCacheState(client, "index-2", 0, 0);
         assertCacheState(client, "index-3", 0, 0);
 
-        final SearchResponse r1 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
-                .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
+        final SearchResponse r1 = client.prepareSearch("index-*")
+            .setSearchType(SearchType.QUERY_THEN_FETCH)
+            .setSize(0)
+            .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
         ElasticsearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(8L));
         assertCacheState(client, "index-1", 0, 1);
@@ -291,16 +301,22 @@ public void testQueryRewriteDatesWithNow() throws Exception {
         // cache miss or cache hit since queries containing now can't be cached
         assertCacheState(client, "index-3", 0, 0);
 
-        final SearchResponse r2 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
-                .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
+        final SearchResponse r2 = client.prepareSearch("index-*")
+            .setSearchType(SearchType.QUERY_THEN_FETCH)
+            .setSize(0)
+            .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now"))
+            .get();
         ElasticsearchAssertions.assertAllSuccessful(r2);
         assertThat(r2.getHits().getTotalHits().value, equalTo(8L));
         assertCacheState(client, "index-1", 1, 1);
         assertCacheState(client, "index-2", 1, 1);
         assertCacheState(client, "index-3", 0, 0);
 
-        final SearchResponse r3 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
-                .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
+        final SearchResponse r3 = client.prepareSearch("index-*")
+            .setSearchType(SearchType.QUERY_THEN_FETCH)
+            .setSize(0)
+            .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now"))
+            .get();
         ElasticsearchAssertions.assertAllSuccessful(r3);
         assertThat(r3.getHits().getTotalHits().value, equalTo(8L));
         assertCacheState(client, "index-1", 2, 1);
diff --git a/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java b/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
index ce424e61787ee..e83d9b14a3845 100644
--- a/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
@@ -117,7 +118,7 @@ public void testCloseWhileRelocatingShards() throws Exception {
             indices[i] = indexName;
         }
 
-        ensureGreen(indices);
+        ensureGreen(TimeValue.timeValueSeconds(60L),indices);
         assertAcked(client().admin().cluster().prepareUpdateSettings()
             .setTransientSettings(Settings.builder()
                 .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString())));
diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
index 8778cbdc8689f..083872ce83768 100644
--- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
@@ -201,8 +201,8 @@ public boolean isReadOnly() {
 
         @Override
         public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                                  IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                                  Map<String, Object> userMetadata, ActionListener<String> listener) {
+                                  IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                                  Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
 
         }
 
diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
index 49ca709118074..22b704389aa26 100644
--- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
@@ -105,8 +105,8 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException {
             final PlainActionFuture<String> future1 = PlainActionFuture.newFuture();
             runGeneric(threadPool, () -> {
                 IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(null);
-                repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, snapshotStatus, Version.CURRENT,
-                    Collections.emptyMap(), future1);
+                repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, null,
+                    snapshotStatus, Version.CURRENT, Collections.emptyMap(), future1);
                 future1.actionGet();
                 IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy();
                 assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount());
@@ -134,8 +134,8 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException {
             final PlainActionFuture<String> future2 = PlainActionFuture.newFuture();
             runGeneric(threadPool, () -> {
                 IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration);
-                repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, snapshotStatus, Version.CURRENT,
-                    Collections.emptyMap(), future2);
+                repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit,
+                    null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future2);
                 future2.actionGet();
                 IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy();
                 assertEquals(2, copy.getIncrementalFileCount());
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index dfcfd644dde34..415fe1bf713fa 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -91,7 +91,7 @@ public void setup() {
         inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
 
         HttpServerTransport httpServerTransport = new TestHttpServerTransport();
-        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService, randomBoolean());
+        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/",
             (request, channel, client) -> channel.sendResponse(
                 new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
@@ -106,7 +106,7 @@ public void testApplyRelevantHeaders() throws Exception {
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         Set<RestHeaderDefinition> headers = new HashSet<>(Arrays.asList(new RestHeaderDefinition("header.1", true),
             new RestHeaderDefinition("header.2", true)));
-        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, randomBoolean());
+        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("true"));
         restHeaders.put("header.2", Collections.singletonList("true"));
@@ -142,7 +142,7 @@ public void testRequestWithDisallowedMultiValuedHeader() {
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         Set<RestHeaderDefinition> headers = new HashSet<>(Arrays.asList(new RestHeaderDefinition("header.1", true),
             new RestHeaderDefinition("header.2", false)));
-        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, randomBoolean());
+        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("boo"));
         restHeaders.put("header.2", List.of("foo", "bar"));
@@ -156,7 +156,7 @@ public void testRequestWithDisallowedMultiValuedHeaderButSameValues() {
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         Set<RestHeaderDefinition> headers = new HashSet<>(Arrays.asList(new RestHeaderDefinition("header.1", true),
             new RestHeaderDefinition("header.2", false)));
-        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, randomBoolean());
+        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("boo"));
         restHeaders.put("header.2", List.of("foo", "foo"));
@@ -210,7 +210,7 @@ public void testRegisterWithDeprecatedHandler() {
     }
 
     public void testRegisterSecondMethodWithDifferentNamedWildcard() {
-        final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, randomBoolean());
+        final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService);
 
         RestRequest.Method firstMethod = randomFrom(RestRequest.Method.values());
         RestRequest.Method secondMethod =
@@ -237,7 +237,7 @@ public void testRestHandlerWrapper() throws Exception {
                 h -> {
                     assertSame(handler, h);
                     return (RestRequest request, RestChannel channel, NodeClient client) -> wrapperCalled.set(true);
-                }, null, circuitBreakerService, usageService, randomBoolean());
+                }, null, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/wrapped", handler);
         RestRequest request = testRestRequest("/wrapped", "{}", XContentType.JSON);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST);
@@ -300,7 +300,7 @@ public void testDispatchRequiresContentTypeForRequestsWithContent() {
         String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead()));
         RestRequest request = testRestRequest("/", content, null);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE);
-        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService, randomBoolean());
+        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/",
             (r, c, client) -> c.sendResponse(
                 new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
@@ -603,25 +603,6 @@ public HttpRequest releaseAndCopy() {
         assertThat(channel.getRestResponse().getHeaders().get("Allow"), hasItem(equalTo(RestRequest.Method.GET.toString())));
     }
 
-    public void testDispatchRestrictSystemIndices() {
-        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService, true);
-        restController.registerHandler(RestRequest.Method.GET, "/",
-            (request, channel, client) -> channel.sendResponse(
-                new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
-        restController.registerHandler(RestRequest.Method.GET, "/error", (request, channel, client) -> {
-            throw new IllegalArgumentException("test error");
-        });
-
-        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build();
-        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
-
-        assertFalse(channel.getSendResponseCalled());
-        ThreadContext context = new ThreadContext(Settings.EMPTY);
-        assertTrue(context.isSystemIndexAccessAllowed());
-        restController.dispatchRequest(fakeRestRequest, channel, context);
-        assertTrue(channel.getSendResponseCalled());
-        assertFalse(context.isSystemIndexAccessAllowed());
-    }
 
     private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
         HttpServerTransport {
diff --git a/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java b/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java
index 5b4183e72f3e2..6a4a8749397ab 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java
@@ -89,7 +89,7 @@ public void testUnsupportedMethodResponseHttpHeader() throws Exception {
         final Settings settings = Settings.EMPTY;
         UsageService usageService = new UsageService();
         RestController restController = new RestController(Collections.emptySet(),
-                null, null, circuitBreakerService, usageService, randomBoolean());
+                null, null, circuitBreakerService, usageService);
 
         // A basic RestHandler handles requests to the endpoint
         RestHandler restHandler = new RestHandler() {
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoActionTests.java
index d757ee095cdc1..456b371b0eb50 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoActionTests.java
@@ -20,71 +20,77 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.rest.action.admin.cluster.RestNodesInfoAction.ALLOWED_METRICS;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.not;
 
 public class RestNodesInfoActionTests extends ESTestCase {
 
     public void testDuplicatedFiltersAreNotRemoved() {
         Map<String, String> params = new HashMap<>();
         params.put("nodeId", "_all,master:false,_all");
-        
+
         RestRequest restRequest = buildRestRequest(params);
         NodesInfoRequest actual = RestNodesInfoAction.prepareRequest(restRequest);
         assertArrayEquals(new String[] { "_all", "master:false", "_all" }, actual.nodesIds());
     }
-    
+
     public void testOnlyMetrics() {
         Map<String, String> params = new HashMap<>();
         int metricsCount = randomIntBetween(1, ALLOWED_METRICS.size());
         List<String> metrics = new ArrayList<>();
-        
+
         for(int i = 0; i < metricsCount; i++) {
             metrics.add(randomFrom(ALLOWED_METRICS));
         }
         params.put("nodeId", String.join(",", metrics));
-        
+
         RestRequest restRequest = buildRestRequest(params);
         NodesInfoRequest actual = RestNodesInfoAction.prepareRequest(restRequest);
         assertArrayEquals(new String[] { "_all" }, actual.nodesIds());
         assertMetrics(metrics, actual);
     }
-    
+
     public void testAllMetricsSelectedWhenNodeAndMetricSpecified() {
         Map<String, String> params = new HashMap<>();
         String nodeId = randomValueOtherThanMany(ALLOWED_METRICS::contains, () -> randomAlphaOfLength(23));
         String metric = randomFrom(ALLOWED_METRICS);
-        
+
         params.put("nodeId", nodeId + "," + metric);
         RestRequest restRequest = buildRestRequest(params);
-        
+
         NodesInfoRequest actual = RestNodesInfoAction.prepareRequest(restRequest);
         assertArrayEquals(new String[] { nodeId, metric }, actual.nodesIds());
         assertAllMetricsTrue(actual);
     }
-    
+
     public void testSeparateNodeIdsAndMetrics() {
         Map<String, String> params = new HashMap<>();
         List<String> nodeIds = new ArrayList<>(5);
         List<String> metrics = new ArrayList<>(5);
-        
+
         for(int i = 0; i < 5; i++) {
             nodeIds.add(randomValueOtherThanMany(ALLOWED_METRICS::contains, () -> randomAlphaOfLength(23)));
             metrics.add(randomFrom(ALLOWED_METRICS));
         }
-        
+
         params.put("nodeId", String.join(",", nodeIds));
         params.put("metrics", String.join(",", metrics));
         RestRequest restRequest = buildRestRequest(params);
-        
+
         NodesInfoRequest actual = RestNodesInfoAction.prepareRequest(restRequest);
         assertArrayEquals(nodeIds.toArray(), actual.nodesIds());
         assertMetrics(metrics, actual);
@@ -97,16 +103,42 @@ public void testExplicitAllMetrics() {
         for(int i = 0; i < 5; i++) {
             nodeIds.add(randomValueOtherThanMany(ALLOWED_METRICS::contains, () -> randomAlphaOfLength(23)));
         }
-        
+
         params.put("nodeId", String.join(",", nodeIds));
         params.put("metrics", "_all");
         RestRequest restRequest = buildRestRequest(params);
-        
+
         NodesInfoRequest actual = RestNodesInfoAction.prepareRequest(restRequest);
         assertArrayEquals(nodeIds.toArray(), actual.nodesIds());
         assertAllMetricsTrue(actual);
     }
 
+    /**
+     * Test that if a user requests a non-existent metric, it's dropped from the
+     * request without an error.
+     */
+    public void testNonexistentMetricsDropped() {
+        Map<String, String> params = new HashMap<>();
+        List<String> nodeIds = new ArrayList<>(5);
+        List<String> metrics = new ArrayList<>(5);
+
+        for(int i = 0; i < 5; i++) {
+            nodeIds.add(randomValueOtherThanMany(ALLOWED_METRICS::contains, () -> randomAlphaOfLength(23)));
+            metrics.add(randomFrom(ALLOWED_METRICS));
+        }
+        String nonAllowedMetric = randomValueOtherThanMany(ALLOWED_METRICS::contains, () -> randomAlphaOfLength(5));
+        metrics.add(nonAllowedMetric);
+
+        params.put("nodeId", String.join(",", nodeIds));
+        params.put("metrics", String.join(",", metrics));
+        RestRequest restRequest = buildRestRequest(params);
+
+        NodesInfoRequest actual = RestNodesInfoAction.prepareRequest(restRequest);
+        assertArrayEquals(nodeIds.toArray(), actual.nodesIds());
+        assertThat(actual.requestedMetrics(), not(hasItem(nonAllowedMetric)));
+        assertMetrics(metrics, actual);
+    }
+
     private FakeRestRequest buildRestRequest(Map<String, String> params) {
         return new FakeRestRequest.Builder(xContentRegistry())
                 .withMethod(RestRequest.Method.GET)
@@ -114,30 +146,13 @@ private FakeRestRequest buildRestRequest(Map<String, String> params) {
                 .withParams(params)
                 .build();
     }
-    
+
     private void assertMetrics(List<String> metrics, NodesInfoRequest nodesInfoRequest) {
-        assertTrue((metrics.contains("http") && nodesInfoRequest.http()) || metrics.contains("http") == false);
-        assertTrue((metrics.contains("ingest") && nodesInfoRequest.ingest()) || metrics.contains("ingest") == false);
-        assertTrue((metrics.contains("indices") && nodesInfoRequest.indices()) || metrics.contains("indices") == false);
-        assertTrue((metrics.contains("jvm") && nodesInfoRequest.jvm()) || metrics.contains("jvm") == false);
-        assertTrue((metrics.contains("os") && nodesInfoRequest.os()) || metrics.contains("os") == false);
-        assertTrue((metrics.contains("plugins") && nodesInfoRequest.plugins()) || metrics.contains("plugins") == false);
-        assertTrue((metrics.contains("process") && nodesInfoRequest.process()) || metrics.contains("process") == false);
-        assertTrue((metrics.contains("settings") && nodesInfoRequest.settings()) || metrics.contains("settings") == false);
-        assertTrue((metrics.contains("thread_pool") && nodesInfoRequest.threadPool()) || metrics.contains("thread_pool") == false);
-        assertTrue((metrics.contains("transport") && nodesInfoRequest.transport()) || metrics.contains("transport") == false);
+        Set<String> validRequestedMetrics = Sets.intersection(new HashSet<>(metrics), ALLOWED_METRICS);
+        assertThat(nodesInfoRequest.requestedMetrics(), equalTo(validRequestedMetrics));
     }
-    
+
     private void assertAllMetricsTrue(NodesInfoRequest nodesInfoRequest) {
-        assertTrue(nodesInfoRequest.http());
-        assertTrue(nodesInfoRequest.ingest());
-        assertTrue(nodesInfoRequest.indices());
-        assertTrue(nodesInfoRequest.jvm());
-        assertTrue(nodesInfoRequest.os());
-        assertTrue(nodesInfoRequest.plugins());
-        assertTrue(nodesInfoRequest.process());
-        assertTrue(nodesInfoRequest.settings());
-        assertTrue(nodesInfoRequest.threadPool());
-        assertTrue(nodesInfoRequest.transport());
+        assertThat(nodesInfoRequest.requestedMetrics(), equalTo(ALLOWED_METRICS));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
index f7c060a48f527..4813e11e15bfc 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
@@ -58,7 +58,7 @@ public class RestValidateQueryActionTests extends AbstractSearchTestCase {
 
     private static UsageService usageService = new UsageService();
     private static RestController controller = new RestController(emptySet(), null, client,
-        new NoneCircuitBreakerService(), usageService, false);
+        new NoneCircuitBreakerService(), usageService);
     private static RestValidateQueryAction action = new RestValidateQueryAction();
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java
index b62b9d7d0d3fd..258844f36b71c 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java
@@ -56,7 +56,7 @@ public class RestCatRecoveryActionTests extends ESTestCase {
     public void testRestRecoveryAction() {
         final Settings settings = Settings.EMPTY;
         UsageService usageService = new UsageService();
-        final RestController restController = new RestController(Collections.emptySet(), null, null, null, usageService, randomBoolean());
+        final RestController restController = new RestController(Collections.emptySet(), null, null, null, usageService);
         final RestCatRecoveryAction action = new RestCatRecoveryAction();
         restController.registerHandler(action);
         final int totalShards = randomIntBetween(1, 32);
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
index 060c2c545875f..aad58e50d69c9 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
@@ -127,8 +127,7 @@ public void testBuildTable() {
             }
         }
 
-        final RestController restController =
-            new RestController(Collections.emptySet(), null, null, null, new UsageService(), randomBoolean());
+        final RestController restController = new RestController(Collections.emptySet(), null, null, null, new UsageService());
         final RestIndicesAction action = new RestIndicesAction();
         restController.registerHandler(action);
         final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetaDatas);
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java
new file mode 100644
index 0000000000000..f661521376cf5
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.cat;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.Table;
+import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class RestShardsActionTests extends ESTestCase {
+
+    public void testBuildTable() {
+        final int numShards = randomIntBetween(1, 5);
+        DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT);
+
+        List<ShardRouting> shardRoutings = new ArrayList<>(numShards);
+        Map<ShardRouting, ShardStats> shardStatsMap = new HashMap<>();
+        String index = "index";
+        for (int i = 0; i < numShards; i++) {
+            ShardRoutingState shardRoutingState = ShardRoutingState.fromValue((byte) randomIntBetween(2, 3));
+            ShardRouting shardRouting = TestShardRouting.newShardRouting(index, i, localNode.getId(), randomBoolean(), shardRoutingState);
+            Path path = createTempDir().resolve("indices").resolve(shardRouting.shardId().getIndex().getUUID())
+                .resolve(String.valueOf(shardRouting.shardId().id()));
+            ShardStats shardStats = new ShardStats(shardRouting, new ShardPath(false, path, path, shardRouting.shardId()),
+                null, null, null, null);
+            shardStatsMap.put(shardRouting, shardStats);
+            shardRoutings.add(shardRouting);
+        }
+
+        IndexStats indexStats = mock(IndexStats.class);
+        when(indexStats.getPrimaries()).thenReturn(new CommonStats());
+        when(indexStats.getTotal()).thenReturn(new CommonStats());
+
+        IndicesStatsResponse stats = mock(IndicesStatsResponse.class);
+        when(stats.asMap()).thenReturn(shardStatsMap);
+
+        DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class);
+        when(discoveryNodes.get(localNode.getId())).thenReturn(localNode);
+
+        ClusterStateResponse state = mock(ClusterStateResponse.class);
+        RoutingTable routingTable = mock(RoutingTable.class);
+        when(routingTable.allShards()).thenReturn(shardRoutings);
+        ClusterState clusterState = mock(ClusterState.class);
+        when(clusterState.routingTable()).thenReturn(routingTable);
+        when(clusterState.nodes()).thenReturn(discoveryNodes);
+        when(state.getState()).thenReturn(clusterState);
+
+        final RestShardsAction action = new RestShardsAction();
+        final Table table = action.buildTable(new FakeRestRequest(), state, stats);
+
+        // now, verify the table is correct
+        List<Table.Cell> headers = table.getHeaders();
+        assertThat(headers.get(0).value, equalTo("index"));
+        assertThat(headers.get(1).value, equalTo("shard"));
+        assertThat(headers.get(2).value, equalTo("prirep"));
+        assertThat(headers.get(3).value, equalTo("state"));
+        assertThat(headers.get(4).value, equalTo("docs"));
+        assertThat(headers.get(5).value, equalTo("store"));
+        assertThat(headers.get(6).value, equalTo("ip"));
+        assertThat(headers.get(7).value, equalTo("id"));
+        assertThat(headers.get(8).value, equalTo("node"));
+
+        final List<List<Table.Cell>> rows = table.getRows();
+        assertThat(rows.size(), equalTo(numShards));
+
+        Iterator<ShardRouting> shardRoutingsIt = shardRoutings.iterator();
+        for (final List<Table.Cell> row : rows) {
+            ShardRouting shardRouting = shardRoutingsIt.next();
+            ShardStats shardStats = shardStatsMap.get(shardRouting);
+            assertThat(row.get(0).value, equalTo(shardRouting.getIndexName()));
+            assertThat(row.get(1).value, equalTo(shardRouting.getId()));
+            assertThat(row.get(2).value, equalTo(shardRouting.primary() ? "p" : "r"));
+            assertThat(row.get(3).value, equalTo(shardRouting.state()));
+            assertThat(row.get(6).value, equalTo(localNode.getHostAddress()));
+            assertThat(row.get(7).value, equalTo(localNode.getId()));
+            assertThat(row.get(69).value, equalTo(shardStats.getDataPath()));
+            assertThat(row.get(70).value, equalTo(shardStats.getStatePath()));
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
index bb549b64c724f..84bc5f3bef340 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
@@ -42,7 +43,9 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class RestIndexActionTests extends RestActionTestCase {
 
@@ -50,9 +53,11 @@ public class RestIndexActionTests extends RestActionTestCase {
 
     @Before
     public void setUpAction() {
+        ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.state()).thenAnswer(invocationOnMock -> clusterStateSupplier.get());
         controller().registerHandler(new RestIndexAction());
         controller().registerHandler(new CreateHandler());
-        controller().registerHandler(new AutoIdHandler(() -> clusterStateSupplier.get().nodes()));
+        controller().registerHandler(new AutoIdHandler(clusterService));
     }
 
     public void testCreateOpTypeValidation() {
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index c6e41d805657a..8945938c39a55 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -57,6 +57,8 @@
 import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.UUID;
 
@@ -76,9 +78,11 @@ public void testPreProcess() throws Exception {
         ShardId shardId = new ShardId("index", UUID.randomUUID().toString(), 1);
         when(shardSearchRequest.shardId()).thenReturn(shardId);
 
+        ThreadPool threadPool = new TestThreadPool(this.getClass().getName());
         IndexShard indexShard = mock(IndexShard.class);
         QueryCachingPolicy queryCachingPolicy = mock(QueryCachingPolicy.class);
         when(indexShard.getQueryCachingPolicy()).thenReturn(queryCachingPolicy);
+        when(indexShard.getThreadPool()).thenReturn(threadPool);
 
         int maxResultWindow = randomIntBetween(50, 100);
         int maxRescoreWindow = randomIntBetween(50, 100);
@@ -120,9 +124,10 @@ public void testPreProcess() throws Exception {
 
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
-            ReaderContext readerWithoutScroll = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher);
+            ReaderContext readerWithoutScroll = new ReaderContext(
+                randomNonNegativeLong(), indexShard, engineSearcher, randomNonNegativeLong(), false);
             DefaultSearchContext contextWithoutScroll = new DefaultSearchContext(readerWithoutScroll, shardSearchRequest, target, null,
-                indexService, indexShard, bigArrays, null, timeout, null);
+                indexService, indexShard, bigArrays, null, timeout, null, false);
             contextWithoutScroll.from(300);
             contextWithoutScroll.close();
 
@@ -135,9 +140,10 @@ public void testPreProcess() throws Exception {
 
             // resultWindow greater than maxResultWindow and scrollContext isn't null
             when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMillis(randomInt(1000))));
-            ReaderContext readerContext = new LegacyReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest);
+            ReaderContext readerContext = new LegacyReaderContext(
+                randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest, randomNonNegativeLong());
             DefaultSearchContext context1 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null, indexService,
-                indexShard, bigArrays, null, timeout, null);
+                indexShard, bigArrays, null, timeout, null, false);
             context1.from(300);
             exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Batch size is too large, size must be less than or equal to: ["
@@ -167,10 +173,10 @@ public void testPreProcess() throws Exception {
                 + "] index level setting."));
 
             readerContext.close();
-            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher);
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, randomNonNegativeLong(), false);
             // rescore is null but sliceBuilder is not null
             DefaultSearchContext context2 = new DefaultSearchContext(readerContext, shardSearchRequest, target,
-                null, indexService, indexShard, bigArrays, null, timeout, null);
+                null, indexService, indexShard, bigArrays, null, timeout, null, false);
 
             SliceBuilder sliceBuilder = mock(SliceBuilder.class);
             int numSlices = maxSlicesPerScroll + randomIntBetween(1, 100);
@@ -187,7 +193,7 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.indexBoost()).thenReturn(AbstractQueryBuilder.DEFAULT_BOOST);
 
             DefaultSearchContext context3 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
-                indexService, indexShard, bigArrays, null, timeout, null);
+                indexService, indexShard, bigArrays, null, timeout, null, false);
             ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery();
             context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false);
             assertEquals(context3.query(), context3.buildFilteredQuery(parsedQuery.query()));
@@ -197,9 +203,9 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
             readerContext.close();
-            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher);
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, randomNonNegativeLong(), false);
             DefaultSearchContext context4 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
-                indexService, indexShard, bigArrays, null, timeout, null);
+                indexService, indexShard, bigArrays, null, timeout, null, false);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
             Query query1 = context4.query();
             context4.sliceBuilder(new SliceBuilder(0,2)).parsedQuery(parsedQuery).preProcess(false);
@@ -207,6 +213,7 @@ public void testPreProcess() throws Exception {
             assertTrue(query1 instanceof MatchNoDocsQuery || query2 instanceof MatchNoDocsQuery);
 
             readerContext.close();
+            threadPool.shutdown();
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
index 3aafdcc575230..1ea831eb7750d 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
@@ -86,8 +86,8 @@ public static void cleanup() throws IOException {
     }
 
     public void testAddingCancellationActions() throws IOException {
-        ContextIndexSearcher searcher = new ContextIndexSearcher(reader,
-                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+        ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
+                IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
         NullPointerException npe = expectThrows(NullPointerException.class, () -> searcher.addQueryCancellation(null));
         assertEquals("cancellation runnable should not be null", npe.getMessage());
 
@@ -100,8 +100,8 @@ public void testAddingCancellationActions() throws IOException {
     public void testCancellableCollector() throws IOException {
         TotalHitCountCollector collector1 = new TotalHitCountCollector();
         Runnable cancellation = () -> { throw new TaskCancelledException("cancelled"); };
-        ContextIndexSearcher searcher = new ContextIndexSearcher(reader,
-            IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+        ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
+                IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
 
         searcher.search(new MatchAllDocsQuery(), collector1);
         assertThat(collector1.getTotalHits(), equalTo(reader.numDocs()));
@@ -116,14 +116,14 @@ public void testCancellableCollector() throws IOException {
         assertThat(collector2.getTotalHits(), equalTo(reader.numDocs()));
     }
 
-    public void testCancellableDirectoryReader() throws IOException {
+    public void testExitableDirectoryReader() throws IOException {
         AtomicBoolean cancelled = new AtomicBoolean(true);
         Runnable cancellation = () -> {
             if (cancelled.get()) {
                 throw new TaskCancelledException("cancelled");
         }};
-        ContextIndexSearcher searcher = new ContextIndexSearcher(reader,
-                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+        ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
+                IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
         searcher.addQueryCancellation(cancellation);
         CompiledAutomaton automaton = new CompiledAutomaton(new RegExp("a.*").toAutomaton());
 
diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index ff26f49a92686..48daf474d0ded 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -436,6 +436,9 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param
         private static TestPipelineAggregationBuilder fromXContent(String name, XContentParser p) {
             return null;
         }
+
+        @Override
+        protected void validate(ValidationContext context) {}
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index a21a57e76b9f6..7957e6e9ab27d 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
@@ -89,7 +90,6 @@
 import org.junit.Before;
 
 import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -462,10 +462,10 @@ public void testMaxScriptFieldsSearch() throws IOException {
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
                 () -> service.createContext(reader, request, null, randomBoolean()));
             assertEquals(
-                "Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxScriptFields + "] but was ["
-                    + (maxScriptFields + 1)
-                    + "]. This limit can be set by changing the [index.max_script_fields] index level setting.",
-                ex.getMessage());
+                    "Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxScriptFields + "] but was ["
+                            + (maxScriptFields + 1)
+                            + "]. This limit can be set by changing the [index.max_script_fields] index level setting.",
+                    ex.getMessage());
         }
     }
 
@@ -520,10 +520,9 @@ public void testMaxOpenScrollContexts() throws Exception {
             client().prepareSearch("index").setSize(1).setScroll("1m").get();
         }
 
-        SearchService.SearchRewriteContext rewriteContext =
-            service.acquireSearcherAndRewrite(new ShardScrollRequestTest(indexShard.shardId()), indexShard);
+        final ShardScrollRequestTest request = new ShardScrollRequestTest(indexShard.shardId());
         ElasticsearchException ex = expectThrows(ElasticsearchException.class,
-            () -> service.createAndPutReaderContext(rewriteContext, randomBoolean()));
+            () -> service.createAndPutReaderContext(request, indexShard, indexShard.acquireSearcher("test"), randomBoolean()));
         assertEquals(
             "Trying to create too many scroll contexts. Must be less than or equal to: [" +
                 SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY) + "]. " +
@@ -546,16 +545,19 @@ public void testOpenScrollContextsConcurrently() throws Exception {
                 try {
                     latch.await();
                     for (; ; ) {
-                        SearchService.SearchRewriteContext rewriteContext =
-                            searchService.acquireSearcherAndRewrite(new ShardScrollRequestTest(indexShard.shardId()), indexShard);
+                        Engine.Searcher searcher = indexShard.acquireSearcher("test");
                         try {
-                            searchService.createAndPutReaderContext(rewriteContext, false);
+                            searchService.createAndPutReaderContext(
+                                new ShardScrollRequestTest(indexShard.shardId()), indexShard, searcher, true);
+                            searcher = null;
                         } catch (ElasticsearchException e) {
                             assertThat(e.getMessage(), equalTo(
                                 "Trying to create too many scroll contexts. Must be less than or equal to: " +
                                     "[" + maxScrollContexts + "]. " +
                                     "This limit can be set by changing the [search.max_open_scroll_context] setting."));
                             return;
+                        } finally {
+                            IOUtils.close(searcher);
                         }
                     }
                 } catch (Exception e) {
@@ -649,7 +651,6 @@ public void testCanMatch() throws Exception {
         final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
         final IndexShard indexShard = indexService.getShard(0);
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
-        int numWrapReader = numWrapInvocations.get();
         assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
             new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch());
 
@@ -673,7 +674,7 @@ public void testCanMatch() throws Exception {
         searchRequest.source(new SearchSourceBuilder().query(new MatchNoneQueryBuilder()));
         assertFalse(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
             new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch());
-        assertEquals(numWrapReader, numWrapInvocations.get());
+        assertEquals(0, numWrapInvocations.get());
 
         ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
             new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
@@ -685,7 +686,7 @@ public void testCanMatch() throws Exception {
             public void onResponse(SearchPhaseResult searchPhaseResult) {
                 try {
                     // make sure that the wrapper is called when the query is actually executed
-                    assertEquals(numWrapReader + 1, numWrapInvocations.get());
+                    assertEquals(1, numWrapInvocations.get());
                 } finally {
                     latch.countDown();
                 }
@@ -981,9 +982,8 @@ public void testLookUpSearchContext() throws Exception {
                     ShardSearchRequest request = new ShardSearchRequest(
                         OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
                         indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
-                    SearchService.SearchRewriteContext rewriteContext = null;
-                    rewriteContext = searchService.acquireSearcherAndRewrite(request, indexShard);
-                    final ReaderContext reader = searchService.createAndPutReaderContext(rewriteContext, randomBoolean());
+                    final Engine.Searcher searcher = indexShard.acquireSearcher("test");
+                    final ReaderContext reader = searchService.createAndPutReaderContext(request, indexShard, searcher, randomBoolean());
                     assertThat(reader.id().getId(), equalTo((long) (i + 1)));
                     contextIds.add(reader.id());
                 }
@@ -1003,8 +1003,6 @@ OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
                     assertFalse(searchService.freeReaderContext(contextId));
                     assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
                 }
-            } catch (IOException e) {
-                throw new UncheckedIOException(e);
             } finally {
                 latch.countDown();
             }
@@ -1014,6 +1012,6 @@ OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
 
     private ReaderContext createReaderContext(IndexShard shard) {
         Engine.Searcher searcher = shard.acquireSearcher("test");
-        return new ReaderContext(randomNonNegativeLong(), shard, searcher);
+        return new ReaderContext(randomNonNegativeLong(), shard, searcher, randomNonNegativeLong(), false);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java
index 0fe02c5e89fd9..e1dacc98789bd 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests;
@@ -33,7 +32,7 @@
 import org.elasticsearch.search.aggregations.pipeline.AvgBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValueTests;
 import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.SumBucketPipelineAggregationBuilder;
 import org.elasticsearch.test.ESTestCase;
@@ -45,9 +44,9 @@
 import java.util.Collections;
 import java.util.List;
 
-import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.equalTo;
 
 public class InternalAggregationsTests extends ESTestCase {
 
@@ -64,13 +63,8 @@ public void testReduceEmptyAggs() {
     public void testNonFinalReduceTopLevelPipelineAggs()  {
         InternalAggregation terms = new StringTerms("name", BucketOrder.key(true),
             10, 1, Collections.emptyList(), Collections.emptyMap(), DocValueFormat.RAW, 25, false, 10, Collections.emptyList(), 0);
-        List<SiblingPipelineAggregator> topLevelPipelineAggs = new ArrayList<>();
-        MaxBucketPipelineAggregationBuilder maxBucketPipelineAggregationBuilder = new MaxBucketPipelineAggregationBuilder("test", "test");
-        topLevelPipelineAggs.add((SiblingPipelineAggregator)maxBucketPipelineAggregationBuilder.create());
-        List<InternalAggregations> aggs = Collections.singletonList(new InternalAggregations(Collections.singletonList(terms),
-            topLevelPipelineAggs));
-        InternalAggregation.ReduceContext reduceContext = InternalAggregationTestCase.emptyReduceContextBuilder().forPartialReduction();
-        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(aggs, reduceContext);
+        List<InternalAggregations> aggs = singletonList(new InternalAggregations(Collections.singletonList(terms)));
+        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(aggs, maxBucketReduceContext().forPartialReduction());
         assertEquals(1, reducedAggs.getTopLevelPipelineAggregators().size());
         assertEquals(1, reducedAggs.aggregations.size());
     }
@@ -79,16 +73,20 @@ public void testFinalReduceTopLevelPipelineAggs()  {
         InternalAggregation terms = new StringTerms("name", BucketOrder.key(true),
             10, 1, Collections.emptyList(), Collections.emptyMap(), DocValueFormat.RAW, 25, false, 10, Collections.emptyList(), 0);
 
-        MaxBucketPipelineAggregationBuilder maxBucketPipelineAggregationBuilder = new MaxBucketPipelineAggregationBuilder("test", "test");
-        SiblingPipelineAggregator siblingPipelineAggregator = (SiblingPipelineAggregator) maxBucketPipelineAggregationBuilder.create();
-        InternalAggregation.ReduceContext reduceContext = InternalAggregation.ReduceContext.forFinalReduction(
-                BigArrays.NON_RECYCLING_INSTANCE, null, b -> {}, new PipelineTree(emptyMap(), singletonList(siblingPipelineAggregator)));
-        InternalAggregations aggs = new InternalAggregations(Collections.singletonList(terms), emptyList());
-        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(Collections.singletonList(aggs), reduceContext);
+        InternalAggregations aggs = new InternalAggregations(Collections.singletonList(terms));
+        InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(Collections.singletonList(aggs),
+                maxBucketReduceContext().forFinalReduction());
         assertEquals(0, reducedAggs.getTopLevelPipelineAggregators().size());
         assertEquals(2, reducedAggs.aggregations.size());
     }
 
+    private InternalAggregation.ReduceContextBuilder maxBucketReduceContext() {
+        MaxBucketPipelineAggregationBuilder maxBucketPipelineAggregationBuilder = new MaxBucketPipelineAggregationBuilder("test", "test");
+        PipelineAggregator.PipelineTree tree =
+                new PipelineAggregator.PipelineTree(emptyMap(), singletonList(maxBucketPipelineAggregationBuilder.create()));
+        return InternalAggregationTestCase.emptyReduceContextBuilder(tree);
+    }
+
     public static InternalAggregations createTestInstance() throws Exception {
         List<InternalAggregation> aggsList = new ArrayList<>();
         if (randomBoolean()) {
@@ -106,7 +104,11 @@ public static InternalAggregations createTestInstance() throws Exception {
             InternalSimpleValueTests simpleValueTests = new InternalSimpleValueTests();
             aggsList.add(simpleValueTests.createTestInstance());
         }
-        List<SiblingPipelineAggregator> topLevelPipelineAggs = new ArrayList<>();
+        return new InternalAggregations(aggsList);
+    }
+
+    private static PipelineAggregator.PipelineTree randomPipelineTree() {
+        List<PipelineAggregator> topLevelPipelineAggs = new ArrayList<>();
         if (randomBoolean()) {
             if (randomBoolean()) {
                 topLevelPipelineAggs.add((SiblingPipelineAggregator)new MaxBucketPipelineAggregationBuilder("name1", "bucket1").create());
@@ -118,7 +120,7 @@ public static InternalAggregations createTestInstance() throws Exception {
                 topLevelPipelineAggs.add((SiblingPipelineAggregator)new SumBucketPipelineAggregationBuilder("name3", "bucket3").create());
             }
         }
-        return new InternalAggregations(aggsList, topLevelPipelineAggs);
+        return new PipelineAggregator.PipelineTree(emptyMap(), topLevelPipelineAggs);
     }
 
     public void testSerialization() throws Exception {
@@ -126,6 +128,14 @@ public void testSerialization() throws Exception {
         writeToAndReadFrom(aggregations, 0);
     }
 
+    public void testGetTopLevelPipelineAggregators() throws Exception {
+        InternalAggregations orig = createTestInstance();
+        PipelineAggregator.PipelineTree tree = randomPipelineTree();
+        InternalAggregations withPipelines = new InternalAggregations(orig.copyResults(), () -> tree);
+        assertThat(withPipelines.aggregations, equalTo(orig.aggregations));
+        assertThat(withPipelines.getTopLevelPipelineAggregators(), equalTo(tree.aggregators()));
+    }
+
     private void writeToAndReadFrom(InternalAggregations aggregations, int iteration) throws IOException {
         Version version = VersionUtils.randomVersion(random());
         try (BytesStreamOutput out = new BytesStreamOutput()) {
@@ -135,18 +145,6 @@ private void writeToAndReadFrom(InternalAggregations aggregations, int iteration
                 in.setVersion(version);
                 InternalAggregations deserialized = new InternalAggregations(in);
                 assertEquals(aggregations.aggregations, deserialized.aggregations);
-                if (aggregations.getTopLevelPipelineAggregators() == null) {
-                    assertEquals(0, deserialized.getTopLevelPipelineAggregators().size());
-                } else {
-                    assertEquals(aggregations.getTopLevelPipelineAggregators().size(),
-                        deserialized.getTopLevelPipelineAggregators().size());
-                    for (int i = 0; i < aggregations.getTopLevelPipelineAggregators().size(); i++) {
-                        SiblingPipelineAggregator siblingPipelineAggregator1 = aggregations.getTopLevelPipelineAggregators().get(i);
-                        SiblingPipelineAggregator siblingPipelineAggregator2 = deserialized.getTopLevelPipelineAggregators().get(i);
-                        assertArrayEquals(siblingPipelineAggregator1.bucketsPaths(), siblingPipelineAggregator2.bucketsPaths());
-                        assertEquals(siblingPipelineAggregator1.name(), siblingPipelineAggregator2.name());
-                    }
-                }
                 if (iteration < 2) {
                     //serialize this enough times to make sure that we are able to write again what we read
                     writeToAndReadFrom(deserialized, iteration + 1);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
index 10f89af509234..882ce66a24ba3 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
@@ -80,7 +80,7 @@ public void testParsingRangeStrict() throws IOException {
             "}";
         XContentParser parser = createParser(JsonXContent.jsonXContent, rangeAggregation);
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> DateRangeAggregationBuilder.parse("aggregationName", parser));
+                () -> DateRangeAggregationBuilder.PARSER.parse(parser, "aggregationName"));
         assertThat(ex.getCause(), notNullValue());
         assertThat(ex.getCause().getMessage(), containsString("badField"));
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
index 6acca86bffa14..b3afa9b25a92b 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
@@ -76,7 +76,7 @@ public void testParsingRangeStrict() throws IOException {
             "}";
         XContentParser parser = createParser(JsonXContent.jsonXContent, rangeAggregation);
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> RangeAggregationBuilder.parse("aggregationName", parser));
+                () -> RangeAggregationBuilder.PARSER.parse(parser, "aggregationName"));
         assertThat(ex.getCause(), notNullValue());
         assertThat(ex.getCause().getMessage(), containsString("badField"));
     }
@@ -92,7 +92,7 @@ public void testParsingNull() throws IOException {
                 "]\n" +
             "}";
         XContentParser parser = createParser(JsonXContent.jsonXContent, rangeAggregation);
-        RangeAggregationBuilder aggregationBuilder = (RangeAggregationBuilder) RangeAggregationBuilder.parse("aggregationName", parser);
+        RangeAggregationBuilder aggregationBuilder = RangeAggregationBuilder.PARSER.parse(parser, "aggregationName");
         assertEquals(1, aggregationBuilder.ranges().size());
         assertEquals(Double.NEGATIVE_INFINITY, aggregationBuilder.ranges().get(0).getFrom(), 0.0);
         assertEquals(Double.POSITIVE_INFINITY, aggregationBuilder.ranges().get(0).getTo(), 0.0);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
index 5075ed9f02aa2..71f962a205233 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
@@ -18,6 +18,11 @@
  */
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParseException;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -26,11 +31,6 @@
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.test.ESTestCase;
 
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-
 public class GeoHashGridParserTests extends ESTestCase {
     public void testParseValidFromInts() throws Exception {
         int precision = randomIntBetween(1, 12);
@@ -39,7 +39,7 @@ public void testParseValidFromInts() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+        assertNotNull(GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
     }
 
     public void testParseValidFromStrings() throws Exception {
@@ -49,7 +49,7 @@ public void testParseValidFromStrings() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+        assertNotNull(GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
     }
 
     public void testParseDistanceUnitPrecision() throws Exception {
@@ -64,7 +64,7 @@ public void testParseDistanceUnitPrecision() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        GeoGridAggregationBuilder builder = GeoHashGridAggregationBuilder.parse("geohash_grid", stParser);
+        GeoGridAggregationBuilder builder = GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid");
         assertNotNull(builder);
         assertThat(builder.precision(), greaterThanOrEqualTo(0));
         assertThat(builder.precision(), lessThanOrEqualTo(12));
@@ -76,7 +76,7 @@ public void testParseInvalidUnitPrecision() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+                () -> GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
         assertThat(ex.getMessage(), containsString("[geohash_grid] failed to parse field [precision]"));
         assertThat(ex.getCause(), instanceOf(NumberFormatException.class));
         assertEquals("For input string: \"10kg\"", ex.getCause().getMessage());
@@ -88,7 +88,7 @@ public void testParseDistanceUnitPrecisionTooSmall() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+                () -> GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
         assertThat(ex.getMessage(), containsString("[geohash_grid] failed to parse field [precision]"));
         assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class));
         assertEquals("precision too high [1cm]", ex.getCause().getMessage());
@@ -99,7 +99,7 @@ public void testParseErrorOnBooleanPrecision() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         XContentParseException e = expectThrows(XContentParseException.class,
-                () -> GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+                () -> GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
         assertThat(e.getMessage(), containsString("[geohash_grid] precision doesn't support values of type: VALUE_BOOLEAN"));
     }
 
@@ -108,7 +108,7 @@ public void testParseErrorOnPrecisionOutOfRange() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         try {
-            GeoHashGridAggregationBuilder.parse("geohash_grid", stParser);
+            GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid");
             fail();
         } catch (XContentParseException ex) {
             assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class));
@@ -128,6 +128,6 @@ public void testParseValidBounds() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoHashGridAggregationBuilder.parse("geohash_grid", stParser));
+        assertNotNull(GeoHashGridAggregationBuilder.PARSER.parse(stParser, "geohash_grid"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java
index aa5253564fb49..a3a31ff85e077 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java
@@ -18,6 +18,9 @@
  */
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.xcontent.XContentParseException;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -26,9 +29,6 @@
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.test.ESTestCase;
 
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
-
 public class GeoTileGridParserTests extends ESTestCase {
     public void testParseValidFromInts() throws Exception {
         int precision = randomIntBetween(0, GeoTileUtils.MAX_ZOOM);
@@ -37,7 +37,7 @@ public void testParseValidFromInts() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoTileGridAggregationBuilder.parse("geotile_grid", stParser));
+        assertNotNull(GeoTileGridAggregationBuilder.PARSER.parse(stParser, "geotile_grid"));
     }
 
     public void testParseValidFromStrings() throws Exception {
@@ -47,7 +47,7 @@ public void testParseValidFromStrings() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoTileGridAggregationBuilder.parse("geotile_grid", stParser));
+        assertNotNull(GeoTileGridAggregationBuilder.PARSER.parse(stParser, "geotile_grid"));
     }
 
     public void testParseErrorOnBooleanPrecision() throws Exception {
@@ -55,7 +55,7 @@ public void testParseErrorOnBooleanPrecision() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         XContentParseException e = expectThrows(XContentParseException.class,
-                () -> GeoTileGridAggregationBuilder.parse("geotile_grid", stParser));
+                () -> GeoTileGridAggregationBuilder.PARSER.parse(stParser, "geotile_grid"));
         assertThat(ExceptionsHelper.stackTrace(e),
                 containsString("[geotile_grid] precision doesn't support values of type: VALUE_BOOLEAN"));
     }
@@ -65,7 +65,7 @@ public void testParseErrorOnPrecisionOutOfRange() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         try {
-            GeoTileGridAggregationBuilder.parse("geotile_grid", stParser);
+            GeoTileGridAggregationBuilder.PARSER.parse(stParser, "geotile_grid");
             fail();
         } catch (XContentParseException ex) {
             assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class));
@@ -85,6 +85,6 @@ public void testParseValidBounds() throws Exception {
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
-        assertNotNull(GeoTileGridAggregationBuilder.parse("geotile_grid", stParser));
+        assertNotNull(GeoTileGridAggregationBuilder.PARSER.parse(stParser, "geotile_grid"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
index 6cb4826ead2c4..aa35aab052b2c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
@@ -36,6 +36,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -46,10 +47,14 @@
 import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregatorFactory.ExecutionMode;
 import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -75,6 +80,27 @@ public void setUpTest() throws Exception {
         fieldType.setName("field");
     }
 
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new SignificantTermsAggregationBuilder("foo", ValueType.STRING).field(fieldName);
+    }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return List.of(CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.BYTES);
+    }
+
+    @Override
+    protected List<String> unsupportedMappedFieldTypes() {
+        return List.of(
+            NumberFieldMapper.NumberType.DOUBLE.typeName(), // floating points are not supported at all
+            NumberFieldMapper.NumberType.FLOAT.typeName(),
+            NumberFieldMapper.NumberType.HALF_FLOAT.typeName(),
+            BinaryFieldMapper.CONTENT_TYPE // binary fields are not supported because they cannot be searched
+        );
+    }
+
     /**
      * For each provided field type, we also register an alias with name {@code <field>-alias}.
      */
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
index 123bb0f60427c..6a354b0a6a496 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
@@ -34,12 +34,17 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.BinaryFieldMapper;
+import org.elasticsearch.index.mapper.GeoPointFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -63,6 +68,28 @@ protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldT
             Function.identity()));
     }
 
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new SignificantTextAggregationBuilder("foo", fieldName);
+    }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        // TODO it is likely accidental that SigText supports anything other than Bytes, and then only text fields
+        return List.of(CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.BYTES,
+            CoreValuesSourceType.RANGE,
+            CoreValuesSourceType.GEOPOINT);
+    }
+
+    @Override
+    protected List<String> unsupportedMappedFieldTypes() {
+        return List.of(
+            BinaryFieldMapper.CONTENT_TYPE, // binary fields are not supported because they do not have analyzers
+            GeoPointFieldMapper.CONTENT_TYPE // geopoint fields cannot use term queries
+        );
+    }
+
     /**
      * Uses the significant text aggregation to find the keywords in text fields
      */
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java
index 46f08d786a856..b34132a51e00a 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java
@@ -52,7 +52,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.io.IOException;
@@ -709,4 +711,16 @@ public void testScriptCaching() throws IOException {
         directory.close();
         unmappedDirectory.close();
     }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return List.of(
+            CoreValuesSourceType.NUMERIC
+        );
+    }
+
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new AvgAggregationBuilder("foo").field(fieldName);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java
index 60a7bdc944d5e..cc58333d379bc 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java
@@ -27,7 +27,6 @@
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexOptions;
@@ -45,7 +44,6 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.CheckedConsumer;
@@ -55,7 +53,6 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.DateFieldMapper;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
@@ -84,10 +81,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.FieldContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 
@@ -278,22 +277,6 @@ public void testUnmappedWithMissingField() throws IOException {
         }, fieldType);
     }
 
-    public void testUnsupportedType() {
-        MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("min").field("not_a_number");
-
-        MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
-        fieldType.setName("not_a_number");
-        fieldType.setHasDocValues(true);
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
-                iw.addDocument(singleton(new SortedSetDocValuesField("string", new BytesRef("foo"))));
-            }, (Consumer<InternalMin>) min -> {
-                fail("Should have thrown exception");
-            }, fieldType));
-        assertEquals(e.getMessage(), "Expected numeric type on field [not_a_number], but got [keyword]");
-    }
-
     public void testBadMissingField() {
         MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("min").field("number").missing("not_a_number");
 
@@ -936,5 +919,13 @@ private <T extends AggregationBuilder, V extends InternalAggregation> void testC
         }
     }
 
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return Collections.singletonList(CoreValuesSourceType.NUMERIC);
+    }
 
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new MinAggregationBuilder("foo").field(fieldName);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java
index 28b1514545506..ccd01c94de2f3 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java
@@ -22,30 +22,62 @@
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.stats;
 
 public class StatsAggregatorTests extends AggregatorTestCase {
-    static final double TOLERANCE = 1e-10;
+
+    private static final double TOLERANCE = 1e-10;
+    private static final String VALUE_SCRIPT_NAME = "value_script";
+    private static final String FIELD_SCRIPT_NAME = "field_script";
 
     public void testEmpty() throws IOException {
-        MappedFieldType ft =
-            new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
         ft.setName("field");
-        testCase(ft, iw -> {},
+        testCase(
+            stats("_name").field(ft.name()),
+            iw -> {},
             stats -> {
                 assertEquals(0d, stats.getCount(), 0);
                 assertEquals(0d, stats.getSum(), 0);
@@ -53,16 +85,17 @@ public void testEmpty() throws IOException {
                 assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0);
                 assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0);
                 assertFalse(AggregationInspectionHelper.hasValue(stats));
-            }
+            },
+            singleton(ft)
         );
     }
 
     public void testRandomDoubles() throws IOException {
-        MappedFieldType ft =
-            new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
+        final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.DOUBLE);
         ft.setName("field");
         final SimpleStatsAggregator expected = new SimpleStatsAggregator();
-        testCase(ft,
+        testCase(
+            stats("_name").field(ft.name()),
             iw -> {
                 int numDocs = randomIntBetween(10, 50);
                 for (int i = 0; i < numDocs; i++) {
@@ -71,7 +104,7 @@ public void testRandomDoubles() throws IOException {
                     for (int j = 0; j < numValues; j++) {
                         double value = randomDoubleBetween(-100d, 100d, true);
                         long valueAsLong = NumericUtils.doubleToSortableLong(value);
-                        doc.add(new SortedNumericDocValuesField("field", valueAsLong));
+                        doc.add(new SortedNumericDocValuesField(ft.name(), valueAsLong));
                         expected.add(value);
                     }
                     iw.addDocument(doc);
@@ -84,30 +117,16 @@ public void testRandomDoubles() throws IOException {
                 assertEquals(expected.max, stats.getMax(), 0);
                 assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE);
                 assertTrue(AggregationInspectionHelper.hasValue(stats));
-            }
+            },
+            singleton(ft)
         );
     }
 
     public void testRandomLongs() throws IOException {
-        MappedFieldType ft =
-            new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
-        ft.setName("field");
-        final SimpleStatsAggregator expected = new SimpleStatsAggregator();
-        testCase(ft,
-            iw -> {
-                int numDocs = randomIntBetween(10, 50);
-                for (int i = 0; i < numDocs; i++) {
-                    Document doc = new Document();
-                    int numValues = randomIntBetween(1, 5);
-                    for (int j = 0; j < numValues; j++) {
-                        long value = randomIntBetween(-100, 100);
-                        doc.add(new SortedNumericDocValuesField("field", value));
-                        expected.add(value);
-                    }
-                    iw.addDocument(doc);
-                }
-            },
-            stats -> {
+        randomLongsTestCase(
+            randomIntBetween(1, 5),
+            stats("_name").field("field"),
+            (expected, stats) -> {
                 assertEquals(expected.count, stats.getCount(), 0);
                 assertEquals(expected.sum, stats.getSum(), TOLERANCE);
                 assertEquals(expected.min, stats.getMin(), 0);
@@ -151,7 +170,7 @@ public void testSummationAccuracy() throws IOException {
 
     private void verifySummationOfDoubles(double[] values, double expectedSum,
                                           double expectedAvg, double delta) throws IOException {
-        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.DOUBLE);
         ft.setName("field");
 
         double max = Double.NEGATIVE_INFINITY;
@@ -162,10 +181,11 @@ private void verifySummationOfDoubles(double[] values, double expectedSum,
         }
         double expectedMax = max;
         double expectedMin = min;
-        testCase(ft,
+        testCase(
+            stats("_name").field(ft.name()),
             iw -> {
                 for (double value : values) {
-                    iw.addDocument(singleton(new NumericDocValuesField("field", NumericUtils.doubleToSortableLong(value))));
+                    iw.addDocument(singleton(new NumericDocValuesField(ft.name(), NumericUtils.doubleToSortableLong(value))));
                 }
             },
             stats -> {
@@ -174,20 +194,251 @@ private void verifySummationOfDoubles(double[] values, double expectedSum,
                 assertEquals(expectedSum, stats.getSum(), delta);
                 assertEquals(expectedMax, stats.getMax(), 0d);
                 assertEquals(expectedMin, stats.getMin(), 0d);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            },
+            singleton(ft)
+        );
+    }
+
+    public void testUnmapped() throws IOException {
+        randomLongsTestCase(
+            randomIntBetween(1, 5),
+            stats("_name").field("unmapped_field"),
+            (expected, stats) -> {
+                assertEquals(0d, stats.getCount(), 0);
+                assertEquals(0d, stats.getSum(), 0);
+                assertEquals(Float.NaN, stats.getAvg(), 0);
+                assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0);
+                assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0);
+                assertFalse(AggregationInspectionHelper.hasValue(stats));
+            }
+        );
+    }
+
+    public void testPartiallyUnmapped() throws IOException {
+        try (Directory mappedDirectory = newDirectory();
+             Directory unmappedDirectory = newDirectory();
+             RandomIndexWriter mappedWriter = new RandomIndexWriter(random(), mappedDirectory);
+             RandomIndexWriter unmappedWriter = new RandomIndexWriter(random(), unmappedDirectory)) {
+
+            final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+            ft.setName("field");
+            final SimpleStatsAggregator expected = new SimpleStatsAggregator();
+            final int numDocs = randomIntBetween(10, 50);
+            for (int i = 0; i < numDocs; i++) {
+                final long value = randomLongBetween(-100, 100);
+                mappedWriter.addDocument(singleton(new SortedNumericDocValuesField(ft.name(), value)));
+                expected.add(value);
+            }
+            final StatsAggregationBuilder builder = stats("_name").field(ft.name());
+
+            try (IndexReader mappedReader = mappedWriter.getReader();
+                 IndexReader unmappedReader = unmappedWriter.getReader();
+                 MultiReader multiReader = new MultiReader(mappedReader, unmappedReader)) {
+
+                final IndexSearcher searcher = new IndexSearcher(multiReader);
+                final InternalStats stats = search(searcher, new MatchAllDocsQuery(), builder, ft);
+
+                assertEquals(expected.count, stats.getCount(), 0);
+                assertEquals(expected.sum, stats.getSum(), TOLERANCE);
+                assertEquals(expected.max, stats.getMax(), 0);
+                assertEquals(expected.min, stats.getMin(), 0);
+                assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            }
+        }
+    }
+
+    public void testValueScriptSingleValuedField() throws IOException {
+        randomLongsTestCase(
+            1,
+            stats("_name")
+                .field("field")
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_NAME, emptyMap())),
+            (expected, stats) -> {
+                final SimpleStatsAggregator adjusted = new SimpleStatsAggregator(
+                    expected.count,
+                    expected.min + 1,
+                    expected.max + 1,
+                    expected.sum + expected.count
+                );
+
+                assertEquals(adjusted.count, stats.getCount(), 0);
+                assertEquals(adjusted.sum, stats.getSum(), TOLERANCE);
+                assertEquals(adjusted.max, stats.getMax(), 0);
+                assertEquals(adjusted.min, stats.getMin(), 0);
+                assertEquals(adjusted.sum / adjusted.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            }
+        );
+    }
+
+    public void testValueScriptMultiValuedField() throws IOException {
+        final int valuesPerField = randomIntBetween(2, 5);
+        randomLongsTestCase(
+            valuesPerField,
+            stats("_name")
+                .field("field")
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_NAME, emptyMap())),
+            (expected, stats) -> {
+                final SimpleStatsAggregator adjusted = new SimpleStatsAggregator(
+                    expected.count,
+                    expected.min + 1,
+                    expected.max + 1,
+                    expected.sum + expected.count
+                );
+
+                assertEquals(adjusted.count, stats.getCount(), 0);
+                assertEquals(adjusted.sum, stats.getSum(), TOLERANCE);
+                assertEquals(adjusted.max, stats.getMax(), 0);
+                assertEquals(adjusted.min, stats.getMin(), 0);
+                assertEquals(adjusted.sum / adjusted.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            }
+        );
+    }
+
+    public void testFieldScriptSingleValuedField() throws IOException {
+        randomLongsTestCase(
+            1,
+            stats("_name")
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_NAME, singletonMap("field", "field"))),
+            (expected, stats) -> {
+                final SimpleStatsAggregator adjusted = new SimpleStatsAggregator(
+                    expected.count,
+                    expected.min + 1,
+                    expected.max + 1,
+                    expected.sum + expected.count
+                );
+
+                assertEquals(adjusted.count, stats.getCount(), 0);
+                assertEquals(adjusted.sum, stats.getSum(), TOLERANCE);
+                assertEquals(adjusted.max, stats.getMax(), 0);
+                assertEquals(adjusted.min, stats.getMin(), 0);
+                assertEquals(adjusted.sum / adjusted.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            }
+        );
+    }
+
+    public void testFieldScriptMultiValuedField() throws IOException {
+        final int valuesPerField = randomIntBetween(2, 5);
+        randomLongsTestCase(
+            valuesPerField,
+            stats("_name")
+                .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_NAME, singletonMap("field", "field"))),
+            (expected, stats) -> {
+                final SimpleStatsAggregator adjusted = new SimpleStatsAggregator(
+                    expected.count,
+                    expected.min + 1,
+                    expected.max + 1,
+                    expected.sum + expected.count
+                );
+
+                assertEquals(adjusted.count, stats.getCount(), 0);
+                assertEquals(adjusted.sum, stats.getSum(), TOLERANCE);
+                assertEquals(adjusted.max, stats.getMax(), 0);
+                assertEquals(adjusted.min, stats.getMin(), 0);
+                assertEquals(adjusted.sum / adjusted.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            }
+        );
+    }
+
+    public void testMissing() throws IOException {
+        final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        ft.setName("field");
+
+        final long missingValue = randomIntBetween(-100, 100);
+
+        final int numDocs = randomIntBetween(100, 200);
+        final List<Set<IndexableField>> docs = new ArrayList<>(numDocs);
+        final SimpleStatsAggregator expected = new SimpleStatsAggregator();
+        for (int i = 0; i < numDocs; i++) {
+            if (randomBoolean()) {
+                final long value = randomLongBetween(-100, 100);
+                docs.add(singleton(new SortedNumericDocValuesField(ft.name(), value)));
+                expected.add(value);
+            } else {
+                docs.add(emptySet());
+                expected.add(missingValue);
+            }
+        }
+
+        testCase(
+            stats("_name")
+                .field(ft.name())
+                .missing(missingValue),
+            iw -> iw.addDocuments(docs),
+            stats -> {
+                assertEquals(expected.count, stats.getCount(), 0);
+                assertEquals(expected.sum, stats.getSum(), TOLERANCE);
+                assertEquals(expected.max, stats.getMax(), 0);
+                assertEquals(expected.min, stats.getMin(), 0);
+                assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
+            },
+            singleton(ft)
+        );
+    }
+
+    public void testMissingUnmapped() throws IOException {
+        final int valuesPerField = randomIntBetween(1, 5);
+        final long missingValue = randomLongBetween(-100, 100);
+        randomLongsTestCase(
+            valuesPerField,
+            stats("_name")
+                .field("unknown_field")
+                .missing(missingValue),
+            (expected, stats) -> {
+                final long numDocs = expected.count / valuesPerField;
+                assertEquals(numDocs, stats.getCount());
+                assertEquals(numDocs * missingValue, stats.getSum(), TOLERANCE);
+                assertEquals(missingValue, stats.getMax(), 0);
+                assertEquals(missingValue, stats.getMin(), 0);
+                assertEquals(missingValue, stats.getAvg(), TOLERANCE);
+                assertTrue(AggregationInspectionHelper.hasValue(stats));
             }
         );
     }
 
-    public void testCase(MappedFieldType ft,
-                         CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
-                         Consumer<InternalStats> verify) throws IOException {
+    private void randomLongsTestCase(int valuesPerField,
+                                     StatsAggregationBuilder builder,
+                                     BiConsumer<SimpleStatsAggregator, InternalStats> verify) throws IOException {
+
+        final MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        ft.setName("field");
+
+        final int numDocs = randomIntBetween(10, 50);
+        final List<Set<IndexableField>> docs = new ArrayList<>(numDocs);
+        final SimpleStatsAggregator expected = new SimpleStatsAggregator();
+        for (int iDoc = 0; iDoc < numDocs; iDoc++) {
+            List<Long> values = randomList(valuesPerField, valuesPerField, () -> randomLongBetween(-100, 100));
+            docs.add(values.stream()
+                .map(value -> new SortedNumericDocValuesField(ft.name(), value))
+                .collect(toSet()));
+            values.forEach(expected::add);
+        }
+
+        testCase(
+            builder,
+            iw -> iw.addDocuments(docs),
+            stats -> verify.accept(expected, stats),
+            singleton(ft)
+        );
+    }
+
+    private void testCase(StatsAggregationBuilder builder,
+                          CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
+                          Consumer<InternalStats> verify,
+                          Collection<MappedFieldType> fieldTypes) throws IOException {
         try (Directory directory = newDirectory();
-             RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+            RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
             buildIndex.accept(indexWriter);
             try (IndexReader reader = indexWriter.getReader()) {
                 IndexSearcher searcher = new IndexSearcher(reader);
-                StatsAggregationBuilder aggBuilder = new StatsAggregationBuilder("my_agg").field("field");
-                InternalStats stats = search(searcher, new MatchAllDocsQuery(), aggBuilder, ft);
+                final MappedFieldType[] fieldTypesArray = fieldTypes.toArray(new MappedFieldType[0]);
+                final InternalStats stats = search(searcher, new MatchAllDocsQuery(), builder, fieldTypesArray);
                 verify.accept(stats);
             }
         }
@@ -199,6 +450,15 @@ static class SimpleStatsAggregator {
         double max = Long.MIN_VALUE;
         double sum = 0;
 
+        SimpleStatsAggregator() {}
+
+        SimpleStatsAggregator(long count, double min, double max, double sum) {
+            this.count = count;
+            this.min = min;
+            this.max = max;
+            this.sum = sum;
+        }
+
         void add(double value) {
             count ++;
             if (Double.compare(value, min) < 0) {
@@ -210,4 +470,32 @@ void add(double value) {
             sum += value;
         }
     }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return singletonList(CoreValuesSourceType.NUMERIC);
+    }
+
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new StatsAggregationBuilder("_name")
+            .field(fieldName);
+    }
+
+    @Override
+    protected ScriptService getMockScriptService() {
+        final Map<String, Function<Map<String, Object>, Object>> scripts = Map.of(
+            VALUE_SCRIPT_NAME, vars -> ((Number) vars.get("_value")).doubleValue() + 1,
+            FIELD_SCRIPT_NAME, vars -> {
+                final String fieldName = (String) vars.get("field");
+                final LeafDocLookup lookup = (LeafDocLookup) vars.get("doc");
+                return lookup.get(fieldName).stream()
+                    .map(value -> ((Number) value).longValue() + 1)
+                    .collect(toList());
+            }
+        );
+        final MockScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, emptyMap());
+        final Map<String, ScriptEngine> engines = singletonMap(engine.getType(), engine);
+        return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
index d393b4f0b2c64..7a084c21d80f5 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
@@ -35,11 +35,8 @@
 
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
-import static java.util.Collections.emptyMap;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
@@ -86,40 +83,6 @@ public void testEmptyAggregation() throws Exception {
         assertThat(Double.isNaN(stats.getAvg()), is(true));
     }
 
-    @Override
-    public void testUnmapped() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx_unmapped")
-                .setQuery(matchAllQuery())
-                .addAggregation(stats("stats").field("value"))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L));
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo(Double.NaN));
-        assertThat(stats.getMin(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(stats.getMax(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(stats.getSum(), equalTo(0.0));
-        assertThat(stats.getCount(), equalTo(0L));
-    }
-
-    public void testPartiallyUnmapped() {
-        Stats s1 = client().prepareSearch("idx")
-                .addAggregation(stats("stats").field("value")).get()
-                .getAggregations().get("stats");
-        Stats s2 = client().prepareSearch("idx", "idx_unmapped")
-                .addAggregation(stats("stats").field("value")).get()
-                .getAggregations().get("stats");
-        assertEquals(s1.getAvg(), s2.getAvg(), 1e-10);
-        assertEquals(s1.getCount(), s2.getCount());
-        assertEquals(s1.getMin(), s2.getMin(), 0d);
-        assertEquals(s1.getMax(), s2.getMax(), 0d);
-    }
-
     @Override
     public void testSingleValuedField() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
@@ -199,77 +162,6 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         assertThat((double) ((InternalAggregation)global).getProperty("stats.count"), equalTo((double) expectedCountValue));
     }
 
-    @Override
-    public void testSingleValuedFieldPartiallyUnmapped() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx", "idx_unmapped")
-                .setQuery(matchAllQuery())
-                .addAggregation(stats("stats").field("value"))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10));
-        assertThat(stats.getMin(), equalTo(1.0));
-        assertThat(stats.getMax(), equalTo(10.0));
-        assertThat(stats.getSum(), equalTo((double) 1+2+3+4+5+6+7+8+9+10));
-        assertThat(stats.getCount(), equalTo(10L));
-    }
-
-    @Override
-    public void testSingleValuedFieldWithValueScript() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .field("value")
-                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + 1", emptyMap())))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10));
-        assertThat(stats.getMin(), equalTo(2.0));
-        assertThat(stats.getMax(), equalTo(11.0));
-        assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11));
-        assertThat(stats.getCount(), equalTo(10L));
-    }
-
-    @Override
-    public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("inc", 1);
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .field("value")
-                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + inc", params)))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10));
-        assertThat(stats.getMin(), equalTo(2.0));
-        assertThat(stats.getMax(), equalTo(11.0));
-        assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11));
-        assertThat(stats.getCount(), equalTo(10L));
-    }
-
     @Override
     public void testMultiValuedField() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx")
@@ -291,157 +183,6 @@ public void testMultiValuedField() throws Exception {
         assertThat(stats.getCount(), equalTo(20L));
     }
 
-    @Override
-    public void testMultiValuedFieldWithValueScript() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .field("values")
-                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap())))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (1+2+3+4+5+6+7+8+9+10+2+3+4+5+6+7+8+9+10+11) / 20));
-        assertThat(stats.getMin(), equalTo(1.0));
-        assertThat(stats.getMax(), equalTo(11.0));
-        assertThat(stats.getSum(), equalTo((double) 1+2+3+4+5+6+7+8+9+10+2+3+4+5+6+7+8+9+10+11));
-        assertThat(stats.getCount(), equalTo(20L));
-    }
-
-    @Override
-    public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("dec", 1);
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .field("values")
-                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - dec", params)))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (1+2+3+4+5+6+7+8+9+10+2+3+4+5+6+7+8+9+10+11) / 20));
-        assertThat(stats.getMin(), equalTo(1.0));
-        assertThat(stats.getMax(), equalTo(11.0));
-        assertThat(stats.getSum(), equalTo((double) 1+2+3+4+5+6+7+8+9+10+2+3+4+5+6+7+8+9+10+11));
-        assertThat(stats.getCount(), equalTo(20L));
-    }
-
-    @Override
-    public void testScriptSingleValued() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value", emptyMap())))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (1+2+3+4+5+6+7+8+9+10) / 10));
-        assertThat(stats.getMin(), equalTo(1.0));
-        assertThat(stats.getMax(), equalTo(10.0));
-        assertThat(stats.getSum(), equalTo((double) 1+2+3+4+5+6+7+8+9+10));
-        assertThat(stats.getCount(), equalTo(10L));
-    }
-
-    @Override
-    public void testScriptSingleValuedWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("inc", 1);
-
-        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value + inc", params);
-
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(stats("stats").script(script))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11) / 10));
-        assertThat(stats.getMin(), equalTo(2.0));
-        assertThat(stats.getMax(), equalTo(11.0));
-        assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11));
-        assertThat(stats.getCount(), equalTo(10L));
-    }
-
-    @Override
-    public void testScriptMultiValued() throws Exception {
-        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['values']", emptyMap());
-
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(stats("stats").script(script))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (2+3+4+5+6+7+8+9+10+11+3+4+5+6+7+8+9+10+11+12) / 20));
-        assertThat(stats.getMin(), equalTo(2.0));
-        assertThat(stats.getMax(), equalTo(12.0));
-        assertThat(stats.getSum(), equalTo((double) 2+3+4+5+6+7+8+9+10+11+3+4+5+6+7+8+9+10+11+12));
-        assertThat(stats.getCount(), equalTo(20L));
-    }
-
-    @Override
-    public void testScriptMultiValuedWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("dec", 1);
-
-        Script script = new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "[ doc['value'].value, doc['value'].value - dec ]",
-            params);
-
-        SearchResponse searchResponse = client().prepareSearch("idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(
-                        stats("stats")
-                                .script(script))
-                .get();
-
-        assertShardExecutionState(searchResponse, 0);
-
-        assertHitCount(searchResponse, 10);
-
-        Stats stats = searchResponse.getAggregations().get("stats");
-        assertThat(stats, notNullValue());
-        assertThat(stats.getName(), equalTo("stats"));
-        assertThat(stats.getAvg(), equalTo((double) (1+2+3+4+5+6+7+8+9+10+0+1+2+3+4+5+6+7+8+9) / 20));
-        assertThat(stats.getMin(), equalTo(0.0));
-        assertThat(stats.getMax(), equalTo(10.0));
-        assertThat(stats.getSum(), equalTo((double) 1+2+3+4+5+6+7+8+9+10+0+1+2+3+4+5+6+7+8+9));
-        assertThat(stats.getCount(), equalTo(20L));
-    }
-
     @Override
     public void testOrderByEmptyAggregation() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index fb6b021eda0bf..42ace8a25f4bf 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -164,34 +164,6 @@ public void testSingleValuedFieldGetProperty() throws Exception {
         assertThat((double) ((InternalAggregation)sum).getProperty("value"), equalTo(expectedSumValue));
     }
 
-    /** This test has been moved to {@link SumAggregatorTests#testPartiallyUnmapped()} */
-    @Override
-    public void testSingleValuedFieldPartiallyUnmapped() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testValueScriptSingleValuedField()} */
-    @Override
-    public void testSingleValuedFieldWithValueScript() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testValueScriptSingleValuedField()} */
-    @Override
-    public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptSingleValuedField()} */
-    @Override
-    public void testScriptSingleValued() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptSingleValuedField()} */
-    @Override
-    public void testScriptSingleValuedWithParams() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptMultiValuedField()} */
-    @Override
-    public void testScriptMultiValued() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testFieldScriptMultiValuedField()} */
-    @Override
-    public void testScriptMultiValuedWithParams() throws Exception {}
-
     @Override
     public void testMultiValuedField() throws Exception {
 
@@ -208,14 +180,6 @@ public void testMultiValuedField() throws Exception {
         assertThat(sum.getValue(), equalTo((double) 2+3+3+4+4+5+5+6+6+7+7+8+8+9+9+10+10+11+11+12));
     }
 
-    /** this test has been moved to {@link SumAggregatorTests#testValueScriptMultiValuedField()} */
-    @Override
-    public void testMultiValuedFieldWithValueScript() throws Exception {}
-
-    /** this test has been moved to {@link SumAggregatorTests#testValueScriptMultiValuedField()} */
-    @Override
-    public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {}
-
     @Override
     public void testOrderByEmptyAggregation() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java
index 8f268cecaedd2..d53a147261cb1 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java
@@ -40,5 +40,4 @@ protected final PAF createTestAggregatorFactory() {
     }
 
     protected abstract PAF doCreateTestAggregatorFactory(String name, String bucketsPath);
-
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketTests.java
index 0dc10cb7a7a13..2c5f0f9f343c3 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
 public class AvgBucketTests extends AbstractBucketMetricsTestCase<AvgBucketPipelineAggregationBuilder> {
 
     @Override
@@ -43,23 +45,18 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final AvgBucketPipelineAggregationBuilder builder = new AvgBucketPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new AvgBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        AvgBucketPipelineAggregationBuilder builder2 = new AvgBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new AvgBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
-
-        // Now try to point to a valid multi-bucket agg (no exception should be thrown)
-        AvgBucketPipelineAggregationBuilder builder3 = new AvgBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+                + " for buckets path: global>metric;"));
 
+        // Now try to point to a valid multi-bucket agg which is valid
+        assertThat(validate(aggBuilders, new AvgBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java
index d5e70aff07b69..ead4870731b1f 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSortIT.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -459,21 +459,21 @@ public void testEmptyBuckets() {
     }
 
     public void testInvalidPath() {
-        SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class,
+        Exception e = expectThrows(ActionRequestValidationException.class,
                 () -> client().prepareSearch(INDEX)
                 .addAggregation(terms("foos").field(TERM_FIELD)
                         .subAggregation(bucketSort("bucketSort", Arrays.asList(new FieldSortBuilder("invalid")))))
                 .get());
-        assertThat(e.getCause().getMessage(), containsString("No aggregation found for path [invalid]"));
+        assertThat(e.getMessage(), containsString("No aggregation found for path [invalid]"));
     }
 
     public void testNeitherSortsNorSizeSpecifiedAndFromIsDefault_ShouldThrowValidation() {
-        SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class,
+        Exception e = expectThrows(ActionRequestValidationException.class,
                 () -> client().prepareSearch(INDEX)
                         .addAggregation(terms("foos").field(TERM_FIELD)
                                 .subAggregation(bucketSort("bucketSort", Collections.emptyList())))
                         .get());
-        assertThat(e.getCause().getMessage(), containsString("[bucketSort] is configured to perform nothing." +
+        assertThat(e.getMessage(), containsString("[bucketSort] is configured to perform nothing." +
                 " Please set either of [sort, size, from] to use bucket_sort"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java
index 9d27663d275f7..dd3e971f4400b 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java
@@ -38,8 +38,6 @@
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.TestAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -52,10 +50,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -291,34 +286,6 @@ public void testNoBuckets() throws IOException {
             }
         });
     }
-    
-    /**
-     * The validation should verify the parent aggregation is allowed.
-     */
-    public void testValidate() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        aggBuilders.add(new CumulativeSumPipelineAggregationBuilder("cusum", "sum"));
-
-        final CumulativeSumPipelineAggregationBuilder builder = new CumulativeSumPipelineAggregationBuilder("name", "valid");
-        builder.validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders);
-    }
-
-    /**
-     * The validation should throw an IllegalArgumentException, since parent
-     * aggregation is not a type of HistogramAggregatorFactory,
-     * DateHistogramAggregatorFactory or AutoDateHistogramAggregatorFactory.
-     */
-    public void testValidateException() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        aggBuilders.add(new CumulativeSumPipelineAggregationBuilder("cusum", "sum"));
-        TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance();
-
-        final CumulativeSumPipelineAggregationBuilder builder = new CumulativeSumPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalStateException ex = expectThrows(IllegalStateException.class,
-                () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders));
-        assertEquals("cumulative_sum aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent",
-                ex.getMessage());
-    }
 
     private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consumer<InternalAggregation> verify) throws IOException {
         executeTestCase(query, aggBuilder, verify, indexWriter -> {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
index edf879ce77f68..f51b8ec479d8b 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
@@ -19,10 +19,18 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 
-public class CumulativeSumTests extends BasePipelineAggregationTestCase<CumulativeSumPipelineAggregationBuilder> {
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
+public class CumulativeSumTests extends BasePipelineAggregationTestCase<CumulativeSumPipelineAggregationBuilder> {
     @Override
     protected CumulativeSumPipelineAggregationBuilder createTestAggregatorFactory() {
         String name = randomAlphaOfLengthBetween(3, 20);
@@ -34,4 +42,23 @@ protected CumulativeSumPipelineAggregationBuilder createTestAggregatorFactory()
         return factory;
     }
 
+    public void testValidate() throws IOException {
+        assertThat(validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(),
+                new CumulativeSumPipelineAggregationBuilder("name", "valid")), nullValue());
+    }
+
+    public void testInvalidParent() throws IOException {
+        AggregationBuilder parent = mock(AggregationBuilder.class);
+        when(parent.getName()).thenReturn("name");
+
+        assertThat(validate(parent, new CumulativeSumPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: cumulative_sum aggregation [name] must have a histogram, date_histogram "
+                + "or auto_date_histogram as parent;"));
+    }
+
+    public void testNoParent() throws IOException {
+        assertThat(validate(emptyList(), new CumulativeSumPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: cumulative_sum aggregation [name] must have a histogram, date_histogram "
+                + "or auto_date_histogram as parent but doesn't have a parent;"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
index 7fd6e1e86266b..662778315f7e4 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
@@ -19,16 +19,20 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.TestAggregatorFactory;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 public class DerivativeTests extends BasePipelineAggregationTestCase<DerivativePipelineAggregationBuilder> {
 
     @Override
@@ -51,16 +55,13 @@ protected DerivativePipelineAggregationBuilder createTestAggregatorFactory() {
         }
         return factory;
     }
-    
+
     /**
      * The validation should verify the parent aggregation is allowed.
      */
     public void testValidate() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        aggBuilders.add(new DerivativePipelineAggregationBuilder("deriv", "der"));
-
-        final DerivativePipelineAggregationBuilder builder = new DerivativePipelineAggregationBuilder("name", "valid");
-        builder.validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders);
+        assertThat(validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(),
+                new DerivativePipelineAggregationBuilder("name", "valid")), nullValue());
     }
 
     /**
@@ -71,12 +72,11 @@ public void testValidate() throws IOException {
     public void testValidateException() throws IOException {
         final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
         aggBuilders.add(new DerivativePipelineAggregationBuilder("deriv", "der"));
-        TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance();
+        AggregationBuilder parent = mock(AggregationBuilder.class);
+        when(parent.getName()).thenReturn("name");
 
-        final DerivativePipelineAggregationBuilder builder = new DerivativePipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalStateException ex = expectThrows(IllegalStateException.class,
-                () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders));
-        assertEquals("derivative aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent",
-                ex.getMessage());
+        assertThat(validate(parent, new DerivativePipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: derivative aggregation [name] must have a histogram, "
+                + "date_histogram or auto_date_histogram as parent;"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java
index 9930541cb007e..90cc9be95e3be 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketTests.java
@@ -26,11 +26,11 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase<ExtendedStatsBucketPipelineAggregationBuilder> {
 
@@ -68,23 +68,17 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final ExtendedStatsBucketPipelineAggregationBuilder builder = new ExtendedStatsBucketPipelineAggregationBuilder("name",
-                "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new ExtendedStatsBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        ExtendedStatsBucketPipelineAggregationBuilder builder2 = new ExtendedStatsBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new ExtendedStatsBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        ExtendedStatsBucketPipelineAggregationBuilder builder3 = new ExtendedStatsBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new ExtendedStatsBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java
index c55152c68c3a6..edbc1cda3eae2 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
 public class MaxBucketTests extends AbstractBucketMetricsTestCase<MaxBucketPipelineAggregationBuilder> {
 
     @Override
@@ -43,23 +45,18 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final MaxBucketPipelineAggregationBuilder builder = new MaxBucketPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new MaxBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        MaxBucketPipelineAggregationBuilder builder2 = new MaxBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new MaxBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        MaxBucketPipelineAggregationBuilder builder3 = new MaxBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new MaxBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java
index 317f1360c7845..057e074a90cfc 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MinBucketTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
 public class MinBucketTests extends AbstractBucketMetricsTestCase<MinBucketPipelineAggregationBuilder> {
 
     @Override
@@ -43,23 +45,18 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final MinBucketPipelineAggregationBuilder builder = new MinBucketPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new MinBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        MinBucketPipelineAggregationBuilder builder2 = new MinBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new MinBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        MinBucketPipelineAggregationBuilder builder3 = new MinBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new MinBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java
similarity index 79%
rename from server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java
rename to server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java
index 9d3b10c39b1ff..e297e8a473f5f 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java
@@ -42,8 +42,6 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.TestAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -53,16 +51,14 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class MovFnUnitTests extends AggregatorTestCase {
+public class MovFnAggrgatorTests extends AggregatorTestCase {
 
     private static final String DATE_FIELD = "date";
     private static final String INSTANT_FIELD = "instant";
@@ -172,34 +168,4 @@ private void executeTestCase(Query query,
     private static long asLong(String dateTime) {
         return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli();
     }
-    
-    /**
-     * The validation should verify the parent aggregation is allowed.
-     */
-    public void testValidate() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, "painless", "test", Collections.emptyMap());
-        aggBuilders.add(new MovFnPipelineAggregationBuilder("mov_fn", "avg", script, 3));
-
-        final MovFnPipelineAggregationBuilder builder = new MovFnPipelineAggregationBuilder("name", "invalid_agg>metric", script, 1);
-        builder.validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders);
-    }
-
-    /**
-     * The validation should throw an IllegalArgumentException, since parent
-     * aggregation is not a type of HistogramAggregatorFactory,
-     * DateHistogramAggregatorFactory or AutoDateHistogramAggregatorFactory.
-     */
-    public void testValidateException() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, "painless", "test", Collections.emptyMap());
-        aggBuilders.add(new MovFnPipelineAggregationBuilder("mov_fn", "avg", script, 3));
-        TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance();
-
-        final MovFnPipelineAggregationBuilder builder = new MovFnPipelineAggregationBuilder("name", "invalid_agg>metric", script, 1);
-        IllegalStateException ex = expectThrows(IllegalStateException.class,
-                () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders));
-        assertEquals("moving_fn aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent",
-                ex.getMessage());
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java
index cb1e2d5249b4a..17fbac3495c56 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java
@@ -19,17 +19,23 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 
 import java.io.IOException;
+import java.util.Collections;
 
-public class MovFnPipelineAggregationBuilderSerializationTests extends AbstractSerializingTestCase<MovFnPipelineAggregationBuilder> {
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
+public class MovFnPipelineAggregationBuilderSerializationTests extends BasePipelineAggregationTestCase<MovFnPipelineAggregationBuilder> {
     @Override
-    protected MovFnPipelineAggregationBuilder createTestInstance() {
+    protected MovFnPipelineAggregationBuilder createTestAggregatorFactory() {
         MovFnPipelineAggregationBuilder builder = new MovFnPipelineAggregationBuilder(
             randomAlphaOfLength(10),
             "foo",
@@ -40,19 +46,27 @@ protected MovFnPipelineAggregationBuilder createTestInstance() {
         return builder;
     }
 
-    @Override
-    protected Writeable.Reader<MovFnPipelineAggregationBuilder> instanceReader() {
-        return MovFnPipelineAggregationBuilder::new;
+    public void testValidParent() throws IOException {
+        Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, "painless", "test", emptyMap());
+        assertThat(validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(),
+                new MovFnPipelineAggregationBuilder("mov_fn", "avg", script, 3)), nullValue());
     }
 
-    @Override
-    protected MovFnPipelineAggregationBuilder doParseInstance(XContentParser parser) throws IOException {
-        return MovFnPipelineAggregationBuilder.parse(parser);
+    public void testInvalidParent() throws IOException {
+        Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, "painless", "test", Collections.emptyMap());
+        AggregationBuilder parent = mock(AggregationBuilder.class);
+        when(parent.getName()).thenReturn("name");
+
+        assertThat(validate(parent, new MovFnPipelineAggregationBuilder("name", "invalid_agg>metric", script, 1)), equalTo(
+                "Validation Failed: 1: moving_fn aggregation [name] must have a histogram, date_histogram"
+                + " or auto_date_histogram as parent;"));
     }
 
-    @Override
-    protected boolean supportsUnknownFields() {
-        return false;
+    public void testNoParent() throws IOException {
+        Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, "painless", "test", Collections.emptyMap());
+                assertThat(validate(emptyList(), new MovFnPipelineAggregationBuilder("name", "invalid_agg>metric", script, 1)), equalTo(
+                "Validation Failed: 1: moving_fn aggregation [name] must have a histogram, date_histogram"
+                + " or auto_date_histogram as parent but doesn't have a parent;"));
     }
 }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java
index 165312a5bdef5..1918d26e1e1af 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketTests.java
@@ -26,11 +26,11 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class PercentilesBucketTests extends AbstractBucketMetricsTestCase<PercentilesBucketPipelineAggregationBuilder> {
 
@@ -72,23 +72,17 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final PercentilesBucketPipelineAggregationBuilder builder = new PercentilesBucketPipelineAggregationBuilder("name",
-                "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new PercentilesBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        PercentilesBucketPipelineAggregationBuilder builder2 = new PercentilesBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new PercentilesBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        PercentilesBucketPipelineAggregationBuilder builder3 = new PercentilesBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new PercentilesBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
index 9a2d5a411d4ad..2d67735468984 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
@@ -20,29 +20,20 @@
 package org.elasticsearch.search.aggregations.pipeline;
 
 
-import org.elasticsearch.common.Rounding;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalOrder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregatorFactory;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorFactory;
-import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
-import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
-
-import static org.mockito.Mockito.mock;
+import java.util.function.Function;
 
 /**
  * Provides helper methods and classes for use in PipelineAggregation tests,
@@ -155,28 +146,12 @@ public static double calculateMetric(double[] values, ValuesSourceAggregationBui
         return 0.0;
     }
 
-    static AggregatorFactory getRandomSequentiallyOrderedParentAgg() throws IOException {
-        AggregatorFactory factory = null;
-        switch (randomIntBetween(0, 2)) {
-            case 0:
-                factory = new HistogramAggregatorFactory("name", mock(ValuesSourceConfig.class), 0.0d, 0.0d,
-                    mock(InternalOrder.class), false, 0L, 0.0d, 1.0d, mock(QueryShardContext.class), null,
-                    new AggregatorFactories.Builder(), Collections.emptyMap());
-                break;
-            case 1:
-                factory = new DateHistogramAggregatorFactory("name", mock(ValuesSourceConfig.class),
-                    mock(InternalOrder.class), false, 0L, mock(Rounding.class), mock(Rounding.class),
-                    mock(ExtendedBounds.class), mock(QueryShardContext.class), mock(AggregatorFactory.class),
-                    new AggregatorFactories.Builder(), Collections.emptyMap());
-                break;
-            case 2:
-            default:
-                AutoDateHistogramAggregationBuilder.RoundingInfo[] roundings = new AutoDateHistogramAggregationBuilder.RoundingInfo[1];
-                factory = new AutoDateHistogramAggregatorFactory("name", mock(ValuesSourceConfig.class),
-                    1, roundings,
-                    mock(QueryShardContext.class), null, new AggregatorFactories.Builder(), Collections.emptyMap());
-        }
-
-        return factory;
+    static AggregationBuilder getRandomSequentiallyOrderedParentAgg() throws IOException {
+        @SuppressWarnings("unchecked")
+        Function<String, AggregationBuilder> builder = randomFrom(
+                HistogramAggregationBuilder::new,
+                DateHistogramAggregationBuilder::new,
+                AutoDateHistogramAggregationBuilder::new);
+        return builder.apply("name");
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
index ef1d13990a308..c5b09a7a5726c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
@@ -19,16 +19,21 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.TestAggregatorFactory;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 public class SerialDifferenceTests extends BasePipelineAggregationTestCase<SerialDiffPipelineAggregationBuilder> {
 
     @Override
@@ -47,32 +52,28 @@ protected SerialDiffPipelineAggregationBuilder createTestAggregatorFactory() {
         }
         return factory;
     }
-    
+
     /**
      * The validation should verify the parent aggregation is allowed.
      */
     public void testValidate() throws IOException {
-        final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        aggBuilders.add(createTestAggregatorFactory());
-
-        final SerialDiffPipelineAggregationBuilder builder = new SerialDiffPipelineAggregationBuilder("name", "valid");
-        builder.validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders);
+        assertThat(validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(),
+                new SerialDiffPipelineAggregationBuilder("name", "valid")), nullValue());
     }
 
-    /**
-     * The validation should throw an IllegalArgumentException, since parent
-     * aggregation is not a type of HistogramAggregatorFactory,
-     * DateHistogramAggregatorFactory or AutoDateHistogramAggregatorFactory.
-     */
-    public void testValidateException() throws IOException {
+    public void testInvalidParent() throws IOException {
         final Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
         aggBuilders.add(createTestAggregatorFactory());
-        TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance();
+        AggregationBuilder parent = mock(AggregationBuilder.class);
+        when(parent.getName()).thenReturn("name");
+        assertThat(validate(parent, new SerialDiffPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: serial_diff aggregation [name] must have a histogram, "
+                + "date_histogram or auto_date_histogram as parent;"));
+    }
 
-        final SerialDiffPipelineAggregationBuilder builder = new SerialDiffPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalStateException ex = expectThrows(IllegalStateException.class,
-                () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders));
-        assertEquals("serial_diff aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent",
-                ex.getMessage());
+    public void testNoParent() {
+        assertThat(validate(emptyList(), new SerialDiffPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: serial_diff aggregation [name] must have a histogram, "
+                + "date_histogram or auto_date_histogram as parent but doesn't have a parent;"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java
index bf2ef7615df66..aac81c19be6bd 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
 public class StatsBucketTests extends AbstractBucketMetricsTestCase<StatsBucketPipelineAggregationBuilder> {
 
     @Override
@@ -44,23 +46,18 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final StatsBucketPipelineAggregationBuilder builder = new StatsBucketPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new StatsBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        StatsBucketPipelineAggregationBuilder builder2 = new StatsBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new StatsBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        StatsBucketPipelineAggregationBuilder builder3 = new StatsBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new StatsBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java
index fdba878524146..0bcbf592458aa 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SumBucketTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
 public class SumBucketTests extends AbstractBucketMetricsTestCase<SumBucketPipelineAggregationBuilder> {
 
     @Override
@@ -43,23 +45,18 @@ public void testValidate() {
         aggBuilders.add(multiBucketAgg);
 
         // First try to point to a non-existent agg
-        final SumBucketPipelineAggregationBuilder builder = new SumBucketPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> builder.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
-                + " aggregation does not exist for aggregation [name]: invalid_agg>metric", ex.getMessage());
+        assertThat(validate(aggBuilders, new SumBucketPipelineAggregationBuilder("name", "invalid_agg>metric")), equalTo(
+                "Validation Failed: 1: " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+                + " aggregation does not exist for aggregation [name]: invalid_agg>metric;"));
 
         // Now try to point to a single bucket agg
-        SumBucketPipelineAggregationBuilder builder2 = new SumBucketPipelineAggregationBuilder("name", "global>metric");
-        ex = expectThrows(IllegalArgumentException.class, () -> builder2.validate(null, aggBuilders, Collections.emptySet()));
-        assertEquals("The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
+        assertThat(validate(aggBuilders, new SumBucketPipelineAggregationBuilder("name", "global>metric")), equalTo(
+                "Validation Failed: 1: The first aggregation in " + PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                 + " must be a multi-bucket aggregation for aggregation [name] found :" + GlobalAggregationBuilder.class.getName()
-                + " for buckets path: global>metric", ex.getMessage());
+                + " for buckets path: global>metric;"));
 
-        // Now try to point to a valid multi-bucket agg (no exception should be
-        // thrown)
-        SumBucketPipelineAggregationBuilder builder3 = new SumBucketPipelineAggregationBuilder("name", "terms>metric");
-        builder3.validate(null, aggBuilders, Collections.emptySet());
+        // Now try to point to a valid multi-bucket agg
+        assertThat(validate(aggBuilders, new SumBucketPipelineAggregationBuilder("name", "terms>metric")), nullValue());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
index 61a75a0716fd2..b2c3a4e85455b 100644
--- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
+++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
@@ -240,7 +240,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
         DocumentSubsetDirectoryReader filteredReader = new DocumentSubsetDirectoryReader(reader, cache, roleQuery);
 
         ContextIndexSearcher searcher = new ContextIndexSearcher(filteredReader, IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
 
         // Assert wrapping
         assertEquals(ExitableDirectoryReader.class, searcher.getIndexReader().getClass());
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java
index a9759c6298e83..d5b699ef226be 100644
--- a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java
@@ -59,25 +59,15 @@ public void testSerialization() throws Exception {
         ShardSearchRequest shardSearchTransportRequest = createShardSearchRequest();
         ShardSearchRequest deserializedRequest =
             copyWriteable(shardSearchTransportRequest, namedWriteableRegistry, ShardSearchRequest::new);
-        assertEquals(deserializedRequest.scroll(), shardSearchTransportRequest.scroll());
-        assertEquals(deserializedRequest.getAliasFilter(), shardSearchTransportRequest.getAliasFilter());
-        assertArrayEquals(deserializedRequest.indices(), shardSearchTransportRequest.indices());
-        assertEquals(deserializedRequest.indicesOptions(), shardSearchTransportRequest.indicesOptions());
-        assertEquals(deserializedRequest.nowInMillis(), shardSearchTransportRequest.nowInMillis());
-        assertEquals(deserializedRequest.source(), shardSearchTransportRequest.source());
-        assertEquals(deserializedRequest.searchType(), shardSearchTransportRequest.searchType());
-        assertEquals(deserializedRequest.shardId(), shardSearchTransportRequest.shardId());
-        assertEquals(deserializedRequest.numberOfShards(), shardSearchTransportRequest.numberOfShards());
-        assertArrayEquals(deserializedRequest.indexRoutings(), shardSearchTransportRequest.indexRoutings());
-        assertEquals(deserializedRequest.preference(), shardSearchTransportRequest.preference());
-        assertEquals(deserializedRequest.cacheKey(), shardSearchTransportRequest.cacheKey());
-        assertNotSame(deserializedRequest, shardSearchTransportRequest);
-        assertEquals(deserializedRequest.getAliasFilter(), shardSearchTransportRequest.getAliasFilter());
-        assertEquals(deserializedRequest.indexBoost(), shardSearchTransportRequest.indexBoost(), 0.0f);
-        assertEquals(deserializedRequest.getClusterAlias(), shardSearchTransportRequest.getClusterAlias());
-        assertEquals(shardSearchTransportRequest.allowPartialSearchResults(), deserializedRequest.allowPartialSearchResults());
-        assertEquals(deserializedRequest.canReturnNullResponseIfMatchNoDocs(),
-            shardSearchTransportRequest.canReturnNullResponseIfMatchNoDocs());
+        assertEquals(shardSearchTransportRequest, deserializedRequest);
+    }
+
+    public void testClone() throws Exception {
+        for (int i = 0; i < 10; i++) {
+            ShardSearchRequest shardSearchTransportRequest = createShardSearchRequest();
+            ShardSearchRequest clone = new ShardSearchRequest(shardSearchTransportRequest);
+            assertEquals(shardSearchTransportRequest, clone);
+        }
     }
 
     private ShardSearchRequest createShardSearchRequest() throws IOException {
@@ -147,6 +137,28 @@ public void testUnknownAliasFilter() throws Exception {
         expectThrows(InvalidAliasNameException.class, () -> aliasFilter(finalIndexMetadata, "unknown"));
     }
 
+    private static void assertEquals(ShardSearchRequest orig, ShardSearchRequest copy) throws IOException {
+        assertEquals(orig.scroll(), copy.scroll());
+        assertEquals(orig.getAliasFilter(), copy.getAliasFilter());
+        assertArrayEquals(orig.indices(), copy.indices());
+        assertEquals(orig.indicesOptions(), copy.indicesOptions());
+        assertEquals(orig.nowInMillis(), copy.nowInMillis());
+        assertEquals(orig.source(), copy.source());
+        assertEquals(orig.searchType(), copy.searchType());
+        assertEquals(orig.shardId(), copy.shardId());
+        assertEquals(orig.numberOfShards(), copy.numberOfShards());
+        assertArrayEquals(orig.indexRoutings(), copy.indexRoutings());
+        assertEquals(orig.preference(), copy.preference());
+        assertEquals(orig.cacheKey(), copy.cacheKey());
+        assertNotSame(orig, copy);
+        assertEquals(orig.getAliasFilter(), copy.getAliasFilter());
+        assertEquals(orig.indexBoost(), copy.indexBoost(), 0.0f);
+        assertEquals(orig.getClusterAlias(), copy.getClusterAlias());
+        assertEquals(orig.allowPartialSearchResults(), copy.allowPartialSearchResults());
+        assertEquals(orig.canReturnNullResponseIfMatchNoDocs(),
+            orig.canReturnNullResponseIfMatchNoDocs());
+    }
+
     public static CompressedXContent filter(QueryBuilder filterBuilder) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         filterBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
index f05684930a9c5..96b2dc57a86f5 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
@@ -43,8 +43,8 @@
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.profile.ProfileResult;
 import org.elasticsearch.test.ESTestCase;
@@ -83,7 +83,7 @@ public static void setup() throws IOException {
         reader = w.getReader();
         w.close();
         searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(), ALWAYS_CACHE_POLICY);
+            IndexSearcher.getDefaultQueryCache(), ALWAYS_CACHE_POLICY, true);
     }
 
     @After
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
index fa9ac428177e2..3901a4ba7877e 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
@@ -910,12 +910,12 @@ public boolean lowLevelCancellation() {
 
     private static ContextIndexSearcher newContextSearcher(IndexReader reader) throws IOException {
         return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
     }
 
     private static ContextIndexSearcher newEarlyTerminationContextSearcher(IndexReader reader, int size) throws IOException {
         return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()) {
+            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true) {
 
             @Override
             public void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
@@ -928,7 +928,7 @@ public void search(List<LeafReaderContext> leaves, Weight weight, Collector coll
     // used to check that numeric long or date sort optimization was run
     private static ContextIndexSearcher newOptimizedContextSearcher(IndexReader reader, int queryType) throws IOException {
         return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
-            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()) {
+            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true) {
 
             @Override
             public void search(List<LeafReaderContext> leaves, Weight weight, CollectorManager manager,
diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
index a0c3fcd89afb7..3fbe1c5b8a4b4 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
@@ -35,9 +35,7 @@
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.Aggregations;
-import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalAggregationsTests;
-import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
@@ -45,8 +43,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 
-import java.util.List;
-
 import static java.util.Collections.emptyList;
 
 public class QuerySearchResultTests extends ESTestCase {
@@ -62,7 +58,7 @@ private static QuerySearchResult createTestInstance() throws Exception {
         ShardId shardId = new ShardId("index", "uuid", randomInt());
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean());
         ShardSearchRequest shardSearchRequest = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shardId, 1,
-            new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, randomNonNegativeLong(), null, null);
+            new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, randomNonNegativeLong(), null, new String[0]);
         QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.base64UUID(), randomLong()),
             new SearchShardTarget("node", shardId, null, OriginalIndices.NONE), shardSearchRequest);
         if (randomBoolean()) {
@@ -93,19 +89,9 @@ public void testSerialization() throws Exception {
         assertEquals(querySearchResult.size(), deserialized.size());
         assertEquals(querySearchResult.hasAggs(), deserialized.hasAggs());
         if (deserialized.hasAggs()) {
-            Aggregations aggs = querySearchResult.consumeAggs();
-            Aggregations deserializedAggs = deserialized.consumeAggs();
+            Aggregations aggs = querySearchResult.consumeAggs().get();
+            Aggregations deserializedAggs = deserialized.consumeAggs().get();
             assertEquals(aggs.asList(), deserializedAggs.asList());
-            List<SiblingPipelineAggregator> pipelineAggs = ((InternalAggregations) aggs).getTopLevelPipelineAggregators();
-            List<SiblingPipelineAggregator> deserializedPipelineAggs =
-                ((InternalAggregations) deserializedAggs).getTopLevelPipelineAggregators();
-            assertEquals(pipelineAggs.size(), deserializedPipelineAggs.size());
-            for (int i = 0; i < pipelineAggs.size(); i++) {
-                SiblingPipelineAggregator pipelineAgg = pipelineAggs.get(i);
-                SiblingPipelineAggregator deserializedPipelineAgg = deserializedPipelineAggs.get(i);
-                assertArrayEquals(pipelineAgg.bucketsPaths(), deserializedPipelineAgg.bucketsPaths());
-                assertEquals(pipelineAgg.name(), deserializedPipelineAgg.name());
-            }
         }
         assertEquals(querySearchResult.terminatedEarly(), deserialized.terminatedEarly());
     }
diff --git a/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
index 6e217a97ddf42..e4003c74969a4 100644
--- a/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
+++ b/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
@@ -602,7 +602,7 @@ public void testInvalidScrollKeepAlive() throws IOException {
         IllegalArgumentException illegalArgumentException =
             (IllegalArgumentException) ExceptionsHelper.unwrap(exc, IllegalArgumentException.class);
         assertNotNull(illegalArgumentException);
-        assertThat(illegalArgumentException.getMessage(), containsString("Keep alive for scroll (2h) is too large"));
+        assertThat(illegalArgumentException.getMessage(), containsString("Keep alive for request (2h) is too large"));
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -619,7 +619,7 @@ public void testInvalidScrollKeepAlive() throws IOException {
         illegalArgumentException =
             (IllegalArgumentException) ExceptionsHelper.unwrap(exc, IllegalArgumentException.class);
         assertNotNull(illegalArgumentException);
-        assertThat(illegalArgumentException.getMessage(), containsString("Keep alive for scroll (3h) is too large"));
+        assertThat(illegalArgumentException.getMessage(), containsString("Keep alive for request (3h) is too large"));
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java b/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
new file mode 100644
index 0000000000000..2e4df00d16aa5
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.searchafter;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.search.ClearReaderAction;
+import org.elasticsearch.action.search.ClearReaderRequest;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.search.SearchContextMissingException;
+import org.elasticsearch.search.SearchService;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class SearchWithReaderContextIT extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
+            .put(SearchService.KEEPALIVE_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(randomIntBetween(100, 500)))
+            .build();
+    }
+
+    public void testBasic() {
+        createIndex("test");
+        int numDocs = randomIntBetween(10, 50);
+        for (int i = 0; i < numDocs; i++) {
+            String id = Integer.toString(i);
+            client().prepareIndex("test").setId(id).setSource("value", i).get();
+        }
+        refresh("test");
+        SearchResponse resp1 = client().prepareSearch("test").setReader(null, TimeValue.timeValueMinutes(2)).get();
+        assertHitCount(resp1, numDocs);
+        int deletedDocs = 0;
+        for (int i = 0; i < numDocs; i++) {
+            if (randomBoolean()) {
+                String id = Integer.toString(i);
+                client().prepareDelete("test", id).get();
+                deletedDocs++;
+            }
+        }
+        refresh("test");
+        if (randomBoolean()) {
+            SearchResponse resp2 = client().prepareSearch("test").setQuery(new MatchAllQueryBuilder()).get();
+            assertNoFailures(resp2);
+            assertHitCount(resp2, numDocs - deletedDocs);
+        }
+        try {
+            SearchResponse resp3 = client().prepareSearch().setQuery(new MatchAllQueryBuilder())
+                .setReader(resp1.getReaderId(), TimeValue.timeValueMinutes(2))
+                .get();
+            assertNoFailures(resp3);
+            assertHitCount(resp3, numDocs);
+        } finally {
+            clearReaderId(resp1.getReaderId());
+        }
+    }
+
+    public void testMultipleIndices() {
+        int numIndices = randomIntBetween(1, 5);
+        for (int i = 1; i <= numIndices; i++) {
+            createIndex("index-" + i);
+        }
+        int numDocs = randomIntBetween(10, 50);
+        for (int i = 0; i < numDocs; i++) {
+            String id = Integer.toString(i);
+            String index = "index-" + randomIntBetween(1, numIndices);
+            client().prepareIndex(index).setId(id).setSource("value", i).get();
+        }
+        refresh();
+        SearchResponse resp1 = client().prepareSearch("*").setReader(null, TimeValue.timeValueMinutes(2)).get();
+        assertNoFailures(resp1);
+        assertHitCount(resp1, numDocs);
+        int moreDocs = randomIntBetween(10, 50);
+        for (int i = 0; i < moreDocs; i++) {
+            String id = "more-" + i;
+            String index = "index-" + randomIntBetween(1, numIndices);
+            client().prepareIndex(index).setId(id).setSource("value", i).get();
+        }
+        refresh();
+        try {
+            SearchResponse resp2 = client().prepareSearch().get();
+            assertNoFailures(resp2);
+            assertHitCount(resp2, numDocs + moreDocs);
+
+            SearchResponse resp3 = client().prepareSearch().setReader(resp1.getReaderId(), null).get();
+            assertNoFailures(resp3);
+            assertHitCount(resp3, numDocs);
+        } finally {
+            clearReaderId(resp1.getReaderId());
+        }
+    }
+
+    public void testReaderIdNotFound() throws Exception {
+        createIndex("index");
+        int index1 = randomIntBetween(10, 50);
+        for (int i = 0; i < index1; i++) {
+            String id = Integer.toString(i);
+            client().prepareIndex("index").setId(id).setSource("value", i).get();
+        }
+        refresh();
+        SearchResponse resp1 = client().prepareSearch("index")
+            .setReader(null, TimeValue.timeValueSeconds(randomIntBetween(2, 5))).get();
+        assertNoFailures(resp1);
+        assertHitCount(resp1, index1);
+        if (rarely()) {
+            assertBusy(() -> {
+                final CommonStats stats = client().admin().indices().prepareStats().setSearch(true).get().getTotal();
+                assertThat(stats.search.getOpenContexts(), equalTo(0L));
+            }, 60, TimeUnit.SECONDS);
+        } else {
+            clearReaderId(resp1.getReaderId());
+        }
+        SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class,
+            () -> client().prepareSearch().setReader(resp1.getReaderId(), null).get());
+        for (ShardSearchFailure failure : e.shardFailures()) {
+            assertThat(ExceptionsHelper.unwrapCause(failure.getCause()), instanceOf(SearchContextMissingException.class));
+        }
+    }
+
+    public void testIndexNotFound() {
+        createIndex("index-1");
+        createIndex("index-2");
+
+        int index1 = randomIntBetween(10, 50);
+        for (int i = 0; i < index1; i++) {
+            String id = Integer.toString(i);
+            client().prepareIndex("index-1").setId(id).setSource("value", i).get();
+        }
+
+        int index2 = randomIntBetween(10, 50);
+        for (int i = 0; i < index2; i++) {
+            String id = Integer.toString(i);
+            client().prepareIndex("index-2").setId(id).setSource("value", i).get();
+        }
+        refresh();
+        SearchResponse resp1 = client().prepareSearch("index-*").setReader(null, TimeValue.timeValueMinutes(2)).get();
+        assertNoFailures(resp1);
+        assertHitCount(resp1, index1 + index2);
+        client().admin().indices().prepareDelete("index-1").get();
+        if (randomBoolean()) {
+            SearchResponse resp2 = client().prepareSearch("index-*").get();
+            assertNoFailures(resp2);
+            assertHitCount(resp2, index2);
+
+        }
+        expectThrows(IndexNotFoundException.class, () -> client().prepareSearch().setReader(resp1.getReaderId(), null).get());
+        clearReaderId(resp1.getReaderId());
+    }
+
+    private void clearReaderId(String readerId) {
+        client().execute(ClearReaderAction.INSTANCE, new ClearReaderRequest(readerId)).actionGet();
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
new file mode 100644
index 0000000000000..a2a9d123266f6
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.ExecutionException;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class BlobStoreIncrementalityIT extends AbstractSnapshotIntegTestCase {
+
+    public void testIncrementalBehaviorOnPrimaryFailover() throws InterruptedException, ExecutionException, IOException {
+        internalCluster().startMasterOnlyNode();
+        final String primaryNode = internalCluster().startDataOnlyNode();
+        final String indexName = "test-index";
+        createIndex(indexName, Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0).build());
+        ensureYellow(indexName);
+        final String newPrimary = internalCluster().startDataOnlyNode();
+        final Collection<String> toDelete = new ArrayList<>();
+        ensureGreen(indexName);
+
+        logger.info("--> adding some documents to test index");
+        for (int j = 0; j < randomIntBetween(1, 10); ++j) {
+            final BulkRequest bulkRequest = new BulkRequest();
+            for (int i = 0; i < scaledRandomIntBetween(1, 100); ++i) {
+                bulkRequest.add(new IndexRequest(indexName).source("foo" + j, "bar" + i));
+            }
+            final BulkResponse bulkResponse = client().bulk(bulkRequest).get();
+            for (BulkItemResponse item : bulkResponse.getItems()) {
+                if (randomBoolean()) {
+                    toDelete.add(item.getId());
+                }
+            }
+        }
+        refresh(indexName);
+
+        final long documentCountOriginal = getCountForIndex(indexName);
+
+        final String snapshot1 = "snap-1";
+        final String repo = "test-repo";
+        logger.info("--> creating repository");
+        assertThat(client().admin().cluster().preparePutRepository(repo)
+                .setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet().isAcknowledged(),
+            is(true));
+
+        logger.info("--> creating snapshot 1");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setIndices(indexName).setWaitForCompletion(true).get();
+
+        logger.info("--> Shutting down initial primary node [{}]", primaryNode);
+        stopNode(primaryNode);
+
+        ensureYellow(indexName);
+        final String snapshot2 = "snap-2";
+        logger.info("--> creating snapshot 2");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setIndices(indexName).setWaitForCompletion(true).get();
+
+        assertTwoIdenticalShardSnapshots(repo, indexName, snapshot1, snapshot2);
+
+        ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot1, "-copy-1");
+        assertCountInIndexThenDelete(indexName + "-copy-1", documentCountOriginal);
+
+        ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot2, "-copy-2");
+        assertCountInIndexThenDelete(indexName + "-copy-2", documentCountOriginal);
+
+        internalCluster().startDataOnlyNode();
+        ensureGreen(indexName);
+
+        logger.info("--> delete some documents from test index");
+        for (final String id : toDelete) {
+            assertThat(client().prepareDelete(indexName, id).get().getResult(), is(DocWriteResponse.Result.DELETED));
+        }
+
+        refresh(indexName);
+
+        final String snapshot3 = "snap-3";
+        logger.info("--> creating snapshot 3");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot3).setIndices(indexName).setWaitForCompletion(true).get();
+
+        logger.info("--> Shutting down new primary node [{}]", newPrimary);
+        stopNode(newPrimary);
+        ensureYellow(indexName);
+
+        final String snapshot4 = "snap-4";
+        logger.info("--> creating snapshot 4");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot4).setIndices(indexName).setWaitForCompletion(true).get();
+
+        assertTwoIdenticalShardSnapshots(repo, indexName, snapshot3, snapshot4);
+
+        final long countAfterDelete = documentCountOriginal - toDelete.size();
+        ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot3, "-copy-3");
+        assertCountInIndexThenDelete(indexName + "-copy-3", countAfterDelete);
+
+        ensureRestoreSingleShardSuccessfully(repo, indexName, snapshot4, "-copy-4");
+        assertCountInIndexThenDelete(indexName + "-copy-4", countAfterDelete);
+    }
+
+    public void testForceMergeCausesFullSnapshot() throws Exception {
+        internalCluster().startMasterOnlyNode();
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        final String indexName = "test-index";
+        createIndex(indexName, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build());
+        ensureGreen(indexName);
+
+        logger.info("--> adding some documents to test index and flush in between to get at least two segments");
+        for (int j = 0; j < 2; j++) {
+            final BulkRequest bulkRequest = new BulkRequest();
+            for (int i = 0; i < scaledRandomIntBetween(1, 100); ++i) {
+                bulkRequest.add(new IndexRequest(indexName).source("foo" + j, "bar" + i));
+            }
+            client().bulk(bulkRequest).get();
+            flushAndRefresh(indexName);
+        }
+        final IndexStats indexStats = client().admin().indices().prepareStats(indexName).get().getIndex(indexName);
+        assertThat(indexStats.getIndexShards().get(0).getPrimary().getSegments().getCount(), greaterThan(1L));
+
+        final String snapshot1 = "snap-1";
+        final String repo = "test-repo";
+        logger.info("--> creating repository");
+        assertThat(client().admin().cluster().preparePutRepository(repo)
+                .setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet().isAcknowledged(),
+            is(true));
+
+        logger.info("--> creating snapshot 1");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setIndices(indexName).setWaitForCompletion(true).get();
+
+        logger.info("--> force merging down to a single segment");
+        final ForceMergeResponse forceMergeResponse =
+            client().admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).setFlush(true).get();
+        assertThat(forceMergeResponse.getFailedShards(), is(0));
+
+        final String snapshot2 = "snap-2";
+        logger.info("--> creating snapshot 2");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setIndices(indexName).setWaitForCompletion(true).get();
+
+        logger.info("--> asserting that the two snapshots refer to different files in the repository");
+        final SnapshotStats secondSnapshotShardStatus =
+            getStats(repo, snapshot2).getIndices().get(indexName).getShards().get(0).getStats();
+        assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), greaterThan(0));
+    }
+
+    private void assertCountInIndexThenDelete(String index, long expectedCount) throws ExecutionException, InterruptedException {
+        logger.info("--> asserting that index [{}] contains [{}] documents", index, expectedCount);
+        assertThat(getCountForIndex(index), is(expectedCount));
+        logger.info("--> deleting index [{}]", index);
+        assertThat(client().admin().indices().prepareDelete(index).get().isAcknowledged(), is(true));
+    }
+
+    private void assertTwoIdenticalShardSnapshots(String repo, String indexName, String snapshot1, String snapshot2) {
+        logger.info(
+            "--> asserting that snapshots [{}] and [{}] are referring to the same files in the repository", snapshot1, snapshot2);
+        final SnapshotStats firstSnapshotShardStatus = getStats(repo, snapshot1).getIndices().get(indexName).getShards().get(0).getStats();
+        final int totalFilesInShard = firstSnapshotShardStatus.getTotalFileCount();
+        assertThat(totalFilesInShard, greaterThan(0));
+        final SnapshotStats secondSnapshotShardStatus =
+            getStats(repo, snapshot2).getIndices().get(indexName).getShards().get(0).getStats();
+        assertThat(secondSnapshotShardStatus.getTotalFileCount(), is(totalFilesInShard));
+        assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), is(0));
+    }
+
+    private SnapshotStatus getStats(String repository, String snapshot) {
+        return client().admin().cluster().prepareSnapshotStatus(repository).setSnapshots(snapshot).get().getSnapshots().get(0);
+    }
+
+    private void ensureRestoreSingleShardSuccessfully(String repo, String indexName, String snapshot, String indexSuffix) {
+        logger.info("--> restoring [{}]", snapshot);
+        final RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot(repo, snapshot)
+            .setIndices(indexName).setRenamePattern("(.+)").setRenameReplacement("$1" + indexSuffix).setWaitForCompletion(true).get();
+        final RestoreInfo restoreInfo = restoreSnapshotResponse.getRestoreInfo();
+        assertThat(restoreInfo.totalShards(), is(1));
+        assertThat(restoreInfo.failedShards(), is(0));
+    }
+
+    private long getCountForIndex(String indexName) throws ExecutionException, InterruptedException {
+        return client().search(new SearchRequest(new SearchRequest(indexName).source(
+            new SearchSourceBuilder().size(0).trackTotalHits(true)))).get().getHits().getTotalHits().value;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java
index 1f09afdc76908..62eeea8b24f34 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java
@@ -95,12 +95,12 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenera
 
                     @Override
                     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus,
-                                              Version repositoryMetaVersion, Map<String, Object> userMetadata,
-                                              ActionListener<String> listener) {
+                                              IndexCommit snapshotIndexCommit, String shardStateIdentifier,
+                                              IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
+                                              Map<String, Object> userMetadata, ActionListener<String> listener) {
                         assertThat(userMetadata, is(Collections.singletonMap(MOCK_FILTERED_META, initialMetaValue)));
-                        super.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, snapshotStatus,
-                            repositoryMetaVersion, userMetadata, listener);
+                        super.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, shardStateIdentifier,
+                            snapshotStatus, repositoryMetaVersion, userMetadata, listener);
                     }
 
                     @Override
diff --git a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
index 68aff95419ee3..01d7b54bfcc18 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
@@ -20,10 +20,12 @@
 package org.elasticsearch.threadpool;
 
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 
 import static org.elasticsearch.threadpool.ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING;
@@ -102,4 +104,35 @@ public void testAssertCurrentMethodIsNotCalledRecursively() {
             equalTo("org.elasticsearch.threadpool.ThreadPoolTests#factorialForked is called recursively"));
         terminate(threadPool);
     }
+
+    public void testInheritContextOnSchedule() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+        final CountDownLatch executed = new CountDownLatch(1);
+
+        TestThreadPool threadPool = new TestThreadPool("test");
+        try {
+            threadPool.getThreadContext().putHeader("foo", "bar");
+            final Integer one = Integer.valueOf(1);
+            threadPool.getThreadContext().putTransient("foo", one);
+            threadPool.schedule(() -> {
+                try {
+                    latch.await();
+                } catch (InterruptedException e) {
+                    fail();
+                }
+                assertEquals(threadPool.getThreadContext().getHeader("foo"), "bar");
+                assertSame(threadPool.getThreadContext().getTransient("foo"), one);
+                assertNull(threadPool.getThreadContext().getHeader("bar"));
+                assertNull(threadPool.getThreadContext().getTransient("bar"));
+                executed.countDown();
+            }, TimeValue.timeValueMillis(randomInt(100)), randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC));
+            threadPool.getThreadContext().putTransient("bar", "boom");
+            threadPool.getThreadContext().putHeader("bar", "boom");
+            latch.countDown();
+            executed.await();
+        } finally {
+            latch.countDown();
+            terminate(threadPool);
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/transport/CompressibleBytesOutputStreamTests.java b/server/src/test/java/org/elasticsearch/transport/CompressibleBytesOutputStreamTests.java
index 90f251bbc5610..aeb92dac73479 100644
--- a/server/src/test/java/org/elasticsearch/transport/CompressibleBytesOutputStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/CompressibleBytesOutputStreamTests.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -34,11 +33,7 @@ public class CompressibleBytesOutputStreamTests extends ESTestCase {
 
     public void testStreamWithoutCompression() throws IOException {
         BytesStream bStream = new ZeroOutOnCloseStream();
-        if (randomBoolean()) {
-            bStream.setVersion(VersionUtils.randomVersion(random()));
-        }
         CompressibleBytesOutputStream stream = new CompressibleBytesOutputStream(bStream, false);
-        assertEquals(bStream.getVersion(), stream.getVersion());
 
         byte[] expectedBytes = randomBytes(randomInt(30));
         stream.write(expectedBytes);
@@ -66,11 +61,7 @@ public void testStreamWithoutCompression() throws IOException {
 
     public void testStreamWithCompression() throws IOException {
         BytesStream bStream = new ZeroOutOnCloseStream();
-        if (randomBoolean()) {
-            bStream.setVersion(VersionUtils.randomVersion(random()));
-        }
         CompressibleBytesOutputStream stream = new CompressibleBytesOutputStream(bStream, true);
-        assertEquals(bStream.getVersion(), stream.getVersion());
 
         byte[] expectedBytes = randomBytes(randomInt(30));
         stream.write(expectedBytes);
@@ -97,11 +88,7 @@ public void testStreamWithCompression() throws IOException {
 
     public void testCompressionWithCallingMaterializeFails() throws IOException {
         BytesStream bStream = new ZeroOutOnCloseStream();
-        if (randomBoolean()) {
-            bStream.setVersion(VersionUtils.randomVersion(random()));
-        }
         CompressibleBytesOutputStream stream = new CompressibleBytesOutputStream(bStream, true);
-        assertEquals(bStream.getVersion(), stream.getVersion());
 
         byte[] expectedBytes = randomBytes(between(1, 30));
         stream.write(expectedBytes);
diff --git a/server/src/test/java/org/elasticsearch/transport/ProxyConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/ProxyConnectionStrategyTests.java
index ee9d98d07b3ec..924b66547ecd2 100644
--- a/server/src/test/java/org/elasticsearch/transport/ProxyConnectionStrategyTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/ProxyConnectionStrategyTests.java
@@ -268,7 +268,7 @@ public void testProxyStrategyWillResolveAddressesEachConnect() throws Exception
         }
     }
 
-    public void testProxyStrategyWillNeedToBeRebuiltIfNumOfSocketsOrAddressesChange() {
+    public void testProxyStrategyWillNeedToBeRebuiltIfNumOfSocketsOrAddressesOrServerNameChange() {
         try (MockTransportService remoteTransport = startTransport("node1", Version.CURRENT)) {
             TransportAddress remoteAddress = remoteTransport.boundAddress().publishAddress();
 
@@ -280,7 +280,7 @@ public void testProxyStrategyWillNeedToBeRebuiltIfNumOfSocketsOrAddressesChange(
                 int numOfConnections = randomIntBetween(4, 8);
                 try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager);
                      ProxyConnectionStrategy strategy = new ProxyConnectionStrategy(clusterAlias, localService, remoteConnectionManager,
-                         numOfConnections, remoteAddress.toString())) {
+                         numOfConnections, remoteAddress.toString(), "server-name")) {
                     PlainActionFuture<Void> connectFuture = PlainActionFuture.newFuture();
                     strategy.connect(connectFuture);
                     connectFuture.actionGet();
@@ -295,11 +295,14 @@ public void testProxyStrategyWillNeedToBeRebuiltIfNumOfSocketsOrAddressesChange(
                         .getConcreteSettingForNamespace("cluster-alias");
                     Setting<?> socketConnections = ProxyConnectionStrategy.REMOTE_SOCKET_CONNECTIONS
                         .getConcreteSettingForNamespace("cluster-alias");
+                    Setting<?> serverName = ProxyConnectionStrategy.SERVER_NAME
+                        .getConcreteSettingForNamespace("cluster-alias");
 
                     Settings noChange = Settings.builder()
                         .put(modeSetting.getKey(), "proxy")
                         .put(addressesSetting.getKey(), remoteAddress.toString())
                         .put(socketConnections.getKey(), numOfConnections)
+                        .put(serverName.getKey(), "server-name")
                         .build();
                     assertFalse(strategy.shouldRebuildConnection(noChange));
                     Settings addressesChanged = Settings.builder()
@@ -313,6 +316,13 @@ public void testProxyStrategyWillNeedToBeRebuiltIfNumOfSocketsOrAddressesChange(
                         .put(socketConnections.getKey(), numOfConnections + 1)
                         .build();
                     assertTrue(strategy.shouldRebuildConnection(socketsChanged));
+                    Settings serverNameChange = Settings.builder()
+                        .put(modeSetting.getKey(), "proxy")
+                        .put(addressesSetting.getKey(), remoteAddress.toString())
+                        .put(socketConnections.getKey(), numOfConnections)
+                        .put(serverName.getKey(), "server-name2")
+                        .build();
+                    assertTrue(strategy.shouldRebuildConnection(serverNameChange));
                 }
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java
index ed71b7f85c863..9bea51ddfef5a 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -34,6 +35,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.transport.RemoteClusterConnectionTests.startTransport;
+import static org.hamcrest.Matchers.equalTo;
 
 public class RemoteClusterClientTests extends ESTestCase {
     private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
@@ -51,7 +53,7 @@ public void testConnectAndExecuteRequest() throws Exception {
             DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode();
 
             Settings localSettings = Settings.builder()
-                .put(RemoteClusterService.ENABLE_REMOTE_CLUSTERS.getKey(), true)
+                .put(Node.NODE_REMOTE_CLUSTER_CLIENT.getKey(), true)
                 .put("cluster.remote.test.seeds",
                     remoteNode.getAddress().getAddress() + ":" + remoteNode.getAddress().getPort()).build();
             try (MockTransportService service = MockTransportService.createNewService(localSettings, Version.CURRENT, threadPool, null)) {
@@ -80,7 +82,7 @@ public void testEnsureWeReconnect() throws Exception {
             remoteSettings)) {
             DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode();
             Settings localSettings = Settings.builder()
-                .put(RemoteClusterService.ENABLE_REMOTE_CLUSTERS.getKey(), true)
+                .put(Node.NODE_REMOTE_CLUSTER_CLIENT.getKey(), true)
                 .put("cluster.remote.test.seeds",
                     remoteNode.getAddress().getAddress() + ":" + remoteNode.getAddress().getPort()).build();
             try (MockTransportService service = MockTransportService.createNewService(localSettings, Version.CURRENT, threadPool, null)) {
@@ -119,4 +121,17 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
             }
         }
     }
+
+    public void testRemoteClusterServiceNotEnabled() {
+        final Settings settings = Settings.builder().put(Node.NODE_REMOTE_CLUSTER_CLIENT.getKey(), false).build();
+        try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) {
+            service.start();
+            service.acceptIncomingRequests();
+            final RemoteClusterService remoteClusterService = service.getRemoteClusterService();
+            final IllegalArgumentException e =
+                expectThrows(IllegalArgumentException.class, () -> remoteClusterService.getRemoteClusterClient(threadPool, "test"));
+            assertThat(e.getMessage(), equalTo("this node does not have the remote_cluster_client role"));
+        }
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
index 088f14ef36c1a..836051dd69bb7 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
@@ -138,7 +138,7 @@ public static MockTransportService startTransport(
                     InternalSearchResponse response = new InternalSearchResponse(searchHits,
                         InternalAggregations.EMPTY, null, null, false, null, 1);
                     SearchResponse searchResponse = new SearchResponse(response, null, 1, 1, 0, 100, ShardSearchFailure.EMPTY_ARRAY,
-                        SearchResponse.Clusters.EMPTY);
+                        SearchResponse.Clusters.EMPTY, null);
                     channel.sendResponse(searchResponse);
                 });
             newService.registerRequestHandler(ClusterStateAction.NAME, ThreadPool.Names.SAME, ClusterStateRequest::new,
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
index 9b35f8367914c..633a668ecd2ae 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -42,6 +43,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -843,10 +845,34 @@ public void testSkipUnavailable() {
         }
     }
 
+    public void testRemoteClusterServiceNotEnabledGetRemoteClusterConnection() {
+        final Settings settings = Settings.builder().put(Node.NODE_REMOTE_CLUSTER_CLIENT.getKey(), false).build();
+        try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) {
+            service.start();
+            service.acceptIncomingRequests();
+            final IllegalArgumentException e =
+                expectThrows(IllegalArgumentException.class, () -> service.getRemoteClusterService().getRemoteClusterConnection("test"));
+            assertThat(e.getMessage(), equalTo("this node does not have the remote_cluster_client role"));
+        }
+    }
+
+    public void testRemoteClusterServiceNotEnabledGetCollectNodes() {
+        final Settings settings = Settings.builder().put(Node.NODE_REMOTE_CLUSTER_CLIENT.getKey(), false).build();
+        try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) {
+            service.start();
+            service.acceptIncomingRequests();
+            final IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> service.getRemoteClusterService().collectNodes(Set.of(), ActionListener.wrap(r -> {}, r -> {})));
+            assertThat(e.getMessage(), equalTo("this node does not have the remote_cluster_client role"));
+        }
+    }
+
     private static Settings createSettings(String clusterAlias, List<String> seeds) {
         Settings.Builder builder = Settings.builder();
         builder.put(SniffConnectionStrategy.REMOTE_CLUSTER_SEEDS.getConcreteSettingForNamespace(clusterAlias).getKey(),
             Strings.collectionToCommaDelimitedString(seeds));
         return builder.build();
     }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterSettingsTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterSettingsTests.java
index 0f2749ffacf6c..0ff382fbd2f53 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterSettingsTests.java
@@ -25,13 +25,13 @@
 
 import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.transport.SniffConnectionStrategy.REMOTE_CLUSTERS_PROXY;
-import static org.elasticsearch.transport.RemoteClusterService.ENABLE_REMOTE_CLUSTERS;
+import static org.elasticsearch.node.Node.NODE_REMOTE_CLUSTER_CLIENT;
 import static org.elasticsearch.transport.RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE;
-import static org.elasticsearch.transport.SniffConnectionStrategy.REMOTE_CLUSTER_SEEDS;
-import static org.elasticsearch.transport.SniffConnectionStrategy.REMOTE_CONNECTIONS_PER_CLUSTER;
 import static org.elasticsearch.transport.RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING;
 import static org.elasticsearch.transport.RemoteClusterService.REMOTE_NODE_ATTRIBUTE;
+import static org.elasticsearch.transport.SniffConnectionStrategy.REMOTE_CLUSTERS_PROXY;
+import static org.elasticsearch.transport.SniffConnectionStrategy.REMOTE_CLUSTER_SEEDS;
+import static org.elasticsearch.transport.SniffConnectionStrategy.REMOTE_CONNECTIONS_PER_CLUSTER;
 import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -49,8 +49,12 @@ public void testRemoteNodeAttributeDefault() {
         assertThat(REMOTE_NODE_ATTRIBUTE.get(Settings.EMPTY), equalTo(""));
     }
 
-    public void testEnableRemoteClustersDefault() {
-        assertTrue(ENABLE_REMOTE_CLUSTERS.get(Settings.EMPTY));
+    public void testRemoteClusterClientDefault() {
+        assertTrue(NODE_REMOTE_CLUSTER_CLIENT.get(Settings.EMPTY));
+    }
+
+    public void testDisableRemoteClusterClient() {
+        assertFalse(NODE_REMOTE_CLUSTER_CLIENT.get(Settings.builder().put(NODE_REMOTE_CLUSTER_CLIENT.getKey(), false).build()));
     }
 
     public void testSkipUnavailableDefault() {
diff --git a/settings.gradle b/settings.gradle
index 133c67b205cca..6e925b4c4b882 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id "com.gradle.enterprise" version "3.1.1"
+  id "com.gradle.enterprise" version "3.2"
 }
 
 String dirName = rootProject.projectDir.name
diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java
index 5498c4ebc033f..1def1439429e6 100644
--- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java
+++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java
@@ -30,7 +30,7 @@ public class AzureHttpFixture {
 
     private AzureHttpFixture(final String address, final int port, final String container) throws IOException {
         this.server = HttpServer.create(new InetSocketAddress(InetAddress.getByName(address), port), 0);
-        server.createContext("/", new AzureHttpHandler(container));
+        server.createContext("/" + container, new AzureHttpHandler(container));
     }
 
     private void start() throws Exception {
diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
index 6cd93fc9347fa..7a94a8c9f2e57 100644
--- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
+++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
@@ -22,21 +22,16 @@
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.RestUtils;
 
-import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -72,21 +67,18 @@ public void handle(final HttpExchange exchange) throws IOException {
             assert read == -1 : "Request body should have been empty but saw [" + read + "]";
         }
         try {
-            // Request body is closed in the finally block
-            final BytesReference requestBody = Streams.readFully(Streams.noCloseStream(exchange.getRequestBody()));
             if (Regex.simpleMatch("PUT /" + container + "/*blockid=*", request)) {
                 // Put Block (https://docs.microsoft.com/en-us/rest/api/storageservices/put-block)
                 final Map<String, String> params = new HashMap<>();
                 RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params);
 
                 final String blockId = params.get("blockid");
-                blobs.put(blockId, requestBody);
+                blobs.put(blockId, Streams.readFully(exchange.getRequestBody()));
                 exchange.sendResponseHeaders(RestStatus.CREATED.getStatus(), -1);
 
             } else if (Regex.simpleMatch("PUT /" + container + "/*comp=blocklist*", request)) {
                 // Put Block List (https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list)
-                final String blockList =
-                    Streams.copyToString(new InputStreamReader(requestBody.streamInput(), StandardCharsets.UTF_8));
+                final String blockList = Streams.copyToString(new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8));
                 final List<String> blockIds = Arrays.stream(blockList.split("<Latest>"))
                     .filter(line -> line.contains("</Latest>"))
                     .map(line -> line.substring(0, line.indexOf("</Latest>")))
@@ -105,12 +97,12 @@ public void handle(final HttpExchange exchange) throws IOException {
                 // PUT Blob (see https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob)
                 final String ifNoneMatch = exchange.getRequestHeaders().getFirst("If-None-Match");
                 if ("*".equals(ifNoneMatch)) {
-                    if (blobs.putIfAbsent(exchange.getRequestURI().getPath(), requestBody) != null) {
+                    if (blobs.putIfAbsent(exchange.getRequestURI().getPath(), Streams.readFully(exchange.getRequestBody())) != null) {
                         sendError(exchange, RestStatus.CONFLICT);
                         return;
                     }
                 } else {
-                    blobs.put(exchange.getRequestURI().getPath(), requestBody);
+                    blobs.put(exchange.getRequestURI().getPath(), Streams.readFully(exchange.getRequestBody()));
                 }
                 exchange.sendResponseHeaders(RestStatus.CREATED.getStatus(), -1);
 
@@ -198,45 +190,6 @@ public void handle(final HttpExchange exchange) throws IOException {
                 exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
                 exchange.getResponseBody().write(response);
 
-            } else if (Regex.simpleMatch("POST /?comp=batch", request)) {
-                // Batch Delete (https://docs.microsoft.com/en-us/rest/api/storageservices/blob-batch)
-                try (BufferedReader reader = new BufferedReader(new InputStreamReader(requestBody.streamInput()))) {
-                    final Set<String> toDelete = reader.lines().filter(l -> l.startsWith("DELETE"))
-                        .map(l -> l.split(" ")[1]).collect(Collectors.toSet());
-                    final BytesStreamOutput baos = new BytesStreamOutput();
-                    final String batchSeparator = "batchresponse_" + UUIDs.randomBase64UUID();
-                    try (Writer writer = new OutputStreamWriter(baos)) {
-                        int contentId = 0;
-                        for (String b : toDelete) {
-                            writer.write("\r\n--" + batchSeparator + "\r\n" +
-                                "Content-Type: application/http \r\n" +
-                                "Content-ID: " + contentId++ + " \r\n");
-                            if (blobs.remove(b) == null) {
-                                writer.write("\r\nHTTP/1.1 404 The specified blob does not exist. \r\n" +
-                                    "x-ms-error-code: BlobNotFound \r\n" +
-                                    "x-ms-request-id: " + UUIDs.randomBase64UUID() + " \r\n" +
-                                    "x-ms-version: 2018-11-09\r\n" +
-                                    "Content-Length: 216 \r\n" +
-                                    "Content-Type: application/xml\r\n\r\n" +
-                                    "<?xml version=\"1.0\" encoding=\"utf-8\"?> \r\n" +
-                                    "<Error><Code>BlobNotFound</Code><Message>The specified blob does not exist.\r\n" +
-                                    "RequestId:" + UUIDs.randomBase64UUID() + "\r\n" +
-                                    "Time:2020-01-01T01:01:01.0000000Z</Message></Error>\r\n");
-                            } else {
-                                writer.write(
-                                    "\r\nHTTP/1.1 202 Accepted \r\n" +
-                                        "x-ms-request-id: " + UUIDs.randomBase64UUID() + " \r\n" +
-                                        "x-ms-version: 2018-11-09\r\n\r\n");
-                            }
-                        }
-                        writer.write("--" + batchSeparator + "--");
-                    }
-                    final Headers headers = exchange.getResponseHeaders();
-                    headers.add("Content-Type",
-                        "multipart/mixed; boundary=" + batchSeparator);
-                    exchange.sendResponseHeaders(RestStatus.ACCEPTED.getStatus(), baos.size());
-                    baos.bytes().writeTo(exchange.getResponseBody());
-                }
             } else {
                 sendError(exchange, RestStatus.BAD_REQUEST);
             }
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index 5dab9d99915b5..8aa0caad8ab1c 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -20,7 +20,7 @@
 apply plugin: 'elasticsearch.build'
 
 dependencies {
-  compile "org.apache.hadoop:hadoop-minicluster:2.8.1"
+  compile "org.apache.hadoop:hadoop-minicluster:2.8.5"
 }
 
 test.enabled = false
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/DeterministicTaskQueue.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/DeterministicTaskQueue.java
index db3818832f6e4..2c2970c71ed83 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/DeterministicTaskQueue.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/DeterministicTaskQueue.java
@@ -381,11 +381,6 @@ public Cancellable scheduleWithFixedDelay(Runnable command, TimeValue interval,
                 return super.scheduleWithFixedDelay(command, interval, executor);
             }
 
-            @Override
-            public Runnable preserveContext(Runnable command) {
-                return command;
-            }
-
             @Override
             public void shutdown() {
                 throw new UnsupportedOperationException();
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index e73c145c3b5b8..b95281530d844 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -838,7 +838,8 @@ protected String snapshotShard(final IndexShard shard,
         final String shardGen;
         try (Engine.IndexCommitRef indexCommitRef = shard.acquireLastIndexCommit(true)) {
             repository.snapshotShard(shard.store(), shard.mapperService(), snapshot.getSnapshotId(), indexId,
-                indexCommitRef.getIndexCommit(), snapshotStatus, Version.CURRENT, Collections.emptyMap(), future);
+                indexCommitRef.getIndexCommit(), null, snapshotStatus, Version.CURRENT,
+                Collections.emptyMap(), future);
             shardGen = future.actionGet();
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
index 4fae0fc14952a..9d697d04dc4ed 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
@@ -133,8 +133,8 @@ public boolean isReadOnly() {
 
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                              Map<String, Object> userMetadata, ActionListener<String> listener) {
+                              IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                              Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index 0709b4c8da5b2..f83bc9e6ed36f 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -18,12 +18,15 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
 import org.apache.lucene.document.HalfFloatPoint;
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.AssertingDirectoryReader;
 import org.apache.lucene.index.CompositeReaderContext;
 import org.apache.lucene.index.DirectoryReader;
@@ -49,6 +52,7 @@
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.util.BigArrays;
@@ -56,6 +60,10 @@
 import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache.Listener;
 import org.elasticsearch.index.cache.query.DisabledQueryCache;
@@ -108,6 +116,7 @@
 import org.junit.After;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -119,7 +128,6 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static java.util.Collections.singleton;
 import static org.elasticsearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS;
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Matchers.anyString;
@@ -142,7 +150,6 @@ public abstract class AggregatorTestCase extends ESTestCase {
         ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects
         GeoShapeFieldMapper.CONTENT_TYPE, // Cannot aggregate geoshapes (yet)
 
-        TextFieldMapper.CONTENT_TYPE, // TODO Does not support doc values, but does support FD, needs a lot of mocking
         ObjectMapper.NESTED_CONTENT_TYPE, // TODO support for nested
         CompletionFieldMapper.CONTENT_TYPE, // TODO support completion
         FieldAliasMapper.CONTENT_TYPE // TODO support alias
@@ -246,7 +253,7 @@ public boolean shouldCache(Query query) {
             }
         };
         ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(),
-            indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, null);
+            indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, false);
 
         SearchContext searchContext = mock(SearchContext.class);
         when(searchContext.numberOfShards()).thenReturn(1);
@@ -484,7 +491,7 @@ protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduc
                 int r = randomIntBetween(1, toReduceSize);
                 List<InternalAggregation> toReduce = aggs.subList(0, r);
                 InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction(
-                        root.context().bigArrays(), getMockScriptService());
+                        root.context().bigArrays(), getMockScriptService(), () -> PipelineAggregator.PipelineTree.EMPTY);
                 A reduced = (A) aggs.get(0).reduce(toReduce, context);
                 aggs = new ArrayList<>(aggs.subList(r, toReduceSize));
                 aggs.add(reduced);
@@ -598,6 +605,18 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy
             "createAggBuilderForTypeTest() must be implemented as well.");
     }
 
+    /**
+     * A method that allows implementors to specifically blacklist particular field types (based on their content_name).
+     * This is needed in some areas where the ValuesSourceType is not granular enough, for example integer values
+     * vs floating points, or `keyword` bytes vs `binary` bytes (which are not searchable)
+     *
+     * This is a blacklist instead of a whitelist because there are vastly more field types than ValuesSourceTypes,
+     * and it's expected that these unsupported cases are exceptional rather than common
+     */
+    protected List<String> unsupportedMappedFieldTypes() {
+        return Collections.emptyList();
+    }
+
     /**
      * This test will validate that an aggregator succeeds or fails to run against all the field types
      * that are registered in {@link IndicesModule} (e.g. all the core field types).  An aggregator
@@ -612,6 +631,7 @@ public final void testSupportedFieldTypes() throws IOException {
         Settings settings = Settings.builder().put("index.version.created", Version.CURRENT.id).build();
         String fieldName = "typeTestFieldName";
         List<ValuesSourceType> supportedVSTypes = getSupportedValuesSourceTypes();
+        List<String> unsupportedMappedFieldTypes = unsupportedMappedFieldTypes();
 
         if (supportedVSTypes.isEmpty()) {
             // If the test says it doesn't support any VStypes, it has not been converted yet so skip
@@ -630,7 +650,11 @@ public final void testSupportedFieldTypes() throws IOException {
 
             Map<String, Object> source = new HashMap<>();
             source.put("type", mappedType.getKey());
-            source.put("doc_values", "true");
+
+            // Text is the only field that doesn't support DVs, instead FD
+            if (mappedType.getKey().equals(TextFieldMapper.CONTENT_TYPE) == false) {
+                source.put("doc_values", "true");
+            }
 
             Mapper.Builder builder = mappedType.getValue().parse(fieldName, source, new MockParserContext());
             FieldMapper mapper = (FieldMapper) builder.build(new BuilderContext(settings, new ContentPath()));
@@ -651,17 +675,19 @@ public final void testSupportedFieldTypes() throws IOException {
                     IndexSearcher indexSearcher = newIndexSearcher(indexReader);
                     AggregationBuilder aggregationBuilder = createAggBuilderForTypeTest(fieldType, fieldName);
 
+                    ValuesSourceType vst = fieldType.getValuesSourceType();
                     // TODO in the future we can make this more explicit with expectThrows(), when the exceptions are standardized
                     try {
                         searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType);
-                        if (supportedVSTypes.contains(fieldType.getValuesSourceType()) == false) {
+                        if (supportedVSTypes.contains(vst) == false || unsupportedMappedFieldTypes.contains(fieldType.typeName())) {
                             fail("Aggregator [" + aggregationBuilder.getType() + "] should not support field type ["
-                                + fieldType.typeName() + "] but executing against the field did not throw an excetion");
+                                + fieldType.typeName() + "] but executing against the field did not throw an exception");
                         }
-                    } catch (Exception e) {
-                        if (supportedVSTypes.contains(fieldType.getValuesSourceType())) {
-                            fail("Aggregator [" + aggregationBuilder.getType() + "] supports field type ["
-                                + fieldType.typeName() + "] but executing against the field threw an exception: [" + e.getMessage() + "]");
+                    } catch (Exception | AssertionError e) {
+                        if (supportedVSTypes.contains(vst) && unsupportedMappedFieldTypes.contains(fieldType.typeName()) == false) {
+                            throw new AssertionError("Aggregator [" + aggregationBuilder.getType() + "] supports field type ["
+                                + fieldType.typeName() + "] but executing against the field threw an exception: [" + e.getMessage() + "]",
+                                e);
                         }
                     }
                 }
@@ -679,34 +705,50 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd
 
         String typeName = fieldType.typeName();
         ValuesSourceType vst = fieldType.getValuesSourceType();
-        
+        Document doc = new Document();
+        String json;
+
         if (vst.equals(CoreValuesSourceType.NUMERIC)) {
             // TODO note: once VS refactor adds DATE/BOOLEAN, this conditional will go away
+            long v;
             if (typeName.equals(DateFieldMapper.CONTENT_TYPE) || typeName.equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) {
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomNonNegativeLong())));
+                // positive integer because date_nanos gets unhappy with large longs
+                v = Math.abs(randomInt());
+                json = "{ \"" + fieldName + "\" : \"" + v + "\" }";
             } else if (typeName.equals(BooleanFieldMapper.CONTENT_TYPE)) {
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomBoolean() ? 0 : 1)));
+                v = randomBoolean() ? 0 : 1;
+                json = "{ \"" + fieldName + "\" : \"" + (v == 0 ? "false" : "true") + "\" }";
             } else if (typeName.equals(NumberFieldMapper.NumberType.DOUBLE.typeName())) {
-                long encoded = NumericUtils.doubleToSortableLong(Math.abs(randomDouble()));
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, encoded)));
+                double d = Math.abs(randomDouble());
+                v = NumericUtils.doubleToSortableLong(d);
+                json = "{ \"" + fieldName + "\" : \"" + d + "\" }";
             } else if (typeName.equals(NumberFieldMapper.NumberType.FLOAT.typeName())) {
-                long encoded = NumericUtils.floatToSortableInt(Math.abs(randomFloat()));
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, encoded)));
+                float f = Math.abs(randomFloat());
+                v = NumericUtils.floatToSortableInt(f);
+                json = "{ \"" + fieldName + "\" : \"" + f + "\" }";
             } else if (typeName.equals(NumberFieldMapper.NumberType.HALF_FLOAT.typeName())) {
-                long encoded = HalfFloatPoint.halfFloatToSortableShort(Math.abs(randomFloat()));
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, encoded)));
+                float f = Math.abs(randomFloat());
+                v = HalfFloatPoint.halfFloatToSortableShort(f);
+                json = "{ \"" + fieldName + "\" : \"" + f + "\" }";
             } else {
-                iw.addDocument(singleton(new SortedNumericDocValuesField(fieldName, randomNonNegativeLong())));
+                // smallest numeric is a byte so we select the smallest
+               v = Math.abs(randomByte());
+               json = "{ \"" + fieldName + "\" : \"" + v + "\" }";
             }
+            doc.add(new SortedNumericDocValuesField(fieldName, v));
+
         } else if (vst.equals(CoreValuesSourceType.BYTES)) {
             if (typeName.equals(BinaryFieldMapper.CONTENT_TYPE)) {
-                iw.addDocument(singleton(new BinaryFieldMapper.CustomBinaryDocValuesField(fieldName, new BytesRef("a").bytes)));
+                doc.add(new BinaryFieldMapper.CustomBinaryDocValuesField(fieldName, new BytesRef("a").bytes));
+                json = "{ \"" + fieldName + "\" : \"a\" }";
             } else if (typeName.equals(IpFieldMapper.CONTENT_TYPE)) {
                 // TODO note: once VS refactor adds IP, this conditional will go away
-                boolean v4 = randomBoolean();
-                iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(randomIp(v4))))));
+                InetAddress ip = randomIp(randomBoolean());
+                json = "{ \"" + fieldName + "\" : \"" + NetworkAddress.format(ip) + "\" }";
+                doc.add(new SortedSetDocValuesField(fieldName, new BytesRef(InetAddressPoint.encode(ip))));
             } else {
-                iw.addDocument(singleton(new SortedSetDocValuesField(fieldName, new BytesRef("a"))));
+                doc.add(new SortedSetDocValuesField(fieldName, new BytesRef("a")));
+                json = "{ \"" + fieldName + "\" : \"a\" }";
             }
         } else if (vst.equals(CoreValuesSourceType.RANGE)) {
             Object start;
@@ -743,19 +785,37 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd
             }
 
             final RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType, start, end, true, true);
-            iw.addDocument(singleton(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton(range)))));
-
+            doc.add(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton(range))));
+            json = "{ \"" + fieldName + "\" : { \n" +
+                "        \"gte\" : \"" + start + "\",\n" +
+                "        \"lte\" : \"" + end + "\"\n" +
+                "      }}";
         }  else if (vst.equals(CoreValuesSourceType.GEOPOINT)) {
-            iw.addDocument(singleton(new LatLonDocValuesField(fieldName, randomDouble(), randomDouble())));
+            double lat = randomDouble();
+            double lon = randomDouble();
+            doc.add(new LatLonDocValuesField(fieldName, lat, lon));
+            json = "{ \"" + fieldName + "\" : \"[" + lon + "," + lat + "]\" }";
         } else {
             throw new IllegalStateException("Unknown field type [" + typeName + "]");
         }
+
+        doc.add(new StoredField("_source", new BytesRef(json)));
+        iw.addDocument(doc);
     }
 
     private class MockParserContext extends Mapper.TypeParser.ParserContext {
         MockParserContext() {
             super(null, null, null, null, null);
         }
+
+        @Override
+        public IndexAnalyzers getIndexAnalyzers() {
+            NamedAnalyzer defaultAnalyzer = new NamedAnalyzer(AnalysisRegistry.DEFAULT_ANALYZER_NAME,
+                AnalyzerScope.GLOBAL, new StandardAnalyzer());
+            return new IndexAnalyzers(Map.of(AnalysisRegistry.DEFAULT_ANALYZER_NAME, defaultAnalyzer), Map.of(), Map.of());
+        }
+
+
     }
 
     @After
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
similarity index 85%
rename from server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 1d59ba476986d..1b533bdab83c5 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -33,13 +33,16 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder.ValidationContext;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 
 import static java.util.Collections.emptyList;
@@ -75,7 +78,7 @@ public void setUp() throws Exception {
             .put("node.name", AbstractQueryTestCase.class.toString())
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
             .build();
-        SearchModule searchModule = new SearchModule(settings, emptyList());
+        SearchModule searchModule = new SearchModule(settings, plugins());
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         entries.addAll(IndicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
@@ -89,6 +92,13 @@ public void setUp() throws Exception {
         }
     }
 
+    /**
+     * Plugins to add to the test.
+     */
+    protected List<SearchPlugin> plugins() {
+        return emptyList();
+    }
+
     /**
      * Generic test that creates new AggregatorFactory from the test
      * AggregatorFactory and checks both for equality and asserts equality on
@@ -196,4 +206,34 @@ public String randomNumericField() {
     protected NamedXContentRegistry xContentRegistry() {
         return xContentRegistry;
     }
+
+    /**
+     * Helper for testing validation.
+     */
+    protected String validate(AggregationBuilder parent, AF builder) {
+        return validate(ValidationContext.forInsideTree(parent, null), builder);
+    }
+
+    /**
+     * Helper for testing validation.
+     */
+    protected String validate(Collection<AggregationBuilder> siblingAggregations, AF builder) {
+        return validate(siblingAggregations, emptyList(), builder);
+    }
+
+    /**
+     * Helper for testing validation.
+     */
+    protected String validate(Collection<AggregationBuilder> siblingAggregations,
+            Collection<PipelineAggregationBuilder> siblingPipelineAggregations, AF builder) {
+        return validate(ValidationContext.forTreeRoot(siblingAggregations, siblingPipelineAggregations, null), builder);
+    }
+
+    /**
+     * Helper for testing validation.
+     */
+    protected String validate(ValidationContext context, AF builder) {
+        builder.validate(context);
+        return context.getValidationException() == null ? null : context.getValidationException().getMessage();
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java
index d3d54643f9461..0fb1af419edcc 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java
@@ -67,33 +67,33 @@ public void setupSuiteScopeCluster() throws Exception {
         ensureSearchable();
     }
 
-    public abstract void testEmptyAggregation() throws Exception;
+    public void testEmptyAggregation() throws Exception {}
 
-    public abstract void testUnmapped() throws Exception;
+    public void testUnmapped() throws Exception {}
 
-    public abstract void testSingleValuedField() throws Exception;
+    public void testSingleValuedField() throws Exception {}
 
-    public abstract void testSingleValuedFieldGetProperty() throws Exception;
+    public void testSingleValuedFieldGetProperty() throws Exception {}
 
-    public abstract void testSingleValuedFieldPartiallyUnmapped() throws Exception;
+    public void testSingleValuedFieldPartiallyUnmapped() throws Exception {}
 
-    public abstract void testSingleValuedFieldWithValueScript() throws Exception;
+    public void testSingleValuedFieldWithValueScript() throws Exception {}
 
-    public abstract void testSingleValuedFieldWithValueScriptWithParams() throws Exception;
+    public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {}
 
-    public abstract void testMultiValuedField() throws Exception;
+    public void testMultiValuedField() throws Exception {}
 
-    public abstract void testMultiValuedFieldWithValueScript() throws Exception;
+    public void testMultiValuedFieldWithValueScript() throws Exception {}
 
-    public abstract void testMultiValuedFieldWithValueScriptWithParams() throws Exception;
+    public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {}
 
-    public abstract void testScriptSingleValued() throws Exception;
+    public void testScriptSingleValued() throws Exception {}
 
-    public abstract void testScriptSingleValuedWithParams() throws Exception;
+    public void testScriptSingleValuedWithParams() throws Exception {}
 
-    public abstract void testScriptMultiValued() throws Exception;
+    public void testScriptMultiValued() throws Exception {}
 
-    public abstract void testScriptMultiValuedWithParams() throws Exception;
+    public void testScriptMultiValuedWithParams() throws Exception {}
 
-    public abstract void testOrderByEmptyAggregation() throws Exception;
+    public void testOrderByEmptyAggregation() throws Exception {}
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 3116792c502d6..18fab17f36d19 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -63,6 +63,7 @@
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.coordination.ElasticsearchNodeCommand;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -76,6 +77,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -91,8 +93,12 @@
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.smile.SmileXContent;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
@@ -149,6 +155,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.List;
@@ -542,6 +549,7 @@ private void afterInternal(boolean afterClass) throws Exception {
                     }
                     ensureClusterSizeConsistency();
                     ensureClusterStateConsistency();
+                    ensureClusterStateCanBeReadByNodeTool();
                     beforeIndexDeletion();
                     cluster().wipe(excludeTemplates()); // wipe after to make sure we fail in the test that didn't ack the delete
                     if (afterClass || currentClusterScope == Scope.TEST) {
@@ -1033,6 +1041,92 @@ protected void ensureClusterStateConsistency() throws IOException {
 
     }
 
+    protected void ensureClusterStateCanBeReadByNodeTool() throws IOException {
+        if (cluster() != null && cluster().size() > 0) {
+            final Client masterClient = client();
+            MetaData metaData = masterClient.admin().cluster().prepareState().all().get().getState().metaData();
+            final Map<String, String> serializationParams = new HashMap<>(2);
+            serializationParams.put("binary", "true");
+            serializationParams.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
+            final ToXContent.Params serializationFormatParams = new ToXContent.MapParams(serializationParams);
+
+            // when comparing XContent output, do not use binary format
+            final Map<String, String> compareParams = new HashMap<>(2);
+            compareParams.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
+            final ToXContent.Params compareFormatParams = new ToXContent.MapParams(compareParams);
+
+            {
+                MetaData metaDataWithoutIndices = MetaData.builder(metaData).removeAllIndices().build();
+
+                XContentBuilder builder = SmileXContent.contentBuilder();
+                builder.startObject();
+                metaDataWithoutIndices.toXContent(builder, serializationFormatParams);
+                builder.endObject();
+                final BytesReference originalBytes = BytesReference.bytes(builder);
+
+                XContentBuilder compareBuilder = SmileXContent.contentBuilder();
+                compareBuilder.startObject();
+                metaDataWithoutIndices.toXContent(compareBuilder, compareFormatParams);
+                compareBuilder.endObject();
+                final BytesReference compareOriginalBytes = BytesReference.bytes(compareBuilder);
+
+                final MetaData loadedMetaData;
+                try (XContentParser parser = createParser(ElasticsearchNodeCommand.namedXContentRegistry,
+                    SmileXContent.smileXContent, originalBytes)) {
+                    loadedMetaData = MetaData.fromXContent(parser);
+                }
+                builder = SmileXContent.contentBuilder();
+                builder.startObject();
+                loadedMetaData.toXContent(builder, compareFormatParams);
+                builder.endObject();
+                final BytesReference parsedBytes = BytesReference.bytes(builder);
+
+                assertNull(
+                    "cluster state XContent serialization does not match, expected " +
+                        XContentHelper.convertToMap(compareOriginalBytes, false, XContentType.SMILE) +
+                        " but got " +
+                        XContentHelper.convertToMap(parsedBytes, false, XContentType.SMILE),
+                    differenceBetweenMapsIgnoringArrayOrder(
+                        XContentHelper.convertToMap(compareOriginalBytes, false, XContentType.SMILE).v2(),
+                        XContentHelper.convertToMap(parsedBytes, false, XContentType.SMILE).v2()));
+            }
+
+            for (IndexMetaData indexMetaData : metaData) {
+                XContentBuilder builder = SmileXContent.contentBuilder();
+                builder.startObject();
+                indexMetaData.toXContent(builder, serializationFormatParams);
+                builder.endObject();
+                final BytesReference originalBytes = BytesReference.bytes(builder);
+
+                XContentBuilder compareBuilder = SmileXContent.contentBuilder();
+                compareBuilder.startObject();
+                indexMetaData.toXContent(compareBuilder, compareFormatParams);
+                compareBuilder.endObject();
+                final BytesReference compareOriginalBytes = BytesReference.bytes(compareBuilder);
+
+                final IndexMetaData loadedIndexMetaData;
+                try (XContentParser parser = createParser(ElasticsearchNodeCommand.namedXContentRegistry,
+                    SmileXContent.smileXContent, originalBytes)) {
+                    loadedIndexMetaData = IndexMetaData.fromXContent(parser);
+                }
+                builder = SmileXContent.contentBuilder();
+                builder.startObject();
+                loadedIndexMetaData.toXContent(builder, compareFormatParams);
+                builder.endObject();
+                final BytesReference parsedBytes = BytesReference.bytes(builder);
+
+                assertNull(
+                    "cluster state XContent serialization does not match, expected " +
+                        XContentHelper.convertToMap(compareOriginalBytes, false, XContentType.SMILE) +
+                        " but got " +
+                        XContentHelper.convertToMap(parsedBytes, false, XContentType.SMILE),
+                    differenceBetweenMapsIgnoringArrayOrder(
+                        XContentHelper.convertToMap(compareOriginalBytes, false, XContentType.SMILE).v2(),
+                        XContentHelper.convertToMap(parsedBytes, false, XContentType.SMILE).v2()));
+            }
+        }
+    }
+
     /**
      * Ensures the cluster is in a searchable state for the given indices. This means a searchable copy of each
      * shard is available on the cluster.
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 0a27993b40d7e..4362c80c4f164 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -1253,6 +1253,14 @@ protected final XContentParser createParser(XContent xContent, BytesReference da
         return xContent.createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, data.streamInput());
     }
 
+    /**
+     * Create a new {@link XContentParser}.
+     */
+    protected final XContentParser createParser(NamedXContentRegistry namedXContentRegistry, XContent xContent,
+                                                BytesReference data) throws IOException {
+        return xContent.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, data.streamInput());
+    }
+
     /**
      * The {@link NamedXContentRegistry} to use for this test. Subclasses should override and use liberally.
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
index ebf60bbfb4b6d..47a752e099696 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
@@ -41,9 +41,9 @@
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer;
 import org.elasticsearch.search.aggregations.ParsedAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.adjacency.ParsedAdjacencyMatrix;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
@@ -53,9 +53,9 @@
 import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter;
 import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
 import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal;
 import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
@@ -138,8 +138,8 @@
 import org.elasticsearch.search.aggregations.pipeline.ParsedStatsBucket;
 import org.elasticsearch.search.aggregations.pipeline.PercentilesBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
+import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -166,16 +166,23 @@ public abstract class InternalAggregationTestCase<T extends InternalAggregation>
      * Builds an {@link InternalAggregation.ReduceContextBuilder} that is valid but empty.
      */
     public static InternalAggregation.ReduceContextBuilder emptyReduceContextBuilder() {
+        return emptyReduceContextBuilder(PipelineTree.EMPTY);
+    }
+
+    /**
+     * Builds an {@link InternalAggregation.ReduceContextBuilder} that is valid and nearly
+     * empty <strong>except</strong> that it contain {@link PipelineAggregator}s.
+     */
+    public static InternalAggregation.ReduceContextBuilder emptyReduceContextBuilder(PipelineTree pipelineTree) {
         return new InternalAggregation.ReduceContextBuilder() {
             @Override
             public InternalAggregation.ReduceContext forPartialReduction() {
-                return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null);
+                return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null, () -> pipelineTree);
             }
 
             @Override
             public ReduceContext forFinalReduction() {
-                return InternalAggregation.ReduceContext.forFinalReduction(
-                        BigArrays.NON_RECYCLING_INSTANCE, null, b -> {}, PipelineTree.EMPTY);
+                return InternalAggregation.ReduceContext.forFinalReduction(BigArrays.NON_RECYCLING_INSTANCE, null, b -> {}, pipelineTree);
             }
         };
     }
@@ -291,7 +298,8 @@ public void testReduceRandom() {
             Collections.shuffle(toReduce, random());
             int r = randomIntBetween(1, toReduceSize);
             List<InternalAggregation> internalAggregations = toReduce.subList(0, r);
-            InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction(bigArrays, mockScriptService);
+            InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction(
+                    bigArrays, mockScriptService, () -> PipelineAggregator.PipelineTree.EMPTY);
             @SuppressWarnings("unchecked")
             T reduced = (T) inputs.get(0).reduce(internalAggregations, context);
             int initialBucketCount = 0;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 445fa98af651f..8fe7464087408 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -22,6 +22,7 @@
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 import org.apache.http.ssl.SSLContexts;
@@ -99,6 +100,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.in;
+import static org.hamcrest.Matchers.notNullValue;
 
 /**
  * Superclass for tests that interact with an external test cluster using Elasticsearch's {@link RestClient}.
@@ -1297,4 +1299,35 @@ private void ensureGlobalCheckpointSynced(String index) throws Exception {
                 });
         }, 60, TimeUnit.SECONDS);
     }
+
+    /**
+     * Wait for the license to be applied and active. The specified admin client is used to check the license and this is done using
+     * {@link ESTestCase#assertBusy(CheckedRunnable)} to give some time to the License to be applied on nodes.
+     *
+     * @param restClient the client to use
+     * @throws Exception if an exception is thrown while checking the status of the license
+     */
+    protected static void waitForActiveLicense(final RestClient restClient) throws Exception {
+        assertBusy(() -> {
+            final Request request = new Request(HttpGet.METHOD_NAME, "/_xpack");
+            request.setOptions(RequestOptions.DEFAULT.toBuilder());
+
+            final Response response = restClient.performRequest(request);
+            assertOK(response);
+
+            try (InputStream is = response.getEntity().getContent()) {
+                XContentType xContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue());
+                final Map<String, ?> map = XContentHelper.convertToMap(xContentType.xContent(), is, true);
+                assertThat(map, notNullValue());
+                assertThat("License must exist", map.containsKey("license"), equalTo(true));
+                @SuppressWarnings("unchecked")
+                final Map<String, ?> license = (Map<String, ?>) map.get("license");
+                assertThat("Expecting non-null license", license, notNullValue());
+                assertThat("License status must exist", license.containsKey("status"), equalTo(true));
+                final String status = (String) license.get("status");
+                assertThat("Expecting non-null license status", status, notNullValue());
+                assertThat("Expecting active license", status, equalTo("active"));
+            }
+        });
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java
index 9ac1269a044df..a5d932a3d1a3d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java
@@ -47,7 +47,7 @@ public void setUpController() {
         controller = new RestController(Collections.emptySet(), null,
             nodeClient,
             new NoneCircuitBreakerService(),
-            new UsageService(), randomBoolean());
+            new UsageService());
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index 0a89b2811b1a6..610e313fb0610 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -2378,15 +2378,15 @@ public String executor() {
                 assertEquals(1, transportStats.getRxCount());
                 assertEquals(2, transportStats.getTxCount());
                 assertEquals(25, transportStats.getRxSize().getBytes());
-                assertEquals(113, transportStats.getTxSize().getBytes());
+                assertEquals(111, transportStats.getTxSize().getBytes());
             });
             sendResponseLatch.countDown();
             responseLatch.await();
             stats = serviceC.transport.getStats(); // response has been received
             assertEquals(2, stats.getRxCount());
             assertEquals(2, stats.getTxCount());
-            assertEquals(52, stats.getRxSize().getBytes());
-            assertEquals(113, stats.getTxSize().getBytes());
+            assertEquals(50, stats.getRxSize().getBytes());
+            assertEquals(111, stats.getTxSize().getBytes());
         } finally {
             serviceC.close();
         }
@@ -2493,7 +2493,7 @@ public String executor() {
                 assertEquals(1, transportStats.getRxCount());
                 assertEquals(2, transportStats.getTxCount());
                 assertEquals(25, transportStats.getRxSize().getBytes());
-                assertEquals(113, transportStats.getTxSize().getBytes());
+                assertEquals(111, transportStats.getTxSize().getBytes());
             });
             sendResponseLatch.countDown();
             responseLatch.await();
@@ -2507,8 +2507,8 @@ public String executor() {
             String failedMessage = "Unexpected read bytes size. The transport exception that was received=" + exception;
             // 49 bytes are the non-exception message bytes that have been received. It should include the initial
             // handshake message and the header, version, etc bytes in the exception message.
-            assertEquals(failedMessage, 55 + streamOutput.bytes().length(), stats.getRxSize().getBytes());
-            assertEquals(113, stats.getTxSize().getBytes());
+            assertEquals(failedMessage, 53 + streamOutput.bytes().length(), stats.getRxSize().getBytes());
+            assertEquals(111, stats.getTxSize().getBytes());
         } finally {
             serviceC.close();
         }
diff --git a/x-pack/docs/en/rest-api/security/create-role-mappings.asciidoc b/x-pack/docs/en/rest-api/security/create-role-mappings.asciidoc
index e80ff662c8305..51ccb39a69eca 100644
--- a/x-pack/docs/en/rest-api/security/create-role-mappings.asciidoc
+++ b/x-pack/docs/en/rest-api/security/create-role-mappings.asciidoc
@@ -23,7 +23,7 @@ Creates and updates role mappings.
 [[security-api-put-role-mapping-desc]]
 ==== {api-description-title}
 
-Role mappings define which roles are assigned to each user. Each mapping has 
+Role mappings define which roles are assigned to each user. Each mapping has
 _rules_ that identify users and a list of _roles_ that are granted to those users.
 
 The role mapping APIs are generally the preferred way to manage role mappings
@@ -37,6 +37,31 @@ roles API>> or <<roles-management-file,roles files>>.
 
 For more information, see <<mapping-roles>>.
 
+[[_role_templates]]
+===== Role templates
+
+The most common use for role mappings is to create a mapping from a known value
+on the user to a fixed role name. For example, all users in the
+`cn=admin,dc=example,dc=com` LDAP group should be given the `superuser` role in
+{es}. The `roles` field is used for this purpose.
+
+For more complex needs, it is possible to use Mustache templates to dynamically
+determine the names of the roles that should be granted to the user. The
+`role_templates` field is used for this purpose.
+
+NOTE: To use role templates successfully, the relevant scripting feature must be
+enabled. Otherwise, all attempts to create a role mapping with role templates
+fail. See <<allowed-script-types-setting>>.
+
+All of the <<role-mapping-resources,user fields>> that are available in the
+role mapping `rules` are also available in the role templates. Thus it is possible
+to assign a user to a role that reflects their `username`, their `groups`, or the
+name of the `realm` to which they authenticated.
+
+By default a template is evaluated to produce a single string that is the name
+of the role which should be assigned to the user. If the `format` of the template
+is set to `"json"` then the template is expected to produce a JSON string or an
+array of JSON strings for the role names.
 
 [[security-api-put-role-mapping-path-params]]
 ==== {api-path-parms-title}
@@ -77,31 +102,7 @@ _Exactly one of `roles` or `role_templates` must be specified_.
 `rules`::
 (Required, object) The rules that determine which users should be matched by the
 mapping. A rule is a logical condition that is expressed by using a JSON DSL.
-See  <<role-mapping-resources>>. 
-
-==== Role Templates
-
-The most common use for role mappings is to create a mapping from a known value
-on the user to a fixed role name.
-For example, all users in the `cn=admin,dc=example,dc=com` LDAP group should be
-given the `superuser` role in {es}.
-The `roles` field is used for this purpose.
-
-For more complex needs it is possible to use Mustache templates to dynamically
-determine the names of the roles that should be granted to the user.
-The `role_templates` field is used for this purpose.
-
-All of the <<role-mapping-resources,user fields>> that are available in the
-role mapping `rules` are also available in the role templates. Thus it is possible
-to assign a user to a role that reflects their `username`, their `groups` or the
-name of the `realm` to which they authenticated.
-
-By default a template is evaluated to produce a single string that is the name
-of the role which should be assigned to the user. If the `format` of the template
-is set to `"json"` then the template is expected to produce a JSON string, or an
-array of JSON strings for the role name(s).
-
-The Examples section below demonstrates the use of templated role names.
+See  <<role-mapping-resources>>.
 
 [[security-api-put-role-mapping-example]]
 ==== {api-examples-title}
@@ -339,4 +340,3 @@ POST /_security/role_mapping/mapping9
 <1> Because it is not possible to specify both `roles` and `role_templates` in
     the same role mapping, we can apply a "fixed name" role by using a template
     that has no substitutions.
-
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java
index 844c661aadf7e..47480d98a00e2 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java
@@ -10,20 +10,15 @@
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketMetricsParser;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
 
 public class CumulativeCardinalityPipelineAggregationBuilder
@@ -90,14 +85,12 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
-                           Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
+    protected void validate(ValidationContext context) {
         if (bucketsPaths.length != 1) {
-            throw new IllegalStateException(BUCKETS_PATH.getPreferredName()
-                + " must contain a single entry for aggregation [" + name + "]");
+            context.addBucketPathValidationError("must contain a single entry for aggregation [" + name + "]");
         }
 
-        validateSequentiallyOrderedParentAggs(parent, NAME, name);
+        context.validateParentAggSequentiallyOrdered(NAME, name);
     }
 
     @Override
@@ -105,6 +98,7 @@ protected final XContentBuilder internalXContent(XContentBuilder builder, Params
         if (format != null) {
             builder.field(BucketMetricsParser.FORMAT.getPreferredName(), format);
         }
+        builder.field(BUCKETS_PATH_FIELD.getPreferredName(), bucketsPaths[0]);
         return builder;
     }
 
@@ -126,4 +120,9 @@ public boolean equals(Object obj) {
     public String getWriteableName() {
         return NAME;
     }
+
+    @Override
+    protected boolean overrideBucketsPath() {
+        return true;
+    }
 }
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/StubAggregatorFactory.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/StubAggregatorFactory.java
deleted file mode 100644
index ace86ddea6bb8..0000000000000
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/StubAggregatorFactory.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.analytics;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.MockBigArrays;
-import org.elasticsearch.common.util.MockPageCacheRecycler;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Test implementation for AggregatorFactory.
- */
-public class StubAggregatorFactory extends AggregatorFactory {
-
-    private final Aggregator aggregator;
-
-    private StubAggregatorFactory(QueryShardContext queryShardContext, Aggregator aggregator) throws IOException {
-        super("_name", queryShardContext, null, new AggregatorFactories.Builder(), Collections.emptyMap());
-        this.aggregator = aggregator;
-    }
-
-    @Override
-    protected Aggregator createInternal(SearchContext searchContext,
-                                            Aggregator parent,
-                                            boolean collectsFromSingleBucket,
-                                            List list, Map metaData) throws IOException {
-        return aggregator;
-    }
-
-    public static StubAggregatorFactory createInstance() throws IOException {
-        BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
-        QueryShardContext queryShardContext = mock(QueryShardContext.class);
-        when(queryShardContext.bigArrays()).thenReturn(bigArrays);
-
-        Aggregator aggregator = mock(Aggregator.class);
-
-        return new StubAggregatorFactory(queryShardContext, aggregator);
-    }
-}
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java
index 31a532d5314be..d8af20e311628 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java
@@ -19,9 +19,16 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.InternalAggregation;
@@ -30,15 +37,51 @@
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
 import static java.util.Collections.singleton;
 import static org.hamcrest.Matchers.equalTo;
 
 public class BoxplotAggregatorTests extends AggregatorTestCase {
 
+    /** Script to return the {@code _value} provided by aggs framework. */
+    public static final String VALUE_SCRIPT = "_value";
+
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new BoxplotAggregationBuilder("foo").field(fieldName);
+    }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return List.of(CoreValuesSourceType.NUMERIC,
+            CoreValuesSourceType.HISTOGRAM);
+    }
+
+    @Override
+    protected ScriptService getMockScriptService() {
+        Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
+
+        scripts.put(VALUE_SCRIPT, vars -> ((Number) vars.get("_value")).doubleValue() + 1);
+
+        MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME,
+            scripts,
+            Collections.emptyMap());
+        Map<String, ScriptEngine> engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine);
+
+        return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+    }
+
+
     public void testNoMatchingField() throws IOException {
         testCase(new MatchAllDocsQuery(), iw -> {
             iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
@@ -122,6 +165,28 @@ public void testSomeMatchesNumericDocValues() throws IOException {
         });
     }
 
+    public void testMissingField() throws IOException {
+        BoxplotAggregationBuilder aggregationBuilder = new BoxplotAggregationBuilder("boxplot").field("number").missing(10L);
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldType.setName("number");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("other", 2)));
+            iw.addDocument(singleton(new NumericDocValuesField("other", 2)));
+            iw.addDocument(singleton(new NumericDocValuesField("other", 3)));
+            iw.addDocument(singleton(new NumericDocValuesField("other", 4)));
+            iw.addDocument(singleton(new NumericDocValuesField("other", 5)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 0)));
+        },  (Consumer<InternalBoxplot>) boxplot -> {
+            assertEquals(0, boxplot.getMin(), 0);
+            assertEquals(10, boxplot.getMax(), 0);
+            assertEquals(10, boxplot.getQ1(), 0);
+            assertEquals(10, boxplot.getQ2(), 0);
+            assertEquals(10, boxplot.getQ3(), 0);
+        }, fieldType);
+    }
+
     public void testUnmappedWithMissingField() throws IOException {
         BoxplotAggregationBuilder aggregationBuilder = new BoxplotAggregationBuilder("boxplot")
             .field("does_not_exist").missing(0L);
@@ -291,6 +356,68 @@ public void testGetProperty() throws IOException {
         }, fieldType);
     }
 
+    public void testValueScript() throws IOException {
+        BoxplotAggregationBuilder aggregationBuilder = new BoxplotAggregationBuilder("boxplot")
+            .field("number")
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap()));
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldType.setName("number");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
+        }, (Consumer<InternalBoxplot>) boxplot -> {
+            assertEquals(2, boxplot.getMin(), 0);
+            assertEquals(8, boxplot.getMax(), 0);
+            assertEquals(2, boxplot.getQ1(), 0);
+            assertEquals(5, boxplot.getQ2(), 0);
+            assertEquals(8, boxplot.getQ3(), 0);
+        }, fieldType);
+    }
+
+    public void testValueScriptUnmapped() throws IOException {
+        BoxplotAggregationBuilder aggregationBuilder = new BoxplotAggregationBuilder("boxplot")
+            .field("does_not_exist")
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap()));
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldType.setName("number");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
+        }, (Consumer<InternalBoxplot>) boxplot -> {
+            assertEquals(Double.POSITIVE_INFINITY, boxplot.getMin(), 0);
+            assertEquals(Double.NEGATIVE_INFINITY, boxplot.getMax(), 0);
+            assertEquals(Double.NaN, boxplot.getQ1(), 0);
+            assertEquals(Double.NaN, boxplot.getQ2(), 0);
+            assertEquals(Double.NaN, boxplot.getQ3(), 0);
+        }, fieldType);
+    }
+
+    public void testValueScriptUnmappedMissing() throws IOException {
+        BoxplotAggregationBuilder aggregationBuilder = new BoxplotAggregationBuilder("boxplot")
+            .field("does_not_exist")
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap()))
+            .missing(1.0);
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldType.setName("number");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
+        }, (Consumer<InternalBoxplot>) boxplot -> {
+            // Note: the way scripts, missing and unmapped interact, these will be the missing value and the script is not invoked
+            assertEquals(1.0, boxplot.getMin(), 0);
+            assertEquals(1.0, boxplot.getMax(), 0);
+            assertEquals(1.0, boxplot.getQ1(), 0);
+            assertEquals(1.0, boxplot.getQ2(), 0);
+            assertEquals(1.0, boxplot.getQ3(), 0);
+        }, fieldType);
+    }
+
     private void testCase(Query query,
                           CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
                           Consumer<InternalBoxplot> verify) throws IOException {
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java
index e5b4b2ce0bf07..fd7adddd4234c 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityAggregatorTests.java
@@ -16,46 +16,27 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.CheckedConsumer;
-import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
-import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalOrder;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
-import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
-import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-import org.elasticsearch.xpack.analytics.StubAggregatorFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Mockito.mock;
 
 public class CumulativeCardinalityAggregatorTests extends AggregatorTestCase {
 
@@ -115,53 +96,6 @@ public void testAllNull() throws IOException {
         });
     }
 
-    public void testParentValidations() throws IOException {
-        ValuesSourceConfig<ValuesSource> valuesSource = new ValuesSourceConfig<>(CoreValuesSourceType.NUMERIC);
-
-        // Histogram
-        Set<PipelineAggregationBuilder> aggBuilders = new HashSet<>();
-        aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum"));
-        AggregatorFactory parent = new HistogramAggregatorFactory("name", valuesSource, 0.0d, 0.0d,
-            mock(InternalOrder.class), false, 0L, 0.0d, 1.0d, mock(QueryShardContext.class), null,
-            new AggregatorFactories.Builder(), Collections.emptyMap());
-        CumulativeCardinalityPipelineAggregationBuilder builder
-            = new CumulativeCardinalityPipelineAggregationBuilder("name", "valid");
-        builder.validate(parent, Collections.emptySet(), aggBuilders);
-
-        // Date Histogram
-        aggBuilders.clear();
-        aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum"));
-        parent = new DateHistogramAggregatorFactory("name", valuesSource,
-            mock(InternalOrder.class), false, 0L, mock(Rounding.class), mock(Rounding.class),
-            mock(ExtendedBounds.class), mock(QueryShardContext.class), mock(AggregatorFactory.class),
-            new AggregatorFactories.Builder(), Collections.emptyMap());
-        builder = new CumulativeCardinalityPipelineAggregationBuilder("name", "valid");
-        builder.validate(parent, Collections.emptySet(), aggBuilders);
-
-        // Auto Date Histogram
-        ValuesSourceConfig<ValuesSource.Numeric> numericVS = new ValuesSourceConfig<>(CoreValuesSourceType.NUMERIC);
-        aggBuilders.clear();
-        aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum"));
-        AutoDateHistogramAggregationBuilder.RoundingInfo[] roundings = new AutoDateHistogramAggregationBuilder.RoundingInfo[1];
-        parent = new AutoDateHistogramAggregatorFactory("name", numericVS,
-            1, roundings,
-            mock(QueryShardContext.class), null, new AggregatorFactories.Builder(), Collections.emptyMap());
-        builder = new CumulativeCardinalityPipelineAggregationBuilder("name", "valid");
-        builder.validate(parent, Collections.emptySet(), aggBuilders);
-
-        // Mocked "test" agg, should fail validation
-        aggBuilders.clear();
-        aggBuilders.add(new CumulativeCardinalityPipelineAggregationBuilder("cumulative_card", "sum"));
-        StubAggregatorFactory parentFactory = StubAggregatorFactory.createInstance();
-
-        CumulativeCardinalityPipelineAggregationBuilder failBuilder
-            = new CumulativeCardinalityPipelineAggregationBuilder("name", "invalid_agg>metric");
-        IllegalStateException ex = expectThrows(IllegalStateException.class,
-            () -> failBuilder.validate(parentFactory, Collections.emptySet(), aggBuilders));
-        assertEquals("cumulative_cardinality aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent",
-            ex.getMessage());
-    }
-
     public void testNonCardinalityAgg() {
         Query query = new MatchAllDocsQuery();
 
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityTests.java
new file mode 100644
index 0000000000000..d9ff43e7d06c8
--- /dev/null
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityTests.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.analytics.cumulativecardinality;
+
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CumulativeCardinalityTests extends BasePipelineAggregationTestCase<CumulativeCardinalityPipelineAggregationBuilder> {
+    @Override
+    protected List<SearchPlugin> plugins() {
+        return singletonList(new SearchPlugin() {
+            @Override
+            public List<PipelineAggregationSpec> getPipelineAggregations() {
+                return singletonList(new PipelineAggregationSpec(
+                        CumulativeCardinalityPipelineAggregationBuilder.NAME,
+                        CumulativeCardinalityPipelineAggregationBuilder::new,
+                        CumulativeCardinalityPipelineAggregator::new,
+                        CumulativeCardinalityPipelineAggregationBuilder.PARSER));
+            }
+        });
+    }
+
+    @Override
+    protected CumulativeCardinalityPipelineAggregationBuilder createTestAggregatorFactory() {
+        String name = randomAlphaOfLengthBetween(3, 20);
+        String bucketsPath = randomAlphaOfLengthBetween(3, 20);
+        CumulativeCardinalityPipelineAggregationBuilder builder =
+                new CumulativeCardinalityPipelineAggregationBuilder(name, bucketsPath);
+        if (randomBoolean()) {
+            builder.format(randomAlphaOfLengthBetween(1, 10));
+        }
+        return builder;
+    }
+
+
+    public void testParentValidations() throws IOException {
+        CumulativeCardinalityPipelineAggregationBuilder builder =
+                new CumulativeCardinalityPipelineAggregationBuilder("name", randomAlphaOfLength(5));
+
+        assertThat(validate(new HistogramAggregationBuilder("name"), builder), nullValue());
+        assertThat(validate(new DateHistogramAggregationBuilder("name"), builder), nullValue());
+        assertThat(validate(new AutoDateHistogramAggregationBuilder("name"), builder), nullValue());
+
+        // Mocked "test" agg, should fail validation
+        AggregationBuilder stubParent = mock(AggregationBuilder.class);
+        when(stubParent.getName()).thenReturn("name");
+        assertThat(validate(stubParent, builder), equalTo(
+                "Validation Failed: 1: cumulative_cardinality aggregation [name] must have a histogram, "
+                + "date_histogram or auto_date_histogram as parent;"));
+
+        assertThat(validate(emptyList(), builder), equalTo(
+                "Validation Failed: 1: cumulative_cardinality aggregation [name] must have a histogram, "
+                + "date_histogram or auto_date_histogram as parent but doesn't have a parent;"));
+    }
+}
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java
index ff4bae131dd8a..c9790638da078 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java
@@ -18,24 +18,46 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.IpFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static java.util.stream.Collectors.toList;
 
 public class StringStatsAggregatorTests extends AggregatorTestCase {
 
+    private static final String VALUE_SCRIPT_NAME = "value_script";
+    private static final String FIELD_SCRIPT_NAME = "field_script";
+
     private void testCase(Query query,
                           CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
                           Consumer<InternalStringStats> verify) throws IOException {
@@ -119,6 +141,32 @@ public void testUnmappedWithMissingField() throws IOException {
         }, null);
     }
 
+    public void testMissing() throws IOException {
+        final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType();
+        fieldType.setName("text");
+        fieldType.setFielddata(true);
+
+        final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name")
+            .field(fieldType.name())
+            .missing("b");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new TextField(fieldType.name(), "a", Field.Store.NO)));
+            iw.addDocument(emptySet());
+            iw.addDocument(singleton(new TextField(fieldType.name(), "a", Field.Store.NO)));
+            iw.addDocument(emptySet());
+        }, stats -> {
+            assertEquals(4, stats.getCount());
+            assertEquals(1, stats.getMaxLength());
+            assertEquals(1, stats.getMinLength());
+            assertEquals(1.0, stats.getAvgLength(), 0);
+            assertEquals(2, stats.getDistribution().size());
+            assertEquals(0.5, stats.getDistribution().get("a"), 0);
+            assertEquals(0.5, stats.getDistribution().get("b"), 0);
+            assertEquals(1.0, stats.getEntropy(), 0);
+        }, fieldType);
+    }
+
     public void testSingleValuedField() throws IOException {
         testCase(new MatchAllDocsQuery(), iw -> {
             for(int i=0; i < 10; i++) {
@@ -258,4 +306,144 @@ public void testNestedAggregation() throws IOException {
         directory.close();
     }
 
+    public void testValueScriptSingleValuedField() throws IOException {
+        final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType();
+        fieldType.setName("text");
+        fieldType.setFielddata(true);
+
+        final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name")
+            .field(fieldType.name())
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_NAME, emptyMap()));
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new TextField(fieldType.name(), "b", Field.Store.NO)));
+            iw.addDocument(singleton(new TextField(fieldType.name(), "b", Field.Store.NO)));
+        }, stats -> {
+            assertEquals(2, stats.getCount());
+            assertEquals(2, stats.getMaxLength());
+            assertEquals(2, stats.getMinLength());
+            assertEquals(2.0, stats.getAvgLength(), 0);
+            assertEquals(2, stats.getDistribution().size());
+            assertEquals(0.5, stats.getDistribution().get("a"), 0);
+            assertEquals(0.5, stats.getDistribution().get("b"), 0);
+            assertEquals(1.0, stats.getEntropy(), 0);
+        }, fieldType);
+    }
+
+    public void testValueScriptMultiValuedField() throws IOException {
+        final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType();
+        fieldType.setName("text");
+        fieldType.setFielddata(true);
+
+        final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name")
+            .field(fieldType.name())
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT_NAME, emptyMap()));
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(Set.of(
+                new TextField(fieldType.name(), "b", Field.Store.NO),
+                new TextField(fieldType.name(), "c", Field.Store.NO)
+            ));
+            iw.addDocument(Set.of(
+                new TextField(fieldType.name(), "b", Field.Store.NO),
+                new TextField(fieldType.name(), "c", Field.Store.NO)
+            ));
+        }, stats -> {
+            assertEquals(4, stats.getCount());
+            assertEquals(2, stats.getMaxLength());
+            assertEquals(2, stats.getMinLength());
+            assertEquals(2.0, stats.getAvgLength(), 0);
+            assertEquals(3, stats.getDistribution().size());
+            assertEquals(0.5, stats.getDistribution().get("a"), 0);
+            assertEquals(0.25, stats.getDistribution().get("b"), 0);
+            assertEquals(0.25, stats.getDistribution().get("c"), 0);
+            assertEquals(1.5, stats.getEntropy(), 0);
+        }, fieldType);
+    }
+
+    public void testFieldScriptSingleValuedField() throws IOException {
+        final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType();
+        fieldType.setName("text");
+        fieldType.setFielddata(true);
+
+        final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name")
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_NAME, singletonMap("field", fieldType.name())));
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new TextField(fieldType.name(), "b", Field.Store.NO)));
+            iw.addDocument(singleton(new TextField(fieldType.name(), "b", Field.Store.NO)));
+        }, stats -> {
+            assertEquals(2, stats.getCount());
+            assertEquals(2, stats.getMaxLength());
+            assertEquals(2, stats.getMinLength());
+            assertEquals(2.0, stats.getAvgLength(), 0);
+            assertEquals(2, stats.getDistribution().size());
+            assertEquals(0.5, stats.getDistribution().get("a"), 0);
+            assertEquals(0.5, stats.getDistribution().get("b"), 0);
+            assertEquals(1.0, stats.getEntropy(), 0);
+        }, fieldType);
+    }
+
+    public void testFieldScriptMultiValuedField() throws IOException {
+        final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType();
+        fieldType.setName("text");
+        fieldType.setFielddata(true);
+
+        final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name")
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, FIELD_SCRIPT_NAME, singletonMap("field", fieldType.name())));
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(Set.of(
+                new TextField(fieldType.name(), "b", Field.Store.NO),
+                new TextField(fieldType.name(), "c", Field.Store.NO)
+            ));
+            iw.addDocument(Set.of(
+                new TextField(fieldType.name(), "b", Field.Store.NO),
+                new TextField(fieldType.name(), "c", Field.Store.NO)
+            ));
+        }, stats -> {
+            assertEquals(4, stats.getCount());
+            assertEquals(2, stats.getMaxLength());
+            assertEquals(2, stats.getMinLength());
+            assertEquals(2.0, stats.getAvgLength(), 0);
+            assertEquals(3, stats.getDistribution().size());
+            assertEquals(0.5, stats.getDistribution().get("a"), 0);
+            assertEquals(0.25, stats.getDistribution().get("b"), 0);
+            assertEquals(0.25, stats.getDistribution().get("c"), 0);
+            assertEquals(1.5, stats.getEntropy(), 0);
+        }, fieldType);
+    }
+
+    @Override
+    protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) {
+        return new StringStatsAggregationBuilder("_name")
+            .field(fieldName);
+    }
+
+    @Override
+    protected List<ValuesSourceType> getSupportedValuesSourceTypes() {
+        return singletonList(CoreValuesSourceType.BYTES);
+    }
+
+    @Override
+    protected List<String> unsupportedMappedFieldTypes() {
+        return singletonList(IpFieldMapper.CONTENT_TYPE);
+    }
+
+    @Override
+    protected ScriptService getMockScriptService() {
+        final Map<String, Function<Map<String, Object>, Object>> scripts = Map.of(
+            VALUE_SCRIPT_NAME, vars -> "a" + vars.get("_value"),
+            FIELD_SCRIPT_NAME, vars -> {
+                final String fieldName = (String) vars.get("field");
+                final LeafDocLookup lookup = (LeafDocLookup) vars.get("doc");
+                return lookup.get(fieldName).stream()
+                    .map(value -> "a" + value)
+                    .collect(toList());
+            }
+        );
+        final MockScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, emptyMap());
+        final Map<String, ScriptEngine> engines = singletonMap(engine.getType(), engine);
+        return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+    }
 }
diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
index 05e0132deb7d3..bfca713db5350 100644
--- a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
+++ b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java
@@ -127,16 +127,16 @@ static Response submitAsyncSearch(String indexName, String query, TimeValue wait
         final Request request = new Request("POST", indexName + "/_async_search");
         setRunAsHeader(request, user);
         request.addParameter("q", query);
-        request.addParameter("wait_for_completion", waitForCompletion.toString());
+        request.addParameter("wait_for_completion_timeout", waitForCompletion.toString());
         // we do the cleanup explicitly
-        request.addParameter("clean_on_completion", "false");
+        request.addParameter("keep_on_completion", "true");
         return client().performRequest(request);
     }
 
     static Response getAsyncSearch(String id, String user) throws IOException {
         final Request request = new Request("GET",  "/_async_search/" + id);
         setRunAsHeader(request, user);
-        request.addParameter("wait_for_completion", "0ms");
+        request.addParameter("wait_for_completion_timeout", "0ms");
         return client().performRequest(request);
     }
 
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
index 94d5e7797cad1..289e9087c8678 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java
@@ -38,9 +38,9 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
-import org.elasticsearch.xpack.core.security.SecurityContext;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
index 479daffc832b5..98e6d099a5ee9 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
@@ -35,6 +35,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
@@ -42,6 +43,7 @@
  * Task that tracks the progress of a currently running {@link SearchRequest}.
  */
 final class AsyncSearchTask extends SearchTask {
+    private final BooleanSupplier checkSubmitCancellation;
     private final AsyncSearchId searchId;
     private final Client client;
     private final ThreadPool threadPool;
@@ -68,6 +70,7 @@ final class AsyncSearchTask extends SearchTask {
      * @param type The type of the task.
      * @param action The action name.
      * @param parentTaskId The parent task id.
+     * @param checkSubmitCancellation A boolean supplier that checks if the submit task has been cancelled.
      * @param originHeaders All the request context headers.
      * @param taskHeaders The filtered request headers for the task.
      * @param searchId The {@link AsyncSearchId} of the task.
@@ -78,6 +81,7 @@ final class AsyncSearchTask extends SearchTask {
                     String type,
                     String action,
                     TaskId parentTaskId,
+                    BooleanSupplier checkSubmitCancellation,
                     TimeValue keepAlive,
                     Map<String, String> originHeaders,
                     Map<String, String> taskHeaders,
@@ -86,6 +90,7 @@ final class AsyncSearchTask extends SearchTask {
                     ThreadPool threadPool,
                     Supplier<InternalAggregation.ReduceContext> aggReduceContextSupplier) {
         super(id, type, action, "async_search", parentTaskId, taskHeaders);
+        this.checkSubmitCancellation = checkSubmitCancellation;
         this.expirationTimeMillis = getStartTime() + keepAlive.getMillis();
         this.originHeaders = originHeaders;
         this.searchId = searchId;
@@ -291,10 +296,13 @@ private AsyncSearchResponse getResponse() {
         return searchResponse.get().toAsyncSearchResponse(this, expirationTimeMillis);
     }
 
-    // cancels the task if it expired
-    private void checkExpiration() {
+    // checks if the search task should be cancelled
+    private void checkCancellation() {
         long now = System.currentTimeMillis();
-        if (expirationTimeMillis < now) {
+        if (expirationTimeMillis < now || checkSubmitCancellation.getAsBoolean()) {
+            // we cancel the search task if the initial submit task was cancelled,
+            // this is needed because the task cancellation mechanism doesn't
+            // handle the cancellation of grand-children.
             cancelTask(() -> {});
         }
     }
@@ -302,30 +310,31 @@ private void checkExpiration() {
     class Listener extends SearchProgressActionListener {
         @Override
         protected void onQueryResult(int shardIndex) {
-            checkExpiration();
+            checkCancellation();
         }
 
         @Override
         protected void onFetchResult(int shardIndex) {
-            checkExpiration();
+            checkCancellation();
         }
 
         @Override
         protected void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
             // best effort to cancel expired tasks
-            checkExpiration();
-            searchResponse.get().addShardFailure(shardIndex, new ShardSearchFailure(exc, shardTarget));
+            checkCancellation();
+            searchResponse.get().addShardFailure(shardIndex,
+                new ShardSearchFailure(exc, shardTarget.getNodeId() != null ? shardTarget : null));
         }
 
         @Override
         protected void onFetchFailure(int shardIndex, Exception exc) {
-            checkExpiration();
+            checkCancellation();
         }
 
         @Override
         protected void onListShards(List<SearchShard> shards, List<SearchShard> skipped, Clusters clusters, boolean fetchPhase) {
             // best effort to cancel expired tasks
-            checkExpiration();
+            checkCancellation();
             searchResponse.compareAndSet(null,
                 new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, aggReduceContextSupplier));
             executeInitListeners();
@@ -334,7 +343,7 @@ protected void onListShards(List<SearchShard> shards, List<SearchShard> skipped,
         @Override
         public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
             // best effort to cancel expired tasks
-            checkExpiration();
+            checkCancellation();
             searchResponse.get().updatePartialResponse(shards.size(),
                 new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs,
                     null, null, false, null, reducePhase), aggs == null);
@@ -343,7 +352,7 @@ public void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, Inter
         @Override
         public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {
             // best effort to cancel expired tasks
-            checkExpiration();
+            checkCancellation();
             searchResponse.get().updatePartialResponse(shards.size(),
                 new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs,
                     null, null, false, null, reducePhase), true);
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
index cce587abd15d3..47d82958da31c 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java
@@ -138,7 +138,7 @@ synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task, lon
             }
             long tookInMillis = TimeValue.timeValueNanos(System.nanoTime() - task.getStartTimeNanos()).getMillis();
             resp = new SearchResponse(sections, null, totalShards, successfulShards,
-                skippedShards, tookInMillis, buildShardFailures(), clusters);
+                skippedShards, tookInMillis, buildShardFailures(), clusters, null);
         } else {
             resp = null;
         }
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java
index 8dc29bf071e0d..466adfa798abf 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java
@@ -34,8 +34,8 @@ public String getName() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
         GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(request.param("id"));
-        if (request.hasParam("wait_for_completion")) {
-            get.setWaitForCompletion(request.paramAsTime("wait_for_completion", get.getWaitForCompletion()));
+        if (request.hasParam("wait_for_completion_timeout")) {
+            get.setWaitForCompletion(request.paramAsTime("wait_for_completion_timeout", get.getWaitForCompletion()));
         }
         if (request.hasParam("keep_alive")) {
             get.setKeepAlive(request.paramAsTime("keep_alive", get.getKeepAlive()));
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
index e1020f8e5a234..2b4eedc1c866b 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java
@@ -45,17 +45,20 @@ public String getName() {
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         SubmitAsyncSearchRequest submit = new SubmitAsyncSearchRequest();
         IntConsumer setSize = size -> submit.getSearchRequest().source().size(size);
+        //for simplicity, we share parsing with ordinary search. That means a couple of unsupported parameters, like scroll,
+        // pre_filter_shard_size and ccs_minimize_roundtrips get set to the search request although the REST spec don't list
+        //them as supported. We rely on SubmitAsyncSearchRequest#validate to fail in case they are set.
         request.withContentOrSourceParamParserOrNull(parser ->
             parseSearchRequest(submit.getSearchRequest(), request, parser, setSize));
 
-        if (request.hasParam("wait_for_completion")) {
-            submit.setWaitForCompletion(request.paramAsTime("wait_for_completion", submit.getWaitForCompletion()));
+        if (request.hasParam("wait_for_completion_timeout")) {
+            submit.setWaitForCompletionTimeout(request.paramAsTime("wait_for_completion_timeout", submit.getWaitForCompletionTimeout()));
         }
         if (request.hasParam("keep_alive")) {
             submit.setKeepAlive(request.paramAsTime("keep_alive", submit.getKeepAlive()));
         }
-        if (request.hasParam("clean_on_completion")) {
-            submit.setCleanOnCompletion(request.paramAsBoolean("clean_on_completion", submit.isCleanOnCompletion()));
+        if (request.hasParam("keep_on_completion")) {
+            submit.setKeepOnCompletion(request.paramAsBoolean("keep_on_completion", submit.isKeepOnCompletion()));
         }
         return channel -> {
             RestStatusToXContentListener<AsyncSearchResponse> listener = new RestStatusToXContentListener<>(channel);
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
index 032c6114a1a44..efdf40da76f73 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
@@ -66,14 +66,14 @@ public TransportSubmitAsyncSearchAction(ClusterService clusterService,
     @Override
     protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionListener<AsyncSearchResponse> submitListener) {
         CancellableTask submitTask = (CancellableTask) task;
-        final SearchRequest searchRequest = createSearchRequest(request, submitTask.getId(), request.getKeepAlive());
+        final SearchRequest searchRequest = createSearchRequest(request, submitTask, request.getKeepAlive());
         AsyncSearchTask searchTask = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest);
         searchAction.execute(searchTask, searchRequest, searchTask.getSearchProgressActionListener());
         searchTask.addCompletionListener(
             new ActionListener<>() {
                 @Override
                 public void onResponse(AsyncSearchResponse searchResponse) {
-                    if (searchResponse.isRunning() || request.isCleanOnCompletion() == false) {
+                    if (searchResponse.isRunning() || request.isKeepOnCompletion()) {
                         // the task is still running and the user cannot wait more so we create
                         // a document for further retrieval
                         try {
@@ -81,7 +81,7 @@ public void onResponse(AsyncSearchResponse searchResponse) {
                                 // the user cancelled the submit so we don't store anything
                                 // and propagate the failure
                                 Exception cause = new TaskCancelledException(submitTask.getReasonCancelled());
-                                onFatalFailure(searchTask, cause, false, submitListener);
+                                onFatalFailure(searchTask, cause, searchResponse.isRunning(), submitListener);
                             } else {
                                 final String docId = searchTask.getSearchId().getDocId();
                                 // creates the fallback response if the node crashes/restarts in the middle of the request
@@ -126,10 +126,10 @@ public void onFailure(Exception exc) {
                 public void onFailure(Exception exc) {
                     submitListener.onFailure(exc);
                 }
-            }, request.getWaitForCompletion());
+            }, request.getWaitForCompletionTimeout());
     }
 
-    private SearchRequest createSearchRequest(SubmitAsyncSearchRequest request, long parentTaskId, TimeValue keepAlive) {
+    private SearchRequest createSearchRequest(SubmitAsyncSearchRequest request, CancellableTask submitTask, TimeValue keepAlive) {
         String docID = UUIDs.randomBase64UUID();
         Map<String, String> originHeaders = nodeClient.threadPool().getThreadContext().getHeaders();
         SearchRequest searchRequest = new SearchRequest(request.getSearchRequest()) {
@@ -138,16 +138,17 @@ public AsyncSearchTask createTask(long id, String type, String action, TaskId pa
                 AsyncSearchId searchId = new AsyncSearchId(docID, new TaskId(nodeClient.getLocalNodeId(), id));
                 Supplier<InternalAggregation.ReduceContext> aggReduceContextSupplier =
                         () -> requestToAggReduceContextBuilder.apply(request.getSearchRequest());
-                return new AsyncSearchTask(id, type, action, parentTaskId, keepAlive, originHeaders, taskHeaders, searchId,
-                    store.getClient(), nodeClient.threadPool(), aggReduceContextSupplier);
+                return new AsyncSearchTask(id, type, action, parentTaskId,
+                    () -> submitTask.isCancelled(), keepAlive, originHeaders, taskHeaders, searchId, store.getClient(),
+                    nodeClient.threadPool(), aggReduceContextSupplier);
             }
         };
-        searchRequest.setParentTask(new TaskId(nodeClient.getLocalNodeId(), parentTaskId));
+        searchRequest.setParentTask(new TaskId(nodeClient.getLocalNodeId(), submitTask.getId()));
         return searchRequest;
     }
 
     private void onFatalFailure(AsyncSearchTask task, Exception error, boolean shouldCancel, ActionListener<AsyncSearchResponse> listener) {
-        if (shouldCancel) {
+        if (shouldCancel && task.isCancelled() == false) {
             task.cancelTask(() -> {
                 try {
                     task.addCompletionListener(finalResponse -> taskManager.unregister(task));
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
index 373030fe12efa..6e9d870e92653 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java
@@ -238,14 +238,14 @@ public void testInvalidId() throws Exception {
 
     public void testNoIndex() throws Exception {
         SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest("invalid-*");
-        request.setWaitForCompletion(TimeValue.timeValueMillis(1));
+        request.setWaitForCompletionTimeout(TimeValue.timeValueMillis(1));
         AsyncSearchResponse response = submitAsyncSearch(request);
         assertNotNull(response.getSearchResponse());
         assertFalse(response.isRunning());
         assertThat(response.getSearchResponse().getTotalShards(), equalTo(0));
 
         request = new SubmitAsyncSearchRequest("invalid");
-        request.setWaitForCompletion(TimeValue.timeValueMillis(1));
+        request.setWaitForCompletionTimeout(TimeValue.timeValueMillis(1));
         response = submitAsyncSearch(request);
         assertNull(response.getSearchResponse());
         assertNotNull(response.getFailure());
@@ -253,4 +253,28 @@ public void testNoIndex() throws Exception {
         ElasticsearchException exc = response.getFailure();
         assertThat(exc.getMessage(), containsString("no such index"));
     }
+
+    public void testCancellation() throws Exception {
+        SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(indexName);
+        request.getSearchRequest().source(
+            new SearchSourceBuilder().aggregation(new CancellingAggregationBuilder("test"))
+        );
+        request.setWaitForCompletionTimeout(TimeValue.timeValueMillis(1));
+        AsyncSearchResponse response = submitAsyncSearch(request);
+        assertNotNull(response.getSearchResponse());
+        assertTrue(response.isRunning());
+        assertThat(response.getSearchResponse().getTotalShards(), equalTo(numShards));
+        assertThat(response.getSearchResponse().getSuccessfulShards(), equalTo(0));
+        assertThat(response.getSearchResponse().getFailedShards(), equalTo(0));
+
+        response = getAsyncSearch(response.getId());
+        assertNotNull(response.getSearchResponse());
+        assertTrue(response.isRunning());
+        assertThat(response.getSearchResponse().getTotalShards(), equalTo(numShards));
+        assertThat(response.getSearchResponse().getSuccessfulShards(), equalTo(0));
+        assertThat(response.getSearchResponse().getFailedShards(), equalTo(0));
+
+        deleteAsyncSearch(response.getId());
+        ensureTaskRemoval(response.getId());
+    }
 }
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
index e5e65a832db8e..f140c218d714c 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java
@@ -5,11 +5,7 @@
  */
 package org.elasticsearch.xpack.search;
 
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.TotalHits;
-import org.apache.lucene.search.Weight;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
@@ -18,22 +14,12 @@
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.AbstractQueryBuilder;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.reindex.ReindexPlugin;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
-import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.TaskId;
@@ -48,15 +34,11 @@
 import org.elasticsearch.xpack.ilm.IndexLifecycle;
 
 import java.io.Closeable;
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -73,7 +55,7 @@ interface SearchResponseIterator extends Iterator<AsyncSearchResponse>, Closeabl
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, IndexLifecycle.class,
-            QueryBlockPlugin.class, ReindexPlugin.class);
+            SearchTestPlugin.class, ReindexPlugin.class);
     }
 
     /**
@@ -143,7 +125,7 @@ protected SearchResponseIterator assertBlockingIterator(String indexName,
                                                             int progressStep) throws Exception {
         SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(source, indexName);
         request.setBatchedReduceSize(progressStep);
-        request.setWaitForCompletion(TimeValue.timeValueMillis(1));
+        request.setWaitForCompletionTimeout(TimeValue.timeValueMillis(1));
         ClusterSearchShardsResponse response = dataNodeClient().admin().cluster()
             .prepareSearchShards(request.getSearchRequest().indices()).get();
         AtomicInteger failures = new AtomicInteger(numFailures);
@@ -152,14 +134,14 @@ protected SearchResponseIterator assertBlockingIterator(String indexName,
             .collect(
                 Collectors.toMap(
                     Function.identity(),
-                    id -> new ShardIdLatch(id, new CountDownLatch(1), failures.decrementAndGet() >= 0)
+                    id -> new ShardIdLatch(id, failures.decrementAndGet() >= 0)
                 )
             );
         ShardIdLatch[] shardLatchArray = shardLatchMap.values().stream()
-            .sorted(Comparator.comparing(ShardIdLatch::shard))
+            .sorted(Comparator.comparing(ShardIdLatch::shardId))
             .toArray(ShardIdLatch[]::new);
         resetPluginsLatch(shardLatchMap);
-        request.getSearchRequest().source().query(new BlockQueryBuilder(shardLatchMap));
+        request.getSearchRequest().source().query(new BlockingQueryBuilder(shardLatchMap));
 
         final AsyncSearchResponse initial = client().execute(SubmitAsyncSearchAction.INSTANCE, request).get();
 
@@ -197,7 +179,7 @@ private AsyncSearchResponse doNext() throws Exception {
                 int step = shardIndex == 0 ? progressStep+1 : progressStep-1;
                 int index = 0;
                 while (index < step && shardIndex < shardLatchArray.length) {
-                    if (shardLatchArray[shardIndex].shouldFail == false) {
+                    if (shardLatchArray[shardIndex].shouldFail() == false) {
                         ++index;
                     }
                     shardLatchArray[shardIndex++].countDown();
@@ -242,8 +224,8 @@ private AsyncSearchResponse doNext() throws Exception {
             @Override
             public void close() {
                 Arrays.stream(shardLatchArray).forEach(shard -> {
-                    if (shard.latch.getCount() == 1) {
-                        shard.latch.countDown();
+                    if (shard.getCount() == 1) {
+                        shard.countDown();
                     }
                 });
             }
@@ -252,143 +234,7 @@ public void close() {
 
     private void resetPluginsLatch(Map<ShardId, ShardIdLatch> newLatch) {
         for (PluginsService pluginsService : internalCluster().getDataNodeInstances(PluginsService.class)) {
-            pluginsService.filterPlugins(QueryBlockPlugin.class).forEach(p -> p.reset(newLatch));
-        }
-    }
-
-    public static class QueryBlockPlugin extends Plugin implements SearchPlugin {
-        private Map<ShardId, ShardIdLatch> shardsLatch;
-
-        public QueryBlockPlugin() {
-            this.shardsLatch = null;
-        }
-
-        public void reset(Map<ShardId, ShardIdLatch> newLatch) {
-            shardsLatch = newLatch;
-        }
-
-        @Override
-        public List<QuerySpec<?>> getQueries() {
-            return Collections.singletonList(
-                new QuerySpec<>("block_match_all",
-                    in -> new BlockQueryBuilder(in, shardsLatch),
-                    p -> BlockQueryBuilder.fromXContent(p, shardsLatch))
-            );
-        }
-    }
-
-    private static class BlockQueryBuilder extends AbstractQueryBuilder<BlockQueryBuilder> {
-        public static final String NAME = "block_match_all";
-        private final Map<ShardId, ShardIdLatch> shardsLatch;
-
-        private BlockQueryBuilder(Map<ShardId, ShardIdLatch> shardsLatch) {
-            super();
-            this.shardsLatch = shardsLatch;
-        }
-
-        BlockQueryBuilder(StreamInput in, Map<ShardId, ShardIdLatch> shardsLatch) throws IOException {
-            super(in);
-            this.shardsLatch = shardsLatch;
-        }
-
-        private BlockQueryBuilder() {
-            this.shardsLatch = null;
-        }
-
-        @Override
-        protected void doWriteTo(StreamOutput out) {}
-
-        @Override
-        protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(NAME);
-            builder.endObject();
-        }
-
-        private static final ObjectParser<BlockQueryBuilder, Void> PARSER = new ObjectParser<>(NAME, BlockQueryBuilder::new);
-
-        public static BlockQueryBuilder fromXContent(XContentParser parser, Map<ShardId, ShardIdLatch> shardsLatch) {
-            try {
-                PARSER.apply(parser, null);
-                return new BlockQueryBuilder(shardsLatch);
-            } catch (IllegalArgumentException e) {
-                throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
-            }
-        }
-
-        @Override
-        protected Query doToQuery(QueryShardContext context) {
-            final Query delegate = Queries.newMatchAllQuery();
-            return new Query() {
-                @Override
-                public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
-                    if (shardsLatch != null) {
-                        try {
-                            final ShardIdLatch latch = shardsLatch.get(new ShardId(context.index(), context.getShardId()));
-                            latch.await();
-                            if (latch.shouldFail) {
-                                throw new IOException("boum");
-                            }
-                        } catch (InterruptedException e) {
-                            throw new RuntimeException(e);
-                        }
-                    }
-                    return delegate.createWeight(searcher, scoreMode, boost);
-                }
-
-                @Override
-                public String toString(String field) {
-                    return delegate.toString(field);
-                }
-
-                @Override
-                public boolean equals(Object obj) {
-                    return false;
-                }
-
-                @Override
-                public int hashCode() {
-                    return 0;
-                }
-            };
-        }
-
-        @Override
-        protected boolean doEquals(BlockQueryBuilder other) {
-            return false;
-        }
-
-        @Override
-        protected int doHashCode() {
-            return 0;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-    }
-
-    private static class ShardIdLatch {
-        private final ShardId shard;
-        private final CountDownLatch latch;
-        private final boolean shouldFail;
-
-        private ShardIdLatch(ShardId shard, CountDownLatch latch, boolean shouldFail) {
-            this.shard = shard;
-            this.latch = latch;
-            this.shouldFail = shouldFail;
-        }
-
-        ShardId shard() {
-            return shard;
-        }
-
-        void countDown() {
-            latch.countDown();
-        }
-
-        void await() throws InterruptedException {
-            latch.await();
+            pluginsService.filterPlugins(SearchTestPlugin.class).forEach(p -> p.resetQueryLatch(newLatch));
         }
     }
 }
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
index 9b68453e5306b..c6cd15de997cf 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
@@ -10,10 +10,15 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentElasticsearchExtension;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.test.ESTestCase;
@@ -25,6 +30,8 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Date;
 import java.util.List;
 
 import static java.util.Collections.emptyList;
@@ -115,7 +122,7 @@ static SearchResponse randomSearchResponse() {
         int skippedShards = totalShards - successfulShards;
         InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
         return new SearchResponse(internalSearchResponse, null, totalShards,
-            successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+            successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
     }
 
     static void assertEqualResponses(AsyncSearchResponse expected, AsyncSearchResponse actual) {
@@ -127,4 +134,36 @@ static void assertEqualResponses(AsyncSearchResponse expected, AsyncSearchRespon
         assertEquals(expected.getStartTime(), actual.getStartTime());
         assertEquals(expected.getExpirationTime(), actual.getExpirationTime());
     }
+
+    public void testToXContent() throws IOException {
+        Date date = new Date();
+        AsyncSearchResponse asyncSearchResponse = new AsyncSearchResponse("id", true, true, date.getTime(), date.getTime());
+
+        try ( XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+            builder.prettyPrint();
+            asyncSearchResponse.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            assertEquals("{\n" +
+                "  \"id\" : \"id\",\n" +
+                "  \"is_partial\" : true,\n" +
+                "  \"is_running\" : true,\n" +
+                "  \"start_time_in_millis\" : " + date.getTime() + ",\n" +
+                "  \"expiration_time_in_millis\" : " + date.getTime() + "\n" +
+                "}", Strings.toString(builder));
+        }
+
+        try ( XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+            builder.prettyPrint();
+            builder.humanReadable(true);
+            asyncSearchResponse.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("human", "true")));
+            assertEquals("{\n" +
+                "  \"id\" : \"id\",\n" +
+                "  \"is_partial\" : true,\n" +
+                "  \"is_running\" : true,\n" +
+                "  \"start_time\" : \"" + XContentElasticsearchExtension.DEFAULT_DATE_PRINTER.print(date.getTime()) + "\",\n" +
+                "  \"start_time_in_millis\" : " + date.getTime() + ",\n" +
+                "  \"expiration_time\" : \"" + XContentElasticsearchExtension.DEFAULT_DATE_PRINTER.print(date.getTime()) + "\",\n" +
+                "  \"expiration_time_in_millis\" : " + date.getTime() + "\n" +
+                "}", Strings.toString(builder));
+        }
+    }
 }
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
index 8c3f57883ec14..b9e76ea5a0a4d 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
@@ -27,6 +28,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -35,6 +37,23 @@
 public class AsyncSearchTaskTests extends ESTestCase {
     private ThreadPool threadPool;
 
+    private static class TestTask extends CancellableTask {
+        private TestTask(long id, String type, String action, String description, TaskId parentTaskId, Map<String, String> headers) {
+            super(id, type, action, description, parentTaskId, headers);
+        }
+
+        @Override
+        public boolean shouldCancelChildrenOnCancellation() {
+            return false;
+        }
+    }
+
+    private static TestTask createSubmitTask() {
+        return new TestTask(0L, "", "", "test", new TaskId("node1", 0), Collections.emptyMap());
+    }
+
+
+
     @Before
     public void beforeTest() {
         threadPool = new TestThreadPool(getTestName());
@@ -46,7 +65,7 @@ public void afterTest() {
     }
 
     public void testWaitForInit() throws InterruptedException {
-        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
+        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), () -> false, TimeValue.timeValueHours(1),
             Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
             new NoOpClient(threadPool), threadPool, null);
         int numShards = randomIntBetween(0, 10);
@@ -86,7 +105,7 @@ public void onFailure(Exception e) {
     }
 
     public void testWithFailure() throws InterruptedException {
-        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
+        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), () -> false, TimeValue.timeValueHours(1),
             Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
             new NoOpClient(threadPool), threadPool, null);
         int numThreads = randomIntBetween(1, 10);
@@ -114,7 +133,7 @@ public void onFailure(Exception e) {
     }
 
     public void testWaitForCompletion() throws InterruptedException {
-        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1),
+        AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), () -> false, TimeValue.timeValueHours(1),
             Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)),
             new NoOpClient(threadPool), threadPool, null);
         int numShards = randomIntBetween(0, 10);
@@ -149,7 +168,7 @@ private static SearchResponse newSearchResponse(int totalShards, int successfulS
         InternalSearchResponse response = new InternalSearchResponse(SearchHits.empty(),
             InternalAggregations.EMPTY, null, null, false, null, 1);
         return new SearchResponse(response, null, totalShards, successfulShards, skippedShards,
-            100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+            100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
     }
 
     private void assertCompletionListeners(AsyncSearchTask task,
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java
new file mode 100644
index 0000000000000..5939cc10458c3
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.ShardId;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * A query builder that blocks shard execution based on the provided {@link ShardIdLatch}.
+ */
+class BlockingQueryBuilder extends AbstractQueryBuilder<BlockingQueryBuilder> {
+    public static final String NAME = "block";
+    private final Map<ShardId, ShardIdLatch> shardsLatch;
+
+    BlockingQueryBuilder(Map<ShardId, ShardIdLatch> shardsLatch) {
+        super();
+        this.shardsLatch = shardsLatch;
+    }
+
+    BlockingQueryBuilder(StreamInput in, Map<ShardId, ShardIdLatch> shardsLatch) throws IOException {
+        super(in);
+        this.shardsLatch = shardsLatch;
+    }
+
+    BlockingQueryBuilder() {
+        this.shardsLatch = null;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) {}
+
+    @Override
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(NAME);
+        builder.endObject();
+    }
+
+    private static final ObjectParser<BlockingQueryBuilder, Void> PARSER = new ObjectParser<>(NAME, BlockingQueryBuilder::new);
+
+    public static BlockingQueryBuilder fromXContent(XContentParser parser, Map<ShardId, ShardIdLatch> shardsLatch) {
+        try {
+            PARSER.apply(parser, null);
+            return new BlockingQueryBuilder(shardsLatch);
+        } catch (IllegalArgumentException e) {
+            throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
+        }
+    }
+
+    @Override
+    protected Query doToQuery(QueryShardContext context) {
+        final Query delegate = Queries.newMatchAllQuery();
+        return new Query() {
+            @Override
+            public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+                if (shardsLatch != null) {
+                    try {
+                        final ShardIdLatch latch = shardsLatch.get(new ShardId(context.index(), context.getShardId()));
+                        latch.await();
+                        if (latch.shouldFail()) {
+                            throw new IOException("boum");
+                        }
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+                return delegate.createWeight(searcher, scoreMode, boost);
+            }
+
+            @Override
+            public String toString(String field) {
+                return delegate.toString(field);
+            }
+
+            @Override
+            public boolean equals(Object obj) {
+                return false;
+            }
+
+            @Override
+            public int hashCode() {
+                return 0;
+            }
+        };
+    }
+
+    @Override
+    protected boolean doEquals(BlockingQueryBuilder other) {
+        return false;
+    }
+
+    @Override
+    protected int doHashCode() {
+        return 0;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java
new file mode 100644
index 0000000000000..354c3c4ec340a
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An aggregation builder that blocks shard search action until the task is cancelled.
+ */
+public class CancellingAggregationBuilder extends AbstractAggregationBuilder<CancellingAggregationBuilder> {
+    static final String NAME = "cancel";
+    static final int SLEEP_TIME = 10;
+
+    public CancellingAggregationBuilder(String name) {
+        super(name);
+    }
+
+    public CancellingAggregationBuilder(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map<String, Object> metaData) {
+        return new CancellingAggregationBuilder(name);
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+    }
+
+    @Override
+    protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    static final ConstructingObjectParser<CancellingAggregationBuilder, String> PARSER =
+        new ConstructingObjectParser<>(NAME, false, (args, name) -> new CancellingAggregationBuilder(name));
+
+
+    static CancellingAggregationBuilder fromXContent(String aggName, XContentParser parser) {
+        try {
+            return PARSER.apply(parser, aggName);
+        } catch (IllegalArgumentException e) {
+            throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
+        }
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected AggregatorFactory doBuild(QueryShardContext queryShardContext, AggregatorFactory parent,
+                                        AggregatorFactories.Builder subfactoriesBuilder) throws IOException {
+        final FilterAggregationBuilder filterAgg = new FilterAggregationBuilder(name, QueryBuilders.matchAllQuery());
+        filterAgg.subAggregations(subfactoriesBuilder);
+        final AggregatorFactory factory = filterAgg.build(queryShardContext, parent);
+        return new AggregatorFactory(name, queryShardContext, parent, subfactoriesBuilder, metaData) {
+            @Override
+            protected Aggregator createInternal(SearchContext searchContext,
+                                                Aggregator parent,
+                                                boolean collectsFromSingleBucket,
+                                                List<PipelineAggregator> pipelineAggregators,
+                                                Map<String, Object> metaData) throws IOException {
+                while (searchContext.isCancelled() == false) {
+                    try {
+                        Thread.sleep(SLEEP_TIME);
+                    } catch (InterruptedException e) {
+                        throw new IOException(e);
+                    }
+                }
+                return factory.create(searchContext, parent, collectsFromSingleBucket);
+            }
+        };
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SearchTestPlugin.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SearchTestPlugin.java
new file mode 100644
index 0000000000000..88daac239fad1
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SearchTestPlugin.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class SearchTestPlugin extends Plugin implements SearchPlugin {
+    private Map<ShardId, ShardIdLatch> shardsLatch;
+
+    public SearchTestPlugin() {
+        this.shardsLatch = null;
+    }
+
+    public void resetQueryLatch(Map<ShardId, ShardIdLatch> newLatch) {
+        shardsLatch = newLatch;
+    }
+
+    @Override
+    public List<QuerySpec<?>> getQueries() {
+        return Collections.singletonList(
+            new QuerySpec<>(BlockingQueryBuilder.NAME,
+                in -> new BlockingQueryBuilder(in, shardsLatch),
+                p -> BlockingQueryBuilder.fromXContent(p, shardsLatch))
+        );
+    }
+
+    @Override
+    public List<AggregationSpec> getAggregations() {
+        return Collections.singletonList(new AggregationSpec(CancellingAggregationBuilder.NAME, CancellingAggregationBuilder::new,
+            CancellingAggregationBuilder.PARSER).addResultReader(InternalFilter::new));
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ShardIdLatch.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ShardIdLatch.java
new file mode 100644
index 0000000000000..dd171b3102894
--- /dev/null
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ShardIdLatch.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.index.shard.ShardId;
+
+import java.util.concurrent.CountDownLatch;
+
+class ShardIdLatch extends CountDownLatch {
+    private final ShardId shard;
+    private final boolean shouldFail;
+
+    ShardIdLatch(ShardId shard, boolean shouldFail) {
+        super(1);
+        this.shard = shard;
+        this.shouldFail = shouldFail;
+    }
+
+    ShardId shardId() {
+        return shard;
+    }
+
+    boolean shouldFail() {
+        return shouldFail;
+    }
+}
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
index 2aae3817205e2..4086e8d19df40 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
@@ -35,9 +35,9 @@ protected SubmitAsyncSearchRequest createTestInstance() {
             searchRequest = new SubmitAsyncSearchRequest();
         }
         if (randomBoolean()) {
-            searchRequest.setWaitForCompletion(TimeValue.parseTimeValue(randomPositiveTimeValue(), "wait_for_completion"));
+            searchRequest.setWaitForCompletionTimeout(TimeValue.parseTimeValue(randomPositiveTimeValue(), "wait_for_completion"));
         }
-        searchRequest.setCleanOnCompletion(randomBoolean());
+        searchRequest.setKeepOnCompletion(randomBoolean());
         if (randomBoolean()) {
             searchRequest.setKeepAlive(TimeValue.parseTimeValue(randomPositiveTimeValue(), "keep_alive"));
         }
@@ -109,4 +109,13 @@ public void testValidateSuggestOnly() {
         assertThat(exc.validationErrors().size(), equalTo(1));
         assertThat(exc.validationErrors().get(0), containsString("suggest"));
     }
+
+    public void testValidatePreFilterShardSize() {
+        SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest();
+        req.getSearchRequest().setPreFilterShardSize(randomIntBetween(2, Integer.MAX_VALUE));
+        ActionRequestValidationException exc = req.validate();
+        assertNotNull(exc);
+        assertThat(exc.validationErrors().size(), equalTo(1));
+        assertThat(exc.validationErrors().get(0), containsString("[pre_filter_shard_size]"));
+    }
 }
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecision.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecision.java
new file mode 100644
index 0000000000000..4f4a3ce70981b
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecision.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Represents an autoscaling decision.
+ */
+public class AutoscalingDecision implements ToXContent, Writeable {
+
+    private final String name;
+
+    public String name() {
+        return name;
+    }
+
+    private final AutoscalingDecisionType type;
+
+    public AutoscalingDecisionType type() {
+        return type;
+    }
+
+    private final String reason;
+
+    public String reason() {
+        return reason;
+    }
+
+    public AutoscalingDecision(final String name, final AutoscalingDecisionType type, final String reason) {
+        this.name = Objects.requireNonNull(name);
+        this.type = Objects.requireNonNull(type);
+        this.reason = Objects.requireNonNull(reason);
+    }
+
+    public AutoscalingDecision(final StreamInput in) throws IOException {
+        this.name = in.readString();
+        this.type = AutoscalingDecisionType.readFrom(in);
+        this.reason = in.readString();
+    }
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        out.writeString(name);
+        type.writeTo(out);
+        out.writeString(reason);
+    }
+
+    @Override
+    public XContentBuilder toXContent(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field("name", name);
+            builder.field("type", type);
+            builder.field("reason", reason);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final AutoscalingDecision that = (AutoscalingDecision) o;
+        return name.equals(that.name) && type == that.type && reason.equals(that.reason);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, type, reason);
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionType.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionType.java
new file mode 100644
index 0000000000000..8590e26426291
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionType.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Locale;
+
+/**
+ * Represents the type of an autoscaling decision: to indicating if a scale down, no scaling event, or a scale up is needed.
+ */
+public enum AutoscalingDecisionType implements Writeable, ToXContentFragment {
+
+    /**
+     * Indicates that a scale down event is needed.
+     */
+    SCALE_DOWN((byte) 0),
+
+    /**
+     * Indicates that no scaling event is needed.
+     */
+    NO_SCALE((byte) 1),
+
+    /**
+     * Indicates that a scale up event is needed.
+     */
+    SCALE_UP((byte) 2);
+
+    private final byte id;
+
+    byte id() {
+        return id;
+    }
+
+    AutoscalingDecisionType(final byte id) {
+        this.id = id;
+    }
+
+    public static AutoscalingDecisionType readFrom(final StreamInput in) throws IOException {
+        final byte id = in.readByte();
+        switch (id) {
+            case 0:
+                return SCALE_DOWN;
+            case 1:
+                return NO_SCALE;
+            case 2:
+                return SCALE_UP;
+            default:
+                throw new IllegalArgumentException("unexpected value [" + id + "] for autoscaling decision type");
+        }
+    }
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        out.writeByte(id);
+    }
+
+    @Override
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        builder.value(name().toLowerCase(Locale.ROOT));
+        return builder;
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisions.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisions.java
new file mode 100644
index 0000000000000..5e7e23fcc35dd
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisions.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Objects;
+
+/**
+ * Represents a collection of individual autoscaling decisions that can be aggregated into a single autoscaling decision.
+ */
+public class AutoscalingDecisions implements ToXContent, Writeable {
+
+    private final Collection<AutoscalingDecision> decisions;
+
+    public AutoscalingDecisions(final Collection<AutoscalingDecision> decisions) {
+        Objects.requireNonNull(decisions);
+        if (decisions.isEmpty()) {
+            throw new IllegalArgumentException("decisions can not be empty");
+        }
+        this.decisions = decisions;
+    }
+
+    public AutoscalingDecisions(final StreamInput in) throws IOException {
+        this.decisions = in.readList(AutoscalingDecision::new);
+    }
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        out.writeCollection(decisions);
+    }
+
+    @Override
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        for (final AutoscalingDecision decision : decisions) {
+            decision.toXContent(builder, params);
+        }
+        return builder;
+    }
+
+    public AutoscalingDecisionType type() {
+        if (decisions.stream().anyMatch(p -> p.type() == AutoscalingDecisionType.SCALE_UP)) {
+            // if any deciders say to scale up
+            return AutoscalingDecisionType.SCALE_UP;
+        } else if (decisions.stream().allMatch(p -> p.type() == AutoscalingDecisionType.SCALE_DOWN)) {
+            // if all deciders say to scale down
+            return AutoscalingDecisionType.SCALE_DOWN;
+        } else {
+            // otherwise, do not scale
+            return AutoscalingDecisionType.NO_SCALE;
+        }
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final org.elasticsearch.xpack.autoscaling.AutoscalingDecisions that = (org.elasticsearch.xpack.autoscaling.AutoscalingDecisions) o;
+        return decisions.equals(that.decisions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(decisions);
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingDecisionAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingDecisionAction.java
index 88ffc0fbc91c7..8e97eb0d3f64e 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingDecisionAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingDecisionAction.java
@@ -14,8 +14,13 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.autoscaling.AutoscalingDecisions;
 
 import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.SortedMap;
+import java.util.TreeMap;
 
 public class GetAutoscalingDecisionAction extends ActionType<GetAutoscalingDecisionAction.Response> {
 
@@ -60,24 +65,37 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
 
     public static class Response extends ActionResponse implements ToXContentObject {
 
-        public Response() {
+        private final SortedMap<String, AutoscalingDecisions> decisions;
 
+        public Response(final SortedMap<String, AutoscalingDecisions> decisions) {
+            this.decisions = Objects.requireNonNull(decisions);
         }
 
         public Response(final StreamInput in) throws IOException {
             super(in);
+            decisions = new TreeMap<>(in.readMap(StreamInput::readString, AutoscalingDecisions::new));
         }
 
         @Override
-        public void writeTo(final StreamOutput out) {
-
+        public void writeTo(final StreamOutput out) throws IOException {
+            out.writeMap(decisions, StreamOutput::writeString, (o, decision) -> decision.writeTo(o));
         }
 
         @Override
         public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
             builder.startObject();
             {
-
+                builder.startArray("decisions");
+                {
+                    for (final Map.Entry<String, AutoscalingDecisions> decision : decisions.entrySet()) {
+                        builder.startObject();
+                        {
+                            builder.field(decision.getKey(), decision.getValue());
+                        }
+                        builder.endObject();
+                    }
+                }
+                builder.endArray();
             }
             builder.endObject();
             return builder;
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingDecisionAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingDecisionAction.java
index 5f6b79358756d..ea1687239b74f 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingDecisionAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingDecisionAction.java
@@ -20,6 +20,8 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.TreeMap;
 
 public class TransportGetAutoscalingDecisionAction extends TransportMasterNodeAction<
     GetAutoscalingDecisionAction.Request,
@@ -61,7 +63,7 @@ protected void masterOperation(
         final ClusterState state,
         final ActionListener<GetAutoscalingDecisionAction.Response> listener
     ) {
-        listener.onResponse(new GetAutoscalingDecisionAction.Response());
+        listener.onResponse(new GetAutoscalingDecisionAction.Response(Collections.unmodifiableSortedMap(new TreeMap<>())));
     }
 
     @Override
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTests.java
new file mode 100644
index 0000000000000..c6c8dfa94d114
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTests.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AutoscalingDecisionTests extends AutoscalingTestCase {
+
+    public void testAutoscalingDecisionType() {
+        final AutoscalingDecisionType type = randomFrom(AutoscalingDecisionType.values());
+        final AutoscalingDecision decision = randomAutoscalingDecisionOfType(type);
+        assertThat(decision.type(), equalTo(type));
+    }
+
+    public void testAutoscalingDecisionTypeSerialization() throws IOException {
+        final AutoscalingDecisionType before = randomFrom(AutoscalingDecisionType.values());
+        final BytesStreamOutput out = new BytesStreamOutput();
+        before.writeTo(out);
+        final AutoscalingDecisionType after = AutoscalingDecisionType.readFrom(out.bytes().streamInput());
+        assertThat(after, equalTo(before));
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTypeWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTypeWireSerializingTests.java
new file mode 100644
index 0000000000000..caa5c6440bbb4
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionTypeWireSerializingTests.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AutoscalingDecisionTypeWireSerializingTests extends AbstractWireSerializingTestCase<AutoscalingDecisionType> {
+
+    @Override
+    protected Writeable.Reader<AutoscalingDecisionType> instanceReader() {
+        return AutoscalingDecisionType::readFrom;
+    }
+
+    @Override
+    protected AutoscalingDecisionType createTestInstance() {
+        return randomFrom(AutoscalingDecisionType.values());
+    }
+
+    @Override
+    protected void assertEqualInstances(final AutoscalingDecisionType expectedInstance, final AutoscalingDecisionType newInstance) {
+        assertSame(expectedInstance, newInstance);
+        assertEquals(expectedInstance, newInstance);
+        assertEquals(expectedInstance.hashCode(), newInstance.hashCode());
+    }
+
+    public void testInvalidAutoscalingDecisionTypeSerialization() throws IOException {
+        final BytesStreamOutput out = new BytesStreamOutput();
+        final Set<Byte> values = Arrays.stream(AutoscalingDecisionType.values())
+            .map(AutoscalingDecisionType::id)
+            .collect(Collectors.toSet());
+        final byte value = randomValueOtherThanMany(values::contains, ESTestCase::randomByte);
+        out.writeByte(value);
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> AutoscalingDecisionType.readFrom(out.bytes().streamInput())
+        );
+        assertThat(e.getMessage(), equalTo("unexpected value [" + value + "] for autoscaling decision type"));
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionWireSerializingTests.java
new file mode 100644
index 0000000000000..f24dbcabdd090
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionWireSerializingTests.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+public class AutoscalingDecisionWireSerializingTests extends AbstractWireSerializingTestCase<AutoscalingDecision> {
+
+    @Override
+    protected Writeable.Reader<AutoscalingDecision> instanceReader() {
+        return AutoscalingDecision::new;
+    }
+
+    @Override
+    protected AutoscalingDecision createTestInstance() {
+        return AutoscalingTestCase.randomAutoscalingDecision();
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsTests.java
new file mode 100644
index 0000000000000..c397f8650f7b6
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsTests.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AutoscalingDecisionsTests extends AutoscalingTestCase {
+
+    public void testAutoscalingDecisionsRejectsEmptyDecisions() {
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new AutoscalingDecisions(List.of()));
+        assertThat(e.getMessage(), equalTo("decisions can not be empty"));
+    }
+
+    public void testAutoscalingDecisionsTypeDown() {
+        final AutoscalingDecisions decisions = randomAutoscalingDecisions(randomIntBetween(1, 8), 0, 0);
+        assertThat(decisions.type(), equalTo(AutoscalingDecisionType.SCALE_DOWN));
+    }
+
+    public void testAutoscalingDecisionsTypeNo() {
+        final AutoscalingDecisions decision = randomAutoscalingDecisions(randomIntBetween(0, 8), randomIntBetween(1, 8), 0);
+        assertThat(decision.type(), equalTo(AutoscalingDecisionType.NO_SCALE));
+    }
+
+    public void testAutoscalingDecisionsTypeUp() {
+        final AutoscalingDecisions decision = randomAutoscalingDecisions(0, 0, randomIntBetween(1, 8));
+        assertThat(decision.type(), equalTo(AutoscalingDecisionType.SCALE_UP));
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsWireSerializingTests.java
new file mode 100644
index 0000000000000..c058d7c81d5cf
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingDecisionsWireSerializingTests.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+public class AutoscalingDecisionsWireSerializingTests extends AbstractWireSerializingTestCase<AutoscalingDecisions> {
+
+    @Override
+    protected Writeable.Reader<AutoscalingDecisions> instanceReader() {
+        return AutoscalingDecisions::new;
+    }
+
+    @Override
+    protected AutoscalingDecisions createTestInstance() {
+        return AutoscalingTestCase.randomAutoscalingDecisions();
+    }
+
+}
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java
new file mode 100644
index 0000000000000..29b9a3c41bda9
--- /dev/null
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.autoscaling;
+
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class AutoscalingTestCase extends ESTestCase {
+
+    static AutoscalingDecision randomAutoscalingDecision() {
+        return randomAutoscalingDecisionOfType(randomFrom(AutoscalingDecisionType.values()));
+    }
+
+    static AutoscalingDecision randomAutoscalingDecisionOfType(final AutoscalingDecisionType type) {
+        return new AutoscalingDecision(randomAlphaOfLength(8), type, randomAlphaOfLength(8));
+    }
+
+    static AutoscalingDecisions randomAutoscalingDecisions() {
+        final int numberOfDecisions = 1 + randomIntBetween(1, 8);
+        final List<AutoscalingDecision> decisions = new ArrayList<>(numberOfDecisions);
+        for (int i = 0; i < numberOfDecisions; i++) {
+            decisions.add(randomAutoscalingDecisionOfType(AutoscalingDecisionType.SCALE_DOWN));
+        }
+        final int numberOfDownDecisions = randomIntBetween(0, 8);
+        final int numberOfNoDecisions = randomIntBetween(0, 8);
+        final int numberOfUpDecisions = randomIntBetween(numberOfDownDecisions + numberOfNoDecisions == 0 ? 1 : 0, 8);
+        return randomAutoscalingDecisions(numberOfDownDecisions, numberOfNoDecisions, numberOfUpDecisions);
+    }
+
+    static AutoscalingDecisions randomAutoscalingDecisions(
+        final int numberOfDownDecisions,
+        final int numberOfNoDecisions,
+        final int numberOfUpDecisions
+    ) {
+        final List<AutoscalingDecision> decisions = new ArrayList<>(numberOfDownDecisions + numberOfNoDecisions + numberOfUpDecisions);
+        for (int i = 0; i < numberOfDownDecisions; i++) {
+            decisions.add(randomAutoscalingDecisionOfType(AutoscalingDecisionType.SCALE_DOWN));
+        }
+        for (int i = 0; i < numberOfNoDecisions; i++) {
+            decisions.add(randomAutoscalingDecisionOfType(AutoscalingDecisionType.NO_SCALE));
+        }
+        for (int i = 0; i < numberOfUpDecisions; i++) {
+            decisions.add(randomAutoscalingDecisionOfType(AutoscalingDecisionType.SCALE_UP));
+        }
+        Randomness.shuffle(decisions);
+        return new AutoscalingDecisions(decisions);
+    }
+
+}
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
index 7f4c045f22643..d4203e335994c 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
@@ -294,8 +294,8 @@ public boolean isReadOnly() {
 
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                              Map<String, Object> userMetadata, ActionListener<String> listener) {
+                              IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                              Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
         throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE);
     }
 
@@ -487,7 +487,7 @@ void restoreFiles(Store store, ActionListener<Void> listener) {
                 ByteSizeValue fileSize = new ByteSizeValue(fileMetaData.length());
                 fileInfos.add(new FileInfo(fileMetaData.name(), fileMetaData, fileSize));
             }
-            SnapshotFiles snapshotFiles = new SnapshotFiles(LATEST, fileInfos);
+            SnapshotFiles snapshotFiles = new SnapshotFiles(LATEST, fileInfos, null);
             restore(snapshotFiles, store, listener);
         }
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
index 99535d59415ff..c3152de55aae3 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
@@ -992,6 +992,7 @@ public void testLeaderIndexSettingNotPercolatedToFollower() throws Exception {
         assertThat(hasFollowIndexBeenClosedChecker.getAsBoolean(), is(false));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/54204")
     public void testUpdateAnalysisLeaderIndexSettings() throws Exception {
         final String leaderIndexSettings = getIndexSettings(1, 0);
         assertAcked(leaderClient().admin().indices().prepareCreate("leader").setSource(leaderIndexSettings, XContentType.JSON));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
index 3b80e605a2931..c11d7fc6b9e5a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
@@ -586,7 +586,7 @@ public boolean isFlattenedAllowed() {
     public boolean isVectorsAllowed() {
         return allowForAllLicenses();
     }
-    
+
 
     /**
      * Determine if Wildcard support should be enabled.
@@ -595,7 +595,7 @@ public boolean isVectorsAllowed() {
      */
     public synchronized boolean isWildcardAllowed() {
         return status.active;
-    }    
+    }
 
     public boolean isOdbcAllowed() {
         return isAllowedByLicense(OperationMode.PLATINUM);
@@ -621,22 +621,11 @@ public boolean isSecurityAvailable() {
     }
 
     /**
-     * @return true if security has been disabled due it being the default setting for this license type.
-     *  The conditions necessary for this are:
-     *         <ul>
-     *             <li>A trial or basic license</li>
-     *             <li>xpack.security.enabled not specified as a setting</li>
-     *         </ul>
+     * Returns whether security is enabled, taking into account the default enabled state
+     * based on the current license level.
      */
-    public boolean isSecurityDisabledByLicenseDefaults() {
-        return checkAgainstStatus(status -> {
-            switch (status.mode) {
-                case TRIAL:
-                case BASIC:
-                    return isSecurityEnabled && isSecurityExplicitlyEnabled == false;
-            }
-            return false;
-        });
+    public boolean isSecurityEnabled() {
+        return isSecurityEnabled(status.mode, isSecurityExplicitlyEnabled, isSecurityEnabled);
     }
 
     public static boolean isTransportTlsRequired(License license, Settings settings) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
index e65fbdace62d6..7fa8bd27b4cfc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
@@ -120,8 +120,8 @@ private static MetaData metadataToSnapshot(Collection<IndexId> indices, MetaData
 
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
-                              IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, Version repositoryMetaVersion,
-                              Map<String, Object> userMetadata, ActionListener<String> listener) {
+                              IndexCommit snapshotIndexCommit, String shardStateIdentifier, IndexShardSnapshotStatus snapshotStatus,
+                              Version repositoryMetaVersion, Map<String, Object> userMetadata, ActionListener<String> listener) {
         if (mapperService.documentMapper() != null // if there is no mapping this is null
             && mapperService.documentMapper().sourceMapper().isComplete() == false) {
             listener.onFailure(
@@ -161,8 +161,8 @@ protected void closeInternal() {
                 Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name()));
             toClose.add(reader);
             IndexCommit indexCommit = reader.getIndexCommit();
-            super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, snapshotStatus, repositoryMetaVersion,
-                userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose)));
+            super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, shardStateIdentifier, snapshotStatus,
+                repositoryMetaVersion, userMetadata, ActionListener.runBefore(listener, () -> IOUtils.close(toClose)));
         } catch (IOException e) {
             try {
                 IOUtils.close(toClose);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
index ed03ab9ffc058..49a457f0a7f55 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
@@ -53,6 +53,7 @@ public final class ClientHelper {
     public static final String ENRICH_ORIGIN = "enrich";
     public static final String TRANSFORM_ORIGIN = "transform";
     public static final String ASYNC_SEARCH_ORIGIN = "async_search";
+    public static final String IDP_ORIGIN = "idp";
 
     private ClientHelper() {}
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java
index 1a292bbebeadd..c9b9d54896dd0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java
@@ -50,7 +50,7 @@ public final class SourceDestValidator {
     public static final String DEST_LOWERCASE = "Destination index [{0}] must be lowercase";
     public static final String NEEDS_REMOTE_CLUSTER_SEARCH = "Source index is configured with a remote index pattern(s) [{0}]"
         + " but the current node [{1}] is not allowed to connect to remote clusters."
-        + " Please enable cluster.remote.connect for all data nodes.";
+        + " Please enable remote.cluster_client for all data nodes.";
     public static final String ERROR_REMOTE_CLUSTER_SEARCH = "Error resolving remote source: {0}";
     public static final String UNKNOWN_REMOTE_CLUSTER_LICENSE = "Error during license check ({0}) for remote cluster "
         + "alias(es) {1}, error: {2}";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
index 52aa9304ce264..fef4592d75067 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
@@ -347,7 +347,7 @@ private void onSearchResponse(SearchResponse searchResponse) {
 
             // allowPartialSearchResults is set to false, so we should never see shard failures here
             assert (searchResponse.getShardFailures().length == 0);
-
+            stats.markStartProcessing();
             stats.incrementNumPages(1);
             IterationResult<JobPosition> iterationResult = doProcess(searchResponse);
 
@@ -355,11 +355,11 @@ private void onSearchResponse(SearchResponse searchResponse) {
                 logger.debug("Finished indexing for job [" + getJobId() + "], saving state and shutting down.");
 
                 position.set(iterationResult.getPosition());
+                stats.markEndProcessing();
                 // execute finishing tasks
                 onFinish(ActionListener.wrap(
                         r -> doSaveState(finishAndSetState(), position.get(), () -> {}),
                         e -> doSaveState(finishAndSetState(), position.get(), () -> {})));
-
                 return;
             }
 
@@ -369,7 +369,7 @@ private void onSearchResponse(SearchResponse searchResponse) {
             if (docs.isEmpty() == false) {
                 final BulkRequest bulkRequest = new BulkRequest();
                 docs.forEach(bulkRequest::add);
-
+                stats.markEndProcessing();
                 stats.markStartIndexing();
                 doNextBulk(bulkRequest, ActionListener.wrap(bulkResponse -> {
                     // TODO we should check items in the response and move after accordingly to
@@ -390,6 +390,7 @@ private void onSearchResponse(SearchResponse searchResponse) {
                     onBulkResponse(bulkResponse, newPosition);
                 }, this::finishWithIndexingFailure));
             } else {
+                stats.markEndProcessing();
                 // no documents need to be indexed, continue with search
                 try {
                     JobPosition newPosition = iterationResult.getPosition();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
index 72134b0553b98..927c71ce8e10c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.indexing;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -31,26 +32,31 @@ public abstract class IndexerJobStats implements ToXContentObject, Writeable {
     protected long searchTime = 0;
     protected long indexTotal = 0;
     protected long searchTotal = 0;
+    protected long processingTime = 0;
+    protected long processingTotal = 0;
     protected long indexFailures = 0;
     protected long searchFailures = 0;
 
     private long startIndexTime;
     private long startSearchTime;
+    private long startProcessingTime;
 
     public IndexerJobStats() {
     }
 
     public IndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations,
-                           long indexTime, long searchTime, long indexTotal, long searchTotal,
-                           long indexFailures, long searchFailures) {
+                           long indexTime, long searchTime, long processingTime, long indexTotal, long searchTotal,
+                           long processingTotal, long indexFailures, long searchFailures) {
         this.numPages = numPages;
         this.numInputDocuments = numInputDocuments;
         this.numOuputDocuments = numOuputDocuments;
         this.numInvocations = numInvocations;
         this.indexTime = indexTime;
         this.searchTime = searchTime;
+        this.processingTime = processingTime;
         this.indexTotal = indexTotal;
         this.searchTotal = searchTotal;
+        this.processingTotal = processingTotal;
         this.indexFailures = indexFailures;
         this.searchFailures = searchFailures;
     }
@@ -66,6 +72,11 @@ public IndexerJobStats(StreamInput in) throws IOException {
         this.searchTotal = in.readVLong();
         this.indexFailures = in.readVLong();
         this.searchFailures = in.readVLong();
+
+        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+            this.processingTime = in.readVLong();
+            this.processingTotal = in.readVLong();
+        }
     }
 
     public long getNumPages() {
@@ -100,6 +111,10 @@ public long getSearchTime() {
         return searchTime;
     }
 
+    public long getProcessingTime() {
+        return processingTime;
+    }
+
     public long getIndexTotal() {
         return indexTotal;
     }
@@ -108,6 +123,10 @@ public long getSearchTotal() {
         return searchTotal;
     }
 
+    public long getProcessingTotal() {
+        return processingTotal;
+    }
+
     public void incrementNumPages(long n) {
         assert(n >= 0);
         numPages += n;
@@ -154,6 +173,15 @@ public void markEndSearch() {
         searchTotal += 1;
     }
 
+    public void markStartProcessing() {
+        this.startProcessingTime = System.nanoTime();
+    }
+
+    public void markEndProcessing() {
+        processingTime += ((System.nanoTime() - startProcessingTime) / 1000000);
+        processingTotal += 1;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(numPages);
@@ -166,6 +194,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(searchTotal);
         out.writeVLong(indexFailures);
         out.writeVLong(searchFailures);
+        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+            out.writeVLong(processingTime);
+            out.writeVLong(processingTotal);
+        }
     }
 
     @Override
@@ -186,15 +218,17 @@ public boolean equals(Object other) {
             && Objects.equals(this.numInvocations, that.numInvocations)
             && Objects.equals(this.indexTime, that.indexTime)
             && Objects.equals(this.searchTime, that.searchTime)
+            && Objects.equals(this.processingTime, that.processingTime)
             && Objects.equals(this.indexFailures, that.indexFailures)
             && Objects.equals(this.searchFailures, that.searchFailures)
             && Objects.equals(this.indexTotal, that.indexTotal)
-            && Objects.equals(this.searchTotal, that.searchTotal);
+            && Objects.equals(this.searchTotal, that.searchTotal)
+            && Objects.equals(this.processingTotal, that.processingTotal);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
-            indexTime, searchTime, indexFailures, searchFailures, indexTotal, searchTotal);
+            indexTime, searchTime, processingTime, indexFailures, searchFailures, indexTotal, searchTotal, processingTotal);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
index 5be6ffe2878d2..fa20e805cc589 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
@@ -132,14 +132,18 @@ public static JobState getJobStateModifiedForReassignments(@Nullable PersistentT
 
     public static DatafeedState getDatafeedState(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) {
         PersistentTasksCustomMetaData.PersistentTask<?> task = getDatafeedTask(datafeedId, tasks);
-        // TODO: report (task != null && task.getState() == null) as STARTING in version 8, and fix side effects
-        if (task != null && task.getState() != null) {
-            return (DatafeedState) task.getState();
-        } else {
+        if (task == null) {
             // If we haven't started a datafeed then there will be no persistent task,
             // which is the same as if the datafeed was't started
             return DatafeedState.STOPPED;
         }
+        DatafeedState taskState = (DatafeedState) task.getState();
+        if (taskState == null) {
+            // If we haven't set a state yet then the task has never been assigned, so
+            // report that it's starting
+            return DatafeedState.STARTING;
+        }
+        return taskState;
     }
 
     public static DataFrameAnalyticsState getDataFrameAnalyticsState(String analyticsId, @Nullable PersistentTasksCustomMetaData tasks) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
index 209058e00468b..e37ccbbefc5b1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
@@ -165,6 +166,9 @@ public static class Stats implements ToXContentObject, Writeable {
              */
             private final List<PhaseProgress> progress;
 
+            @Nullable
+            private final DataCounts dataCounts;
+
             @Nullable
             private final MemoryUsage memoryUsage;
 
@@ -177,12 +181,13 @@ public static class Stats implements ToXContentObject, Writeable {
             private final String assignmentExplanation;
 
             public Stats(String id, DataFrameAnalyticsState state, @Nullable String failureReason, List<PhaseProgress> progress,
-                         @Nullable MemoryUsage memoryUsage, @Nullable AnalysisStats analysisStats, @Nullable DiscoveryNode node,
-                         @Nullable String assignmentExplanation) {
+                         @Nullable DataCounts dataCounts, @Nullable MemoryUsage memoryUsage, @Nullable AnalysisStats analysisStats,
+                         @Nullable DiscoveryNode node, @Nullable String assignmentExplanation) {
                 this.id = Objects.requireNonNull(id);
                 this.state = Objects.requireNonNull(state);
                 this.failureReason = failureReason;
                 this.progress = Objects.requireNonNull(progress);
+                this.dataCounts = dataCounts;
                 this.memoryUsage = memoryUsage;
                 this.analysisStats = analysisStats;
                 this.node = node;
@@ -198,6 +203,11 @@ public Stats(StreamInput in) throws IOException {
                 } else {
                     progress = in.readList(PhaseProgress::new);
                 }
+                if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+                    dataCounts = in.readOptionalWriteable(DataCounts::new);
+                } else {
+                    dataCounts = null;
+                }
                 if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
                     memoryUsage = in.readOptionalWriteable(MemoryUsage::new);
                 } else {
@@ -261,6 +271,11 @@ public List<PhaseProgress> getProgress() {
                 return progress;
             }
 
+            @Nullable
+            public DataCounts getDataCounts() {
+                return dataCounts;
+            }
+
             @Nullable
             public MemoryUsage getMemoryUsage() {
                 return memoryUsage;
@@ -293,6 +308,9 @@ public XContentBuilder toUnwrappedXContent(XContentBuilder builder) throws IOExc
                 if (progress != null) {
                     builder.field("progress", progress);
                 }
+                if (dataCounts != null) {
+                    builder.field("data_counts", dataCounts);
+                }
                 if (memoryUsage != null) {
                     builder.field("memory_usage", memoryUsage);
                 }
@@ -331,6 +349,9 @@ public void writeTo(StreamOutput out) throws IOException {
                 } else {
                     out.writeList(progress);
                 }
+                if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+                    out.writeOptionalWriteable(dataCounts);
+                }
                 if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
                     out.writeOptionalWriteable(memoryUsage);
                 }
@@ -369,7 +390,8 @@ private void writeProgressToLegacy(StreamOutput out) throws IOException {
 
             @Override
             public int hashCode() {
-                return Objects.hash(id, state, failureReason, progress, memoryUsage, analysisStats, node, assignmentExplanation);
+                return Objects.hash(id, state, failureReason, progress, dataCounts, memoryUsage, analysisStats, node,
+                    assignmentExplanation);
             }
 
             @Override
@@ -385,6 +407,7 @@ public boolean equals(Object obj) {
                         && Objects.equals(this.state, other.state)
                         && Objects.equals(this.failureReason, other.failureReason)
                         && Objects.equals(this.progress, other.progress)
+                        && Objects.equals(this.dataCounts, other.dataCounts)
                         && Objects.equals(this.memoryUsage, other.memoryUsage)
                         && Objects.equals(this.analysisStats, other.analysisStats)
                         && Objects.equals(this.node, other.node)
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
index ef0b7e19d100e..f71819e2a732a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java
@@ -54,7 +54,7 @@ public class Classification implements DataFrameAnalysis {
     /**
      * The max number of classes classification supports
      */
-    private static final int MAX_DEPENDENT_VARIABLE_CARDINALITY = 30;
+    public static final int MAX_DEPENDENT_VARIABLE_CARDINALITY = 30;
 
     private static ConstructingObjectParser<Classification, Void> createParser(boolean lenient) {
         ConstructingObjectParser<Classification, Void> parser = new ConstructingObjectParser<>(
@@ -286,9 +286,11 @@ public List<FieldCardinalityConstraint> getFieldCardinalityConstraints() {
     @SuppressWarnings("unchecked")
     @Override
     public Map<String, Object> getExplicitlyMappedFields(Map<String, Object> mappingsProperties, String resultsFieldName) {
+        Map<String, Object> additionalProperties = new HashMap<>();
+        additionalProperties.put(resultsFieldName + ".feature_importance", MapUtils.featureImportanceMapping());
         Object dependentVariableMapping = extractMapping(dependentVariable, mappingsProperties);
         if ((dependentVariableMapping instanceof Map) == false) {
-            return Collections.emptyMap();
+            return additionalProperties;
         }
         Map<String, Object> dependentVariableMappingAsMap = (Map) dependentVariableMapping;
         // If the source field is an alias, fetch the concrete field that the alias points to.
@@ -299,9 +301,8 @@ public Map<String, Object> getExplicitlyMappedFields(Map<String, Object> mapping
         // We may have updated the value of {@code dependentVariableMapping} in the "if" block above.
         // Hence, we need to check the "instanceof" condition again.
         if ((dependentVariableMapping instanceof Map) == false) {
-            return Collections.emptyMap();
+            return additionalProperties;
         }
-        Map<String, Object> additionalProperties = new HashMap<>();
         additionalProperties.put(resultsFieldName + "." + predictionFieldName, dependentVariableMapping);
         additionalProperties.put(resultsFieldName + ".top_classes.class_name", dependentVariableMapping);
         return additionalProperties;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MapUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MapUtils.java
new file mode 100644
index 0000000000000..5440bc850c258
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MapUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ *//*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.analyses;
+
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+final class MapUtils {
+
+    private static final Map<String, Object> FEATURE_IMPORTANCE_MAPPING;
+    static {
+        Map<String, Object> featureImportanceMappingProperties = new HashMap<>();
+        featureImportanceMappingProperties.put("feature_name", Collections.singletonMap("type", KeywordFieldMapper.CONTENT_TYPE));
+        featureImportanceMappingProperties.put("importance",
+            Collections.singletonMap("type", NumberFieldMapper.NumberType.DOUBLE.typeName()));
+        Map<String, Object> featureImportanceMapping = new HashMap<>();
+        // TODO sorted indices don't support nested types
+        //featureImportanceMapping.put("dynamic", true);
+        //featureImportanceMapping.put("type", ObjectMapper.NESTED_CONTENT_TYPE);
+        featureImportanceMapping.put("properties", featureImportanceMappingProperties);
+        FEATURE_IMPORTANCE_MAPPING = Collections.unmodifiableMap(featureImportanceMapping);
+    }
+
+    static Map<String, Object> featureImportanceMapping() {
+        return FEATURE_IMPORTANCE_MAPPING;
+    }
+
+    private MapUtils() {}
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
index d8c490ddcb804..0d40ef6ba35df 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -187,9 +188,13 @@ public List<FieldCardinalityConstraint> getFieldCardinalityConstraints() {
 
     @Override
     public Map<String, Object> getExplicitlyMappedFields(Map<String, Object> mappingsProperties, String resultsFieldName) {
+        Map<String, Object> additionalProperties = new HashMap<>();
+        additionalProperties.put(resultsFieldName + ".feature_importance", MapUtils.featureImportanceMapping());
         // Prediction field should be always mapped as "double" rather than "float" in order to increase precision in case of
         // high (over 10M) values of dependent variable.
-        return Collections.singletonMap(resultsFieldName + "." + predictionFieldName, Collections.singletonMap("type", "double"));
+        additionalProperties.put(resultsFieldName + "." + predictionFieldName,
+            Collections.singletonMap("type", NumberFieldMapper.NumberType.DOUBLE.typeName()));
+        return additionalProperties;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCounts.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCounts.java
new file mode 100644
index 0000000000000..f77cc781c746a
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCounts.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.common;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DataCounts implements ToXContentObject, Writeable {
+
+    public static final String TYPE_VALUE = "analytics_data_counts";
+
+    public static final ParseField TRAINING_DOCS_COUNT = new ParseField("training_docs_count");
+    public static final ParseField TEST_DOCS_COUNT = new ParseField("test_docs_count");
+    public static final ParseField SKIPPED_DOCS_COUNT = new ParseField("skipped_docs_count");
+
+    public static final ConstructingObjectParser<DataCounts, Void> STRICT_PARSER = createParser(false);
+    public static final ConstructingObjectParser<DataCounts, Void> LENIENT_PARSER = createParser(true);
+
+    private static ConstructingObjectParser<DataCounts, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<DataCounts, Void> parser = new ConstructingObjectParser<>(TYPE_VALUE, ignoreUnknownFields,
+            a -> new DataCounts((String) a[0], (long) a[1], (long) a[2], (long) a[3]));
+
+        parser.declareString((bucket, s) -> {}, Fields.TYPE);
+        parser.declareString(ConstructingObjectParser.constructorArg(), Fields.JOB_ID);
+        parser.declareLong(ConstructingObjectParser.constructorArg(), TRAINING_DOCS_COUNT);
+        parser.declareLong(ConstructingObjectParser.constructorArg(), TEST_DOCS_COUNT);
+        parser.declareLong(ConstructingObjectParser.constructorArg(), SKIPPED_DOCS_COUNT);
+        return parser;
+    }
+
+    private final String jobId;
+    private final long trainingDocsCount;
+    private final long testDocsCount;
+    private final long skippedDocsCount;
+
+    public DataCounts(String jobId, long trainingDocsCount, long testDocsCount, long skippedDocsCount) {
+        this.jobId = Objects.requireNonNull(jobId);
+        this.trainingDocsCount = trainingDocsCount;
+        this.testDocsCount = testDocsCount;
+        this.skippedDocsCount = skippedDocsCount;
+    }
+
+    public DataCounts(StreamInput in) throws IOException {
+        this.jobId = in.readString();
+        this.trainingDocsCount = in.readVLong();
+        this.testDocsCount = in.readVLong();
+        this.skippedDocsCount = in.readVLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(jobId);
+        out.writeVLong(trainingDocsCount);
+        out.writeVLong(testDocsCount);
+        out.writeVLong(skippedDocsCount);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
+            builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE);
+            builder.field(Fields.JOB_ID.getPreferredName(), jobId);
+        }
+        builder.field(TRAINING_DOCS_COUNT.getPreferredName(), trainingDocsCount);
+        builder.field(TEST_DOCS_COUNT.getPreferredName(), testDocsCount);
+        builder.field(SKIPPED_DOCS_COUNT.getPreferredName(), skippedDocsCount);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DataCounts that = (DataCounts) o;
+        return Objects.equals(jobId, that.jobId)
+            && trainingDocsCount == that.trainingDocsCount
+            && testDocsCount == that.testDocsCount
+            && skippedDocsCount == that.skippedDocsCount;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, trainingDocsCount, testDocsCount, skippedDocsCount);
+    }
+
+    public static String documentId(String jobId) {
+        return TYPE_VALUE + "_" + jobId;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public long getTrainingDocsCount() {
+        return trainingDocsCount;
+    }
+
+    public long getTestDocsCount() {
+        return testDocsCount;
+    }
+
+    public long getSkippedDocsCount() {
+        return skippedDocsCount;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
index e1e24f36b87b9..6096496e792b0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java
@@ -26,7 +26,7 @@ public class OutlierDetectionStats implements AnalysisStats {
     public static final String TYPE_VALUE = "outlier_detection_stats";
 
     public static final ParseField PARAMETERS = new ParseField("parameters");
-    public static final ParseField TIMING_STATS = new ParseField("timings_stats");
+    public static final ParseField TIMING_STATS = new ParseField("timing_stats");
 
     public static final ConstructingObjectParser<OutlierDetectionStats, Void> STRICT_PARSER = createParser(false);
     public static final ConstructingObjectParser<OutlierDetectionStats, Void> LENIENT_PARSER = createParser(true);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java
index a354ae74bf23c..d196506adc187 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java
@@ -35,13 +35,13 @@ public ClassificationInferenceResults(double value,
                                           String classificationLabel,
                                           List<TopClassEntry> topClasses,
                                           InferenceConfig config) {
-        this(value, classificationLabel, topClasses, Collections.emptyMap(), (ClassificationConfig)config);
+        this(value, classificationLabel, topClasses, Collections.emptyList(), (ClassificationConfig)config);
     }
 
     public ClassificationInferenceResults(double value,
                                           String classificationLabel,
                                           List<TopClassEntry> topClasses,
-                                          Map<String, Double> featureImportance,
+                                          List<FeatureImportance> featureImportance,
                                           InferenceConfig config) {
         this(value, classificationLabel, topClasses, featureImportance, (ClassificationConfig)config);
     }
@@ -49,7 +49,7 @@ public ClassificationInferenceResults(double value,
     private ClassificationInferenceResults(double value,
                                            String classificationLabel,
                                            List<TopClassEntry> topClasses,
-                                           Map<String, Double> featureImportance,
+                                           List<FeatureImportance> featureImportance,
                                            ClassificationConfig classificationConfig) {
         super(value,
             SingleValueInferenceResults.takeTopFeatureImportances(featureImportance,
@@ -118,7 +118,10 @@ public void writeResult(IngestDocument document, String parentResultField) {
                 topClasses.stream().map(TopClassEntry::asValueMap).collect(Collectors.toList()));
         }
         if (getFeatureImportance().size() > 0) {
-            document.setFieldValue(parentResultField + ".feature_importance", getFeatureImportance());
+            document.setFieldValue(parentResultField + ".feature_importance", getFeatureImportance()
+                .stream()
+                .map(FeatureImportance::toMap)
+                .collect(Collectors.toList()));
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportance.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportance.java
new file mode 100644
index 0000000000000..a6576a25e3141
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportance.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.inference.results;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class FeatureImportance implements Writeable {
+
+    private final Map<String, Double> classImportance;
+    private final double importance;
+    private final String featureName;
+    private static final String IMPORTANCE = "importance";
+    private static final String FEATURE_NAME = "feature_name";
+
+    public static FeatureImportance forRegression(String featureName, double importance) {
+        return new FeatureImportance(featureName, importance, null);
+    }
+
+    public static FeatureImportance forClassification(String featureName, Map<String, Double> classImportance) {
+        return new FeatureImportance(featureName, classImportance.values().stream().mapToDouble(Math::abs).sum(), classImportance);
+    }
+
+    private FeatureImportance(String featureName, double importance, Map<String, Double> classImportance) {
+        this.featureName = Objects.requireNonNull(featureName);
+        this.importance = importance;
+        this.classImportance = classImportance == null ? null : Collections.unmodifiableMap(classImportance);
+    }
+
+    public FeatureImportance(StreamInput in) throws IOException {
+        this.featureName = in.readString();
+        this.importance = in.readDouble();
+        if (in.readBoolean()) {
+            this.classImportance = in.readMap(StreamInput::readString, StreamInput::readDouble);
+        } else {
+            this.classImportance = null;
+        }
+    }
+
+    public Map<String, Double> getClassImportance() {
+        return classImportance;
+    }
+
+    public double getImportance() {
+        return importance;
+    }
+
+    public String getFeatureName() {
+        return featureName;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(this.featureName);
+        out.writeDouble(this.importance);
+        out.writeBoolean(this.classImportance != null);
+        if (this.classImportance != null) {
+            out.writeMap(this.classImportance, StreamOutput::writeString, StreamOutput::writeDouble);
+        }
+    }
+
+    public Map<String, Object> toMap() {
+        Map<String, Object> map = new LinkedHashMap<>();
+        map.put(FEATURE_NAME, featureName);
+        map.put(IMPORTANCE, importance);
+        if (classImportance != null) {
+            classImportance.forEach(map::put);
+        }
+        return map;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) { return true; }
+        if (object == null || getClass() != object.getClass()) { return false; }
+        FeatureImportance that = (FeatureImportance) object;
+        return Objects.equals(featureName, that.featureName)
+            && Objects.equals(importance, that.importance)
+            && Objects.equals(classImportance, that.classImportance);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(featureName, importance, classImportance);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java
index 2c166e39f8067..216d708a6b471 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java
@@ -18,9 +18,9 @@ public class RawInferenceResults implements InferenceResults {
     public static final String NAME = "raw";
 
     private final double[] value;
-    private final Map<String, Double> featureImportance;
+    private final Map<String, double[]> featureImportance;
 
-    public RawInferenceResults(double[] value, Map<String, Double> featureImportance) {
+    public RawInferenceResults(double[] value, Map<String, double[]> featureImportance) {
         this.value = value;
         this.featureImportance = featureImportance;
     }
@@ -29,7 +29,7 @@ public double[] getValue() {
         return value;
     }
 
-    public Map<String, Double> getFeatureImportance() {
+    public Map<String, double[]> getFeatureImportance() {
         return featureImportance;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java
index a0647b8dffa2d..985b1233cede8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java
@@ -14,8 +14,9 @@
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.Map;
+import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 public class RegressionInferenceResults extends SingleValueInferenceResults {
 
@@ -24,14 +25,14 @@ public class RegressionInferenceResults extends SingleValueInferenceResults {
     private final String resultsField;
 
     public RegressionInferenceResults(double value, InferenceConfig config) {
-        this(value, (RegressionConfig) config, Collections.emptyMap());
+        this(value, (RegressionConfig) config, Collections.emptyList());
     }
 
-    public RegressionInferenceResults(double value, InferenceConfig config, Map<String, Double> featureImportance) {
+    public RegressionInferenceResults(double value, InferenceConfig config, List<FeatureImportance> featureImportance) {
         this(value, (RegressionConfig)config, featureImportance);
     }
 
-    private RegressionInferenceResults(double value, RegressionConfig regressionConfig, Map<String, Double> featureImportance) {
+    private RegressionInferenceResults(double value, RegressionConfig regressionConfig, List<FeatureImportance> featureImportance) {
         super(value,
             SingleValueInferenceResults.takeTopFeatureImportances(featureImportance,
                 regressionConfig.getNumTopFeatureImportanceValues()));
@@ -70,7 +71,10 @@ public void writeResult(IngestDocument document, String parentResultField) {
         ExceptionsHelper.requireNonNull(parentResultField, "parentResultField");
         document.setFieldValue(parentResultField + "." + this.resultsField, value());
         if (getFeatureImportance().size() > 0) {
-            document.setFieldValue(parentResultField + ".feature_importance", getFeatureImportance());
+            document.setFieldValue(parentResultField + ".feature_importance", getFeatureImportance()
+                .stream()
+                .map(FeatureImportance::toMap)
+                .collect(Collectors.toList()));
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SingleValueInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SingleValueInferenceResults.java
index cd739b557abea..249a966ff6294 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SingleValueInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SingleValueInferenceResults.java
@@ -8,45 +8,46 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.List;
+import java.util.stream.Collectors;
 
 public abstract class SingleValueInferenceResults implements InferenceResults {
 
     private final double value;
-    private final Map<String, Double> featureImportance;
+    private final List<FeatureImportance> featureImportance;
 
-    static Map<String, Double> takeTopFeatureImportances(Map<String, Double> unsortedFeatureImportances, int numTopFeatures) {
-        return unsortedFeatureImportances.entrySet()
-            .stream()
-            .sorted((l, r)-> Double.compare(Math.abs(r.getValue()), Math.abs(l.getValue())))
+    static List<FeatureImportance> takeTopFeatureImportances(List<FeatureImportance> unsortedFeatureImportances, int numTopFeatures) {
+        if (unsortedFeatureImportances == null || unsortedFeatureImportances.isEmpty()) {
+            return unsortedFeatureImportances;
+        }
+        return unsortedFeatureImportances.stream()
+            .sorted((l, r)-> Double.compare(Math.abs(r.getImportance()), Math.abs(l.getImportance())))
             .limit(numTopFeatures)
-            .collect(LinkedHashMap::new, (h, e) -> h.put(e.getKey(), e.getValue()) , LinkedHashMap::putAll);
+            .collect(Collectors.toList());
     }
 
     SingleValueInferenceResults(StreamInput in) throws IOException {
         value = in.readDouble();
         if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
-            this.featureImportance = in.readMap(StreamInput::readString, StreamInput::readDouble);
+            this.featureImportance = in.readList(FeatureImportance::new);
         } else {
-            this.featureImportance = Collections.emptyMap();
+            this.featureImportance = Collections.emptyList();
         }
     }
 
-    SingleValueInferenceResults(double value, Map<String, Double> featureImportance) {
+    SingleValueInferenceResults(double value, List<FeatureImportance> featureImportance) {
         this.value = value;
-        this.featureImportance = ExceptionsHelper.requireNonNull(featureImportance, "featureImportance");
+        this.featureImportance = featureImportance == null ? Collections.emptyList() : featureImportance;
     }
 
     public Double value() {
         return value;
     }
 
-    public Map<String, Double> getFeatureImportance() {
+    public List<FeatureImportance> getFeatureImportance() {
         return featureImportance;
     }
 
@@ -58,7 +59,7 @@ public String valueAsString() {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeDouble(value);
         if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
-            out.writeMap(this.featureImportance, StreamOutput::writeString, StreamOutput::writeDouble);
+            out.writeList(this.featureImportance);
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java
index 52cabb4f7b9f3..05b0693ac04c5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceHelpers.java
@@ -8,12 +8,14 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.xpack.core.ml.inference.results.ClassificationInferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.results.FeatureImportance;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -100,18 +102,46 @@ public static Double toDouble(Object value) {
         return null;
     }
 
-    public static Map<String, Double> decodeFeatureImportances(Map<String, String> processedFeatureToOriginalFeatureMap,
-                                                               Map<String, Double> featureImportances) {
+    public static Map<String, double[]> decodeFeatureImportances(Map<String, String> processedFeatureToOriginalFeatureMap,
+                                                                 Map<String, double[]> featureImportances) {
         if (processedFeatureToOriginalFeatureMap == null || processedFeatureToOriginalFeatureMap.isEmpty()) {
             return featureImportances;
         }
 
-        Map<String, Double> originalFeatureImportance = new HashMap<>();
+        Map<String, double[]> originalFeatureImportance = new HashMap<>();
         featureImportances.forEach((feature, importance) -> {
             String featureName = processedFeatureToOriginalFeatureMap.getOrDefault(feature, feature);
-            originalFeatureImportance.compute(featureName, (f, v1) -> v1 == null ? importance : v1 + importance);
+            originalFeatureImportance.compute(featureName, (f, v1) -> v1 == null ? importance : sumDoubleArrays(importance, v1));
         });
-
         return originalFeatureImportance;
     }
+
+    public static List<FeatureImportance> transformFeatureImportance(Map<String, double[]> featureImportance,
+                                                                     @Nullable List<String> classificationLabels) {
+        List<FeatureImportance> importances = new ArrayList<>(featureImportance.size());
+        featureImportance.forEach((k, v) -> {
+            // This indicates regression, or logistic regression
+            // If the length > 1, we assume multi-class classification.
+            if (v.length == 1) {
+                importances.add(FeatureImportance.forRegression(k, v[0]));
+            } else {
+                Map<String, Double> classImportance = new LinkedHashMap<>(v.length, 1.0f);
+                // If the classificationLabels exist, their length must match leaf_value length
+                assert classificationLabels == null || classificationLabels.size() == v.length;
+                for (int i = 0; i < v.length; i++) {
+                    classImportance.put(classificationLabels == null ? String.valueOf(i) : classificationLabels.get(i), v[i]);
+                }
+                importances.add(FeatureImportance.forClassification(k, classImportance));
+            }
+        });
+        return importances;
+    }
+
+    public static double[] sumDoubleArrays(double[] sumTo, double[] inc) {
+        assert sumTo != null && inc != null && sumTo.length == inc.length;
+        for (int i = 0; i < inc.length; i++) {
+            sumTo[i] += inc[i];
+        }
+        return sumTo;
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModel.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModel.java
index 8901c62bb00e8..33cff4d0bb47b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModel.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModel.java
@@ -60,9 +60,9 @@ public interface TrainedModel extends NamedXContentObject, NamedWriteable, Accou
      * NOTE: Must be thread safe
      * @param fields The fields inferring against
      * @param featureDecoder A Map translating processed feature names to their original feature names
-     * @return A {@code Map<String, Double>} mapping each featureName to its importance
+     * @return A {@code Map<String, double[]>} mapping each featureName to its importance
      */
-    Map<String, Double> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder);
+    Map<String, double[]> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder);
 
     default Version getMinimalCompatibilityVersion() {
         return Version.V_7_6_0;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
index 5955ecda6f276..7ee3ef1668b82 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/Ensemble.java
@@ -45,6 +45,8 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceHelpers.classificationLabel;
+import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceHelpers.decodeFeatureImportances;
+import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceHelpers.transformFeatureImportance;
 
 public class Ensemble implements LenientlyParsedTrainedModel, StrictlyParsedTrainedModel {
 
@@ -139,7 +141,7 @@ public InferenceResults infer(Map<String, Object> fields, InferenceConfig config
                 "Cannot infer using configuration for [{}] when model target_type is [{}]", config.getName(), targetType.toString());
         }
         double[][] inferenceResults = new double[this.models.size()][];
-        List<Map<String, Double>> featureInfluence = new ArrayList<>();
+        List<Map<String, double[]>> featureInfluence = new ArrayList<>();
         int i = 0;
         NullInferenceConfig subModelInferenceConfig = new NullInferenceConfig(config.requestingImportance());
         for (TrainedModel model : models) {
@@ -152,7 +154,9 @@ public InferenceResults infer(Map<String, Object> fields, InferenceConfig config
             }
         }
         double[] processed = outputAggregator.processValues(inferenceResults);
-        return buildResults(processed, featureInfluence, config, featureDecoderMap);
+        return buildResults(processed,
+            decodeFeatureImportances(featureDecoderMap, mergeFeatureImportances(featureInfluence)),
+            config);
     }
 
     @Override
@@ -161,19 +165,19 @@ public TargetType targetType() {
     }
 
     private InferenceResults buildResults(double[] processedInferences,
-                                          List<Map<String, Double>> featureInfluence,
-                                          InferenceConfig config,
-                                          Map<String, String> featureDecoderMap) {
+                                          Map<String, double[]> featureInfluence,
+                                          InferenceConfig config) {
         // Indicates that the config is useless and the caller just wants the raw value
         if (config instanceof NullInferenceConfig) {
-            return new RawInferenceResults(new double[] {outputAggregator.aggregate(processedInferences)},
-                InferenceHelpers.decodeFeatureImportances(featureDecoderMap, mergeFeatureImportances(featureInfluence)));
+            return new RawInferenceResults(
+                new double[] {outputAggregator.aggregate(processedInferences)},
+                featureInfluence);
         }
         switch(targetType) {
             case REGRESSION:
                 return new RegressionInferenceResults(outputAggregator.aggregate(processedInferences),
                     config,
-                    InferenceHelpers.decodeFeatureImportances(featureDecoderMap, mergeFeatureImportances(featureInfluence)));
+                    transformFeatureImportance(featureInfluence, null));
             case CLASSIFICATION:
                 ClassificationConfig classificationConfig = (ClassificationConfig) config;
                 assert classificationWeights == null || processedInferences.length == classificationWeights.length;
@@ -186,7 +190,7 @@ private InferenceResults buildResults(double[] processedInferences,
                 return new ClassificationInferenceResults((double)topClasses.v1(),
                     classificationLabel(topClasses.v1(), classificationLabels),
                     topClasses.v2(),
-                    InferenceHelpers.decodeFeatureImportances(featureDecoderMap, mergeFeatureImportances(featureInfluence)),
+                    transformFeatureImportance(featureInfluence, classificationLabels),
                     config);
             default:
                 throw new UnsupportedOperationException("unsupported target_type [" + targetType + "] for inference on ensemble model");
@@ -313,20 +317,23 @@ public boolean supportsFeatureImportance() {
         return models.stream().allMatch(TrainedModel::supportsFeatureImportance);
     }
 
-    Map<String, Double> featureImportance(Map<String, Object> fields) {
+    Map<String, double[]> featureImportance(Map<String, Object> fields) {
         return featureImportance(fields, Collections.emptyMap());
     }
 
     @Override
-    public Map<String, Double> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
-        Map<String, Double> collapsed = mergeFeatureImportances(models.stream()
+    public Map<String, double[]> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
+        Map<String, double[]> collapsed = mergeFeatureImportances(models.stream()
             .map(trainedModel -> trainedModel.featureImportance(fields, Collections.emptyMap()))
             .collect(Collectors.toList()));
-        return InferenceHelpers.decodeFeatureImportances(featureDecoder, collapsed);
+        return decodeFeatureImportances(featureDecoder, collapsed);
     }
 
-    private static Map<String, Double> mergeFeatureImportances(List<Map<String, Double>> featureImportances) {
-        return featureImportances.stream().collect(HashMap::new, (a, b) -> b.forEach((k, v) -> a.merge(k, v, Double::sum)), Map::putAll);
+    private static Map<String, double[]> mergeFeatureImportances(List<Map<String, double[]>> featureImportances) {
+        return featureImportances.stream()
+            .collect(HashMap::new,
+                (a, b) -> b.forEach((k, v) -> a.merge(k, v, InferenceHelpers::sumDoubleArrays)),
+                Map::putAll);
     }
 
     public static Builder builder() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetwork.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetwork.java
index f9a5601f8345a..5d31caba2eff1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetwork.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetwork.java
@@ -142,7 +142,7 @@ public InferenceResults infer(Map<String, Object> fields, InferenceConfig config
         return new ClassificationInferenceResults(topClasses.v1(),
             LANGUAGE_NAMES.get(topClasses.v1()),
             topClasses.v2(),
-            Collections.emptyMap(),
+            Collections.emptyList(),
             classificationConfig);
     }
 
@@ -170,7 +170,7 @@ public boolean supportsFeatureImportance() {
     }
 
     @Override
-    public Map<String, Double> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
+    public Map<String, double[]> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
         throw new UnsupportedOperationException("[lang_ident] does not support feature importance");
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/Tree.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/Tree.java
index e59a7299645bc..ddb1d6027ef6e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/Tree.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/Tree.java
@@ -91,8 +91,8 @@ public static Tree fromXContentLenient(XContentParser parser) {
     private final List<String> classificationLabels;
     private final CachedSupplier<Double> highestOrderCategory;
     // populated lazily when feature importance is calculated
-    private double[] nodeEstimates;
     private Integer maxDepth;
+    private Integer leafSize;
 
     Tree(List<String> featureNames, List<TreeNode> nodes, TargetType targetType, List<String> classificationLabels) {
         this.featureNames = Collections.unmodifiableList(ExceptionsHelper.requireNonNull(featureNames, FEATURE_NAMES));
@@ -137,7 +137,7 @@ public InferenceResults infer(Map<String, Object> fields, InferenceConfig config
             .map(f -> InferenceHelpers.toDouble(MapHelper.dig(f, fields)))
             .collect(Collectors.toList());
 
-        Map<String, Double> featureImportance = config.requestingImportance() ?
+        Map<String, double[]> featureImportance = config.requestingImportance() ?
             featureImportance(features, featureDecoderMap) :
             Collections.emptyMap();
 
@@ -149,7 +149,7 @@ public InferenceResults infer(Map<String, Object> fields, InferenceConfig config
         return buildResult(node.getLeafValue(), featureImportance, config);
     }
 
-    private InferenceResults buildResult(double[] value, Map<String, Double> featureImportance, InferenceConfig config) {
+    private InferenceResults buildResult(double[] value, Map<String, double[]> featureImportance, InferenceConfig config) {
         assert value != null && value.length > 0;
         // Indicates that the config is useless and the caller just wants the raw value
         if (config instanceof NullInferenceConfig) {
@@ -166,10 +166,12 @@ private InferenceResults buildResult(double[] value, Map<String, Double> feature
                 return new ClassificationInferenceResults(topClasses.v1(),
                     classificationLabel(topClasses.v1(), classificationLabels),
                     topClasses.v2(),
-                    featureImportance,
+                    InferenceHelpers.transformFeatureImportance(featureImportance, classificationLabels),
                     config);
             case REGRESSION:
-                return new RegressionInferenceResults(value[0], config, featureImportance);
+                return new RegressionInferenceResults(value[0],
+                    config,
+                    InferenceHelpers.transformFeatureImportance(featureImportance, null));
             default:
                 throw new UnsupportedOperationException("unsupported target_type [" + targetType + "] for inference on tree model");
         }
@@ -283,7 +285,7 @@ public void validate() {
     }
 
     @Override
-    public Map<String, Double> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
+    public Map<String, double[]> featureImportance(Map<String, Object> fields, Map<String, String> featureDecoder) {
         if (nodes.stream().allMatch(n -> n.getNumberSamples() == 0)) {
             throw ExceptionsHelper.badRequestException("[tree_structure.number_samples] must be greater than zero for feature importance");
         }
@@ -293,9 +295,12 @@ public Map<String, Double> featureImportance(Map<String, Object> fields, Map<Str
         return featureImportance(features, featureDecoder);
     }
 
-    private Map<String, Double> featureImportance(List<Double> fieldValues, Map<String, String> featureDecoder) {
-        calculateNodeEstimatesIfNeeded();
-        double[] featureImportance = new double[fieldValues.size()];
+    private Map<String, double[]> featureImportance(List<Double> fieldValues, Map<String, String> featureDecoder) {
+        calculateDepthAndLeafValueSize();
+        double[][] featureImportance = new double[fieldValues.size()][leafSize];
+        for (int i = 0; i < fieldValues.size(); i++) {
+            featureImportance[i] = new double[leafSize];
+        }
         int arrSize = ((this.maxDepth + 1) * (this.maxDepth + 2))/2;
         ShapPath.PathElement[] elements = new ShapPath.PathElement[arrSize];
         for (int i = 0; i < arrSize; i++) {
@@ -303,24 +308,22 @@ private Map<String, Double> featureImportance(List<Double> fieldValues, Map<Stri
         }
         double[] scale = new double[arrSize];
         ShapPath initialPath = new ShapPath(elements, scale);
-        shapRecursive(fieldValues, this.nodeEstimates, initialPath, 0, 1.0, 1.0, -1, featureImportance, 0);
+        shapRecursive(fieldValues, initialPath, 0, 1.0, 1.0, -1, featureImportance, 0);
         return InferenceHelpers.decodeFeatureImportances(featureDecoder,
             IntStream.range(0, featureImportance.length)
                 .boxed()
                 .collect(Collectors.toMap(featureNames::get, i -> featureImportance[i])));
     }
 
-    private void calculateNodeEstimatesIfNeeded() {
-        if (this.nodeEstimates != null && this.maxDepth != null) {
+    private void calculateDepthAndLeafValueSize() {
+        if (this.maxDepth != null && this.leafSize != null) {
             return;
         }
         synchronized (this) {
-            if (this.nodeEstimates != null && this.maxDepth != null) {
+            if (this.maxDepth != null && this.leafSize != null) {
                 return;
             }
-            double[] estimates = new double[nodes.size()];
-            this.maxDepth = fillNodeEstimates(estimates, 0, 0);
-            this.nodeEstimates = estimates;
+            this.maxDepth = getDepth(0, 0);
         }
     }
 
@@ -331,23 +334,24 @@ private void calculateNodeEstimatesIfNeeded() {
      * side first and then ported to the Java side.
      */
     private void shapRecursive(List<Double> processedFeatures,
-                               double[] nodeValues,
                                ShapPath parentSplitPath,
                                int nodeIndex,
                                double parentFractionZero,
                                double parentFractionOne,
                                int parentFeatureIndex,
-                               double[] featureImportance,
+                               double[][] featureImportance,
                                int nextIndex) {
         ShapPath splitPath = new ShapPath(parentSplitPath, nextIndex);
         TreeNode currNode = nodes.get(nodeIndex);
         nextIndex = splitPath.extend(parentFractionZero, parentFractionOne, parentFeatureIndex, nextIndex);
         if (currNode.isLeaf()) {
-            double leafValue = nodeValues[nodeIndex];
+            double[] leafValue = currNode.getLeafValue();
             for (int i = 1; i < nextIndex; ++i) {
-                double scale = splitPath.sumUnwoundPath(i, nextIndex);
                 int inputColumnIndex = splitPath.featureIndex(i);
-                featureImportance[inputColumnIndex] += scale * (splitPath.fractionOnes(i) - splitPath.fractionZeros(i)) * leafValue;
+                double scaled = splitPath.sumUnwoundPath(i, nextIndex) * (splitPath.fractionOnes(i) - splitPath.fractionZeros(i));
+                for (int j = 0; j < leafValue.length; j++) {
+                    featureImportance[inputColumnIndex][j] += scaled * leafValue[j];
+                }
             }
         } else {
             int hotIndex = currNode.compare(processedFeatures);
@@ -365,41 +369,32 @@ private void shapRecursive(List<Double> processedFeatures,
 
             double hotFractionZero = nodes.get(hotIndex).getNumberSamples() / (double)currNode.getNumberSamples();
             double coldFractionZero = nodes.get(coldIndex).getNumberSamples() / (double)currNode.getNumberSamples();
-            shapRecursive(processedFeatures, nodeValues, splitPath,
+            shapRecursive(processedFeatures, splitPath,
                 hotIndex, incomingFractionZero * hotFractionZero,
                 incomingFractionOne, splitFeature, featureImportance, nextIndex);
-            shapRecursive(processedFeatures, nodeValues, splitPath,
+            shapRecursive(processedFeatures, splitPath,
                 coldIndex, incomingFractionZero * coldFractionZero,
                 0.0, splitFeature, featureImportance, nextIndex);
         }
     }
 
     /**
-     * This recursively populates the provided {@code double[]} with the node estimated values
+     * Get the depth of the tree and sets leafSize if it is null
      *
-     * Used when calculating feature importance.
-     * @param nodeEstimates Array to update in place with the node estimated values
      * @param nodeIndex Current node index
      * @param depth Current depth
      * @return The current max depth
      */
-    private int fillNodeEstimates(double[] nodeEstimates, int nodeIndex, int depth) {
+    private int getDepth(int nodeIndex, int depth) {
         TreeNode node = nodes.get(nodeIndex);
         if (node.isLeaf()) {
-            // TODO multi-value????
-            nodeEstimates[nodeIndex] = node.getLeafValue()[0];
+            if (leafSize == null) {
+                this.leafSize = node.getLeafValue().length;
+            }
             return 0;
         }
-
-        int depthLeft = fillNodeEstimates(nodeEstimates, node.getLeftChild(), depth + 1);
-        int depthRight = fillNodeEstimates(nodeEstimates, node.getRightChild(), depth + 1);
-        long leftWeight = nodes.get(node.getLeftChild()).getNumberSamples();
-        long rightWeight = nodes.get(node.getRightChild()).getNumberSamples();
-        long divisor = leftWeight + rightWeight;
-        double averageValue = divisor == 0 ?
-            0.0 :
-            (leftWeight * nodeEstimates[node.getLeftChild()] + rightWeight * nodeEstimates[node.getRightChild()]) / divisor;
-        nodeEstimates[nodeIndex] = averageValue;
+        int depthLeft = getDepth(node.getLeftChild(), depth + 1);
+        int depthRight = getDepth(node.getRightChild(), depth + 1);
         return Math.max(depthLeft, depthRight) + 1;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
index 69a99e34ca146..413d241dda693 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
@@ -51,7 +51,7 @@ public final class Messages {
     public static final String DATAFEED_ID_ALREADY_TAKEN = "A datafeed with id [{0}] already exists";
     public static final String DATAFEED_NEEDS_REMOTE_CLUSTER_SEARCH = "Datafeed [{0}] is configured with a remote index pattern(s) {1}" +
         " but the current node [{2}] is not allowed to connect to remote clusters." +
-        " Please enable cluster.remote.connect for all machine learning nodes.";
+        " Please enable node.remote_cluster_client for all machine learning nodes.";
 
     public static final String DATA_FRAME_ANALYTICS_BAD_QUERY_FORMAT = "Data Frame Analytics config query is not parsable";
     public static final String DATA_FRAME_ANALYTICS_BAD_FIELD_FILTER = "No field [{0}] could be detected";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStats.java
index f038228ae76a9..2843358ceef44 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStats.java
@@ -27,15 +27,18 @@ public class RollupIndexerJobStats extends IndexerJobStats {
     private static ParseField NUM_INVOCATIONS = new ParseField("trigger_count");
     private static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
     private static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
+    private static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
     private static ParseField INDEX_TOTAL = new ParseField("index_total");
     private static ParseField SEARCH_TOTAL = new ParseField("search_total");
+    private static ParseField PROCESSING_TOTAL = new ParseField("processing_total");
     private static ParseField SEARCH_FAILURES = new ParseField("search_failures");
     private static ParseField INDEX_FAILURES = new ParseField("index_failures");
 
     public static final ConstructingObjectParser<RollupIndexerJobStats, Void> PARSER =
         new ConstructingObjectParser<>(NAME.getPreferredName(),
             args -> new RollupIndexerJobStats((long) args[0], (long) args[1], (long) args[2], (long) args[3],
-                (long) args[4], (long) args[5], (long) args[6], (long) args[7],  (long) args[8], (long) args[9]));
+                (long) args[4], (long) args[5], (long) args[6], (long) args[7],  (long) args[8], (long) args[9],
+                (long) args[10], (long) args[11]));
 
     static {
         PARSER.declareLong(constructorArg(), NUM_PAGES);
@@ -44,8 +47,10 @@ public class RollupIndexerJobStats extends IndexerJobStats {
         PARSER.declareLong(constructorArg(), NUM_INVOCATIONS);
         PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS);
         PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
+        PARSER.declareLong(constructorArg(), PROCESSING_TIME_IN_MS);
         PARSER.declareLong(constructorArg(), INDEX_TOTAL);
         PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
+        PARSER.declareLong(constructorArg(), PROCESSING_TOTAL);
         PARSER.declareLong(constructorArg(), INDEX_FAILURES);
         PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
     }
@@ -55,10 +60,10 @@ public RollupIndexerJobStats() {
     }
 
     public RollupIndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations,
-                                 long indexTime, long searchTime, long indexTotal, long searchTotal, long indexFailures,
-                                 long searchFailures) {
-        super(numPages, numInputDocuments, numOuputDocuments, numInvocations, indexTime, searchTime,
-            indexTotal, searchTotal, indexFailures, searchFailures);
+                                 long indexTime, long searchTime, long processingTime, long indexTotal, long searchTotal,
+                                 long processingTotal, long indexFailures, long searchFailures) {
+        super(numPages, numInputDocuments, numOuputDocuments, numInvocations, indexTime, searchTime, processingTime,
+            indexTotal, searchTotal, processingTotal, indexFailures, searchFailures);
     }
 
     public RollupIndexerJobStats(StreamInput in) throws IOException {
@@ -78,6 +83,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(SEARCH_TIME_IN_MS.getPreferredName(), searchTime);
         builder.field(SEARCH_TOTAL.getPreferredName(), searchTotal);
         builder.field(SEARCH_FAILURES.getPreferredName(), searchFailures);
+        builder.field(PROCESSING_TIME_IN_MS.getPreferredName(), processingTime);
+        builder.field(PROCESSING_TOTAL.getPreferredName(), processingTotal);
         builder.endObject();
         return builder;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
index ad46baf891864..8e362136e19fc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java
@@ -177,8 +177,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
         builder.field("is_partial", isPartial);
         builder.field("is_running", isRunning);
-        builder.field("start_time_in_millis", startTimeMillis);
-        builder.field("expiration_time_in_millis", expirationTimeMillis);
+        builder.timeField("start_time_in_millis", "start_time", startTimeMillis);
+        builder.timeField("expiration_time_in_millis", "expiration_time", expirationTimeMillis);
 
         if (searchResponse != null) {
             builder.field("response");
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
index a6397cbf08c3b..6dc18d3581eae 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java
@@ -30,8 +30,8 @@
 public class SubmitAsyncSearchRequest extends ActionRequest {
     public static long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis();
 
-    private TimeValue waitForCompletion = TimeValue.timeValueSeconds(1);
-    private boolean cleanOnCompletion = true;
+    private TimeValue waitForCompletionTimeout = TimeValue.timeValueSeconds(1);
+    private boolean keepOnCompletion = false;
     private TimeValue keepAlive = TimeValue.timeValueDays(5);
 
     private final SearchRequest request;
@@ -56,17 +56,17 @@ public SubmitAsyncSearchRequest(SearchSourceBuilder source, String... indices) {
 
     public SubmitAsyncSearchRequest(StreamInput in) throws IOException {
         this.request = new SearchRequest(in);
-        this.waitForCompletion = in.readTimeValue();
+        this.waitForCompletionTimeout = in.readTimeValue();
         this.keepAlive = in.readTimeValue();
-        this.cleanOnCompletion = in.readBoolean();
+        this.keepOnCompletion = in.readBoolean();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         request.writeTo(out);
-        out.writeTimeValue(waitForCompletion);
+        out.writeTimeValue(waitForCompletionTimeout);
         out.writeTimeValue(keepAlive);
-        out.writeBoolean(cleanOnCompletion);
+        out.writeBoolean(keepOnCompletion);
     }
 
     /**
@@ -84,13 +84,13 @@ public int getBatchReduceSize() {
     /**
      * Sets the minimum time that the request should wait before returning a partial result (defaults to 1 second).
      */
-    public SubmitAsyncSearchRequest setWaitForCompletion(TimeValue waitForCompletion) {
-        this.waitForCompletion = waitForCompletion;
+    public SubmitAsyncSearchRequest setWaitForCompletionTimeout(TimeValue waitForCompletionTimeout) {
+        this.waitForCompletionTimeout = waitForCompletionTimeout;
         return this;
     }
 
-    public TimeValue getWaitForCompletion() {
-        return waitForCompletion;
+    public TimeValue getWaitForCompletionTimeout() {
+        return waitForCompletionTimeout;
     }
 
     /**
@@ -115,20 +115,20 @@ public SearchRequest getSearchRequest() {
     /**
      * Should the resource be removed on completion or failure (defaults to true).
      */
-    public SubmitAsyncSearchRequest setCleanOnCompletion(boolean value) {
-        this.cleanOnCompletion = value;
+    public SubmitAsyncSearchRequest setKeepOnCompletion(boolean value) {
+        this.keepOnCompletion = value;
         return this;
     }
 
-    public boolean isCleanOnCompletion() {
-        return cleanOnCompletion;
+    public boolean isKeepOnCompletion() {
+        return keepOnCompletion;
     }
 
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = request.validate();
         if (request.scroll() != null) {
-            addValidationError("[scroll] queries are not supported", validationException);
+            validationException = addValidationError("[scroll] queries are not supported", validationException);
         }
         if (request.isSuggestOnly()) {
             validationException = addValidationError("suggest-only queries are not supported", validationException);
@@ -141,6 +141,10 @@ public ActionRequestValidationException validate() {
             validationException =
                 addValidationError("[ccs_minimize_roundtrips] is not supported on async search queries", validationException);
         }
+        if (request.getPreFilterShardSize() == null || request.getPreFilterShardSize() != 1) {
+            validationException =
+                addValidationError("[pre_filter_shard_size] cannot be changed for async search queries", validationException);
+        }
 
         return validationException;
     }
@@ -150,7 +154,8 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
         return new CancellableTask(id, type, action, toString(), parentTaskId, headers) {
             @Override
             public boolean shouldCancelChildrenOnCancellation() {
-                return true;
+                // we cancel the underlying search action explicitly in the submit action
+                return false;
             }
         };
     }
@@ -160,22 +165,22 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         SubmitAsyncSearchRequest request1 = (SubmitAsyncSearchRequest) o;
-        return cleanOnCompletion == request1.cleanOnCompletion &&
-            waitForCompletion.equals(request1.waitForCompletion) &&
+        return keepOnCompletion == request1.keepOnCompletion &&
+            waitForCompletionTimeout.equals(request1.waitForCompletionTimeout) &&
             keepAlive.equals(request1.keepAlive) &&
             request.equals(request1.request);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(waitForCompletion, cleanOnCompletion, keepAlive, request);
+        return Objects.hash(waitForCompletionTimeout, keepOnCompletion, keepAlive, request);
     }
 
     @Override
     public String toString() {
         return "SubmitAsyncSearchRequest{" +
-            "waitForCompletion=" + waitForCompletion +
-            ", cleanOnCompletion=" + cleanOnCompletion +
+            "waitForCompletionTimeout=" + waitForCompletionTimeout +
+            ", keepOnCompletion=" + keepOnCompletion +
             ", keepAlive=" + keepAlive +
             ", request=" + request +
             '}';
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
index a12f12ae53017..53704147438a6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
@@ -7,7 +7,9 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
@@ -41,13 +43,27 @@ public SecurityContext(Settings settings, ThreadContext threadContext) {
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
     }
 
+    /**
+     * Returns the current user information, or throws {@link org.elasticsearch.ElasticsearchSecurityException}
+     * if the current request has no authentication information.
+     */
+    public User requireUser() {
+        User user = getUser();
+        if (user == null) {
+            throw new ElasticsearchSecurityException("there is no user available in the current context");
+        }
+        return user;
+    }
+
     /** Returns the current user information, or null if the current request has no authentication info. */
+    @Nullable
     public User getUser() {
         Authentication authentication = getAuthentication();
         return authentication == null ? null : authentication.getUser();
     }
 
     /** Returns the authentication information, or null if the current request has no authentication info. */
+    @Nullable
     public Authentication getAuthentication() {
         try {
             return authenticationSerializer.readFromContext(threadContext);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
index 3f1e49098ff5e..4aa850e1aa60d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
@@ -77,6 +77,10 @@ public RealmRef getLookedUpBy() {
         return lookedUpBy;
     }
 
+    /**
+     * Get the realm where the effective user comes from.
+     * The effective user is the es-security-runas-user if present or the authenticated user.
+     */
     public RealmRef getSourceRealm() {
         return lookedUpBy == null ? authenticatedBy : lookedUpBy;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecondaryAuthentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecondaryAuthentication.java
index 8f032f480e07f..213c94e7dad3f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecondaryAuthentication.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecondaryAuthentication.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.user.User;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -55,6 +56,10 @@ public Authentication getAuthentication() {
         return authentication;
     }
 
+    public User getUser() {
+        return authentication.getUser();
+    }
+
     public <T> T execute(Function<ThreadContext.StoredContext, T> body) {
         return this.securityContext.executeWithAuthentication(this.authentication, body);
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java
index 59f9eafec1c00..e5a039b89a1ad 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java
@@ -97,6 +97,18 @@ public List<String> getRoleNames(ScriptService scriptService, ExpressionModel mo
         }
     }
 
+    public void validate(ScriptService scriptService) {
+        try {
+            parseTemplate(scriptService, Collections.emptyMap());
+        } catch (IllegalArgumentException e) {
+            throw e;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        } catch (Exception e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
     private List<String> convertJsonToList(String evaluation) throws IOException {
         final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY,
             LoggingDeprecationHandler.INSTANCE, evaluation);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java
index b1e6c117a04be..51ae560be1d3b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
 import org.elasticsearch.xpack.core.security.authz.permission.ClusterPermission;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 
@@ -23,7 +24,6 @@
 public class ManageOwnApiKeyClusterPrivilege implements NamedClusterPrivilege {
     public static final ManageOwnApiKeyClusterPrivilege INSTANCE = new ManageOwnApiKeyClusterPrivilege();
     private static final String PRIVILEGE_NAME = "manage_own_api_key";
-    private static final String API_KEY_REALM_TYPE = "_es_api_key";
     private static final String API_KEY_ID_KEY = "_security_api_key_id";
 
     private ManageOwnApiKeyClusterPrivilege() {
@@ -78,7 +78,7 @@ private boolean checkIfUserIsOwnerOfApiKeys(Authentication authentication, Strin
                  * TODO bizybot we need to think on how we can propagate appropriate error message to the end user when username, realm name
                  *   is missing. This is similar to the problem of propagating right error messages in case of access denied.
                  */
-                if (authentication.getSourceRealm().getType().equals(API_KEY_REALM_TYPE)) {
+                if (AuthenticationType.API_KEY == authentication.getAuthenticationType()) {
                     // API key cannot own any other API key so deny access
                     return false;
                 } else if (ownedByAuthenticatedUser) {
@@ -93,7 +93,7 @@ private boolean checkIfUserIsOwnerOfApiKeys(Authentication authentication, Strin
         }
 
         private boolean isCurrentAuthenticationUsingSameApiKeyIdFromRequest(Authentication authentication, String apiKeyId) {
-            if (authentication.getSourceRealm().getType().equals(API_KEY_REALM_TYPE)) {
+            if (AuthenticationType.API_KEY == authentication.getAuthenticationType()) {
                 // API key id from authentication must match the id from request
                 final String authenticatedApiKeyId = (String) authentication.getMetadata().get(API_KEY_ID_KEY);
                 if (Strings.hasText(apiKeyId)) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
index 4b6c35bb7ce85..da45dd96dea73 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
@@ -126,11 +126,16 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
                                         .indices(".monitoring-*").privileges("read", "read_cross_cluster").build(),
                                 RoleDescriptor.IndicesPrivileges.builder()
                                         .indices(".management-beats").privileges("create_index", "read", "write").build(),
-                                // .apm-* is for APM's agent configuration and custom link index creation
+                                // APM agent configuration
                                 RoleDescriptor.IndicesPrivileges.builder()
                                         .indices(".apm-agent-configuration").privileges("all").build(),
+                                // APM custom link index creation
                                 RoleDescriptor.IndicesPrivileges.builder()
                                         .indices(".apm-custom-link").privileges("all").build(),
+                                // APM telemetry queries APM indices in kibana task runner
+                                RoleDescriptor.IndicesPrivileges.builder()
+                                    .indices("apm-*")
+                                    .privileges("read", "read_cross_cluster").build(),
                         },
                         null,
                         new ConfigurableClusterPrivilege[] { new ManageApplicationPrivileges(Collections.singleton("kibana-*")) },
@@ -179,7 +184,7 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
                 .put("machine_learning_admin", new RoleDescriptor("machine_learning_admin", new String[] { "manage_ml" },
                         new RoleDescriptor.IndicesPrivileges[] {
                                 RoleDescriptor.IndicesPrivileges.builder()
-                                        .indices(".ml-anomalies*", ".ml-notifications*", ".ml-state*", ".ml-meta*")
+                                        .indices(".ml-anomalies*", ".ml-notifications*", ".ml-state*", ".ml-meta*", ".ml-stats-*")
                                         .privileges("view_index_metadata", "read").build(),
                                 RoleDescriptor.IndicesPrivileges.builder().indices(".ml-annotations*")
                                         .privileges("view_index_metadata", "read", "write").build()
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/RestorableContextClassLoader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java
similarity index 96%
rename from x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/RestorableContextClassLoader.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java
index 95c68eab9c28a..f3c7ec44a5249 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/RestorableContextClassLoader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.security.support;
+package org.elasticsearch.xpack.core.security.support;
 
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java
index 452ca90205671..2422f0636fffc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java
@@ -19,7 +19,6 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class TransformIndexerStats extends IndexerJobStats {
@@ -33,8 +32,10 @@ public class TransformIndexerStats extends IndexerJobStats {
     public static ParseField NUM_INVOCATIONS = new ParseField("trigger_count");
     public static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
     public static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
+    public static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
     public static ParseField INDEX_TOTAL = new ParseField("index_total");
     public static ParseField SEARCH_TOTAL = new ParseField("search_total");
+    public static ParseField PROCESSING_TOTAL = new ParseField("processing_total");
     public static ParseField SEARCH_FAILURES = new ParseField("search_failures");
     public static ParseField INDEX_FAILURES = new ParseField("index_failures");
     public static ParseField EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS = new ParseField("exponential_avg_checkpoint_duration_ms");
@@ -50,33 +51,37 @@ public class TransformIndexerStats extends IndexerJobStats {
         NAME,
         true,
         args -> new TransformIndexerStats(
-            (long) args[0],
-            (long) args[1],
-            (long) args[2],
-            (long) args[3],
-            (long) args[4],
-            (long) args[5],
-            (long) args[6],
-            (long) args[7],
-            (long) args[8],
-            (long) args[9],
-            (Double) args[10],
-            (Double) args[11],
-            (Double) args[12]
+            unboxSafe(args[0], 0L),
+            unboxSafe(args[1], 0L),
+            unboxSafe(args[2], 0L),
+            unboxSafe(args[3], 0L),
+            unboxSafe(args[4], 0L),
+            unboxSafe(args[5], 0L),
+            unboxSafe(args[6], 0L),
+            unboxSafe(args[7], 0L),
+            unboxSafe(args[8], 0L),
+            unboxSafe(args[9], 0L),
+            unboxSafe(args[10], 0L),
+            unboxSafe(args[11], 0L),
+            unboxSafe(args[12], 0.0),
+            unboxSafe(args[13], 0.0),
+            unboxSafe(args[14], 0.0)
         )
     );
 
     static {
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_PAGES);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS);
-        LENIENT_PARSER.declareLong(constructorArg(), NUM_INVOCATIONS);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_TOTAL);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
-        LENIENT_PARSER.declareLong(constructorArg(), INDEX_FAILURES);
-        LENIENT_PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_PAGES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INPUT_DOCUMENTS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_OUTPUT_DOCUMENTS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INVOCATIONS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TIME_IN_MS);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TOTAL);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_FAILURES);
+        LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_FAILURES);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_INDEXED);
         LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_PROCESSED);
@@ -100,13 +105,15 @@ public TransformIndexerStats(
         long numInvocations,
         long indexTime,
         long searchTime,
+        long processingTime,
         long indexTotal,
         long searchTotal,
+        long processingTotal,
         long indexFailures,
         long searchFailures,
-        Double expAvgCheckpointDurationMs,
-        Double expAvgDocumentsIndexed,
-        Double expAvgDocumentsProcessed
+        double expAvgCheckpointDurationMs,
+        double expAvgDocumentsIndexed,
+        double expAvgDocumentsProcessed
     ) {
         super(
             numPages,
@@ -115,14 +122,16 @@ public TransformIndexerStats(
             numInvocations,
             indexTime,
             searchTime,
+            processingTime,
             indexTotal,
             searchTotal,
+            processingTotal,
             indexFailures,
             searchFailures
         );
-        this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs == null ? 0.0 : expAvgCheckpointDurationMs;
-        this.expAvgDocumentsIndexed = expAvgDocumentsIndexed == null ? 0.0 : expAvgDocumentsIndexed;
-        this.expAvgDocumentsProcessed = expAvgDocumentsProcessed == null ? 0.0 : expAvgDocumentsProcessed;
+        this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs;
+        this.expAvgDocumentsIndexed = expAvgDocumentsIndexed;
+        this.expAvgDocumentsProcessed = expAvgDocumentsProcessed;
     }
 
     public TransformIndexerStats(TransformIndexerStats other) {
@@ -133,8 +142,10 @@ public TransformIndexerStats(TransformIndexerStats other) {
             other.numInvocations,
             other.indexTime,
             other.searchTime,
+            other.processingTime,
             other.indexTotal,
             other.searchTotal,
+            other.processingTotal,
             other.indexFailures,
             other.searchFailures,
             other.expAvgCheckpointDurationMs,
@@ -181,6 +192,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(SEARCH_TIME_IN_MS.getPreferredName(), searchTime);
         builder.field(SEARCH_TOTAL.getPreferredName(), searchTotal);
         builder.field(SEARCH_FAILURES.getPreferredName(), searchFailures);
+        builder.field(PROCESSING_TIME_IN_MS.getPreferredName(), processingTime);
+        builder.field(PROCESSING_TOTAL.getPreferredName(), processingTotal);
         builder.field(EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(), this.expAvgCheckpointDurationMs);
         builder.field(EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(), this.expAvgDocumentsIndexed);
         builder.field(EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(), this.expAvgDocumentsProcessed);
@@ -238,10 +251,12 @@ public boolean equals(Object other) {
             && Objects.equals(this.numInvocations, that.numInvocations)
             && Objects.equals(this.indexTime, that.indexTime)
             && Objects.equals(this.searchTime, that.searchTime)
+            && Objects.equals(this.processingTime, that.processingTime)
             && Objects.equals(this.indexFailures, that.indexFailures)
             && Objects.equals(this.searchFailures, that.searchFailures)
             && Objects.equals(this.indexTotal, that.indexTotal)
             && Objects.equals(this.searchTotal, that.searchTotal)
+            && Objects.equals(this.processingTotal, that.processingTotal)
             && Objects.equals(this.expAvgCheckpointDurationMs, that.expAvgCheckpointDurationMs)
             && Objects.equals(this.expAvgDocumentsIndexed, that.expAvgDocumentsIndexed)
             && Objects.equals(this.expAvgDocumentsProcessed, that.expAvgDocumentsProcessed);
@@ -256,10 +271,12 @@ public int hashCode() {
             numInvocations,
             indexTime,
             searchTime,
+            processingTime,
             indexFailures,
             searchFailures,
             indexTotal,
             searchTotal,
+            processingTotal,
             expAvgCheckpointDurationMs,
             expAvgDocumentsIndexed,
             expAvgDocumentsProcessed
@@ -278,4 +295,13 @@ public static TransformIndexerStats fromXContent(XContentParser parser) {
             throw new RuntimeException(e);
         }
     }
+
+    @SuppressWarnings("unchecked")
+    private static <T> T unboxSafe(Object l, T default_value) {
+        if (l == null) {
+            return default_value;
+        } else {
+            return (T) l;
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java
index 474943f0e8c26..59a80f6c9596f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java
@@ -24,7 +24,7 @@ public final class TransformInternalIndexConstants {
     // internal index
 
     // version is not a rollover pattern, however padded because sort is string based
-    public static final String INDEX_VERSION = "004";
+    public static final String INDEX_VERSION = "005";
     public static final String INDEX_PATTERN = ".transform-internal-";
     public static final String LATEST_INDEX_VERSIONED_NAME = INDEX_PATTERN + INDEX_VERSION;
     public static final String LATEST_INDEX_NAME = LATEST_INDEX_VERSIONED_NAME;
diff --git a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
index 70abb67a76240..7a848345ec96b 100644
--- a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
@@ -2,6 +2,10 @@ grant {
   // bouncy castle
   permission java.security.SecurityPermission "putProviderProperty.BC";
 
+  // needed in (cf. o.e.x.c.s.s.RestorableContextClassLoader)
+  permission java.lang.RuntimePermission "getClassLoader";
+  permission java.lang.RuntimePermission "setContextClassLoader";
+
   // needed for x-pack security extension
   permission java.security.SecurityPermission "createPolicy.JavaPolicy";
   permission java.security.SecurityPermission "getPolicy";
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json
index 5b742c9f6d91e..7a846f5743b91 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/stats_index_mappings.json
@@ -85,6 +85,9 @@
       "peak_usage_bytes" : {
         "type" : "long"
       },
+      "skipped_docs_count": {
+        "type": "long"
+      },
       "timestamp" : {
         "type" : "date"
       },
@@ -98,6 +101,12 @@
           }
         }
       },
+      "test_docs_count": {
+        "type": "long"
+      },
+      "training_docs_count": {
+        "type": "long"
+      },
       "type" : {
         "type" : "keyword"
       },
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json
new file mode 100644
index 0000000000000..8dfbcd246615b
--- /dev/null
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/idp/saml-service-provider-template.json
@@ -0,0 +1,99 @@
+{
+  "index_patterns": [
+    "saml-service-provider-*"
+  ],
+  "aliases": {
+    "saml-service-provider": {}
+  },
+  "order": 100,
+  "settings": {
+    "number_of_shards": 1,
+    "number_of_replicas": 0,
+    "auto_expand_replicas": "0-1",
+    "index.priority": 10,
+    "index.refresh_interval": "1s",
+    "index.format": 1
+  },
+  "mappings": {
+    "_doc": {
+      "_meta": {
+        "idp-version": "${idp.template.version}"
+      },
+      "dynamic": "strict",
+      "properties": {
+        "name": {
+          "type": "text"
+        },
+        "entity_id": {
+          "type": "keyword"
+        },
+        "acs": {
+          "type": "keyword"
+        },
+        "enabled": {
+          "type": "boolean"
+        },
+        "created": {
+          "type": "date",
+          "format": "epoch_millis"
+        },
+        "last_modified": {
+          "type": "date",
+          "format": "epoch_millis"
+        },
+        "name_id_format": {
+          "type": "keyword"
+        },
+        "sign_messages": {
+          "type": "keyword"
+        },
+        "authn_expiry_ms": {
+          "type": "long"
+        },
+        "privileges": {
+          "type": "object",
+          "properties": {
+            "resource": {
+              "type": "keyword"
+            },
+            "roles": {
+              "type": "object",
+              "dynamic": false
+            }
+          }
+        },
+        "attributes": {
+          "type": "object",
+          "properties": {
+            "principal": {
+              "type": "keyword"
+            },
+            "email": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword"
+            },
+            "roles": {
+              "type": "keyword"
+            }
+          }
+        },
+        "certificates": {
+          "type": "object",
+          "properties": {
+            "sp_signing": {
+              "type": "text"
+            },
+            "idp_signing": {
+              "type": "text"
+            },
+            "idp_metadata": {
+              "type": "text"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java
index 084d965a6e74b..d7799959f6cce 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java
@@ -80,7 +80,7 @@ public void testLicenseMetadataParsingDoesNotSwallowOtherMetaData() throws Excep
         builder = metaDataBuilder.build().toXContent(builder, params);
         builder.endObject();
         // deserialize metadata again
-        MetaData metaData = MetaData.Builder.fromXContent(createParser(builder), randomBoolean());
+        MetaData metaData = MetaData.Builder.fromXContent(createParser(builder));
         // check that custom metadata still present
         assertThat(metaData.custom(licensesMetaData.getWriteableName()), notNullValue());
         assertThat(metaData.custom(repositoriesMetaData.getWriteableName()), notNullValue());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
index b08f9e45c8c52..101830a40bc96 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
@@ -110,7 +110,7 @@ public void testSecurityBasicWithoutExplicitSecurityEnabled() {
         assertThat(licenseState.isApiKeyServiceAllowed(), is(false));
 
         assertThat(licenseState.isSecurityAvailable(), is(true));
-        assertThat(licenseState.isSecurityDisabledByLicenseDefaults(), is(true));
+        assertThat(licenseState.isSecurityEnabled(), is(false));
     }
 
     public void testSecurityBasicWithExplicitSecurityEnabled() {
@@ -128,7 +128,7 @@ public void testSecurityBasicWithExplicitSecurityEnabled() {
         assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
 
         assertThat(licenseState.isSecurityAvailable(), is(true));
-        assertThat(licenseState.isSecurityDisabledByLicenseDefaults(), is(false));
+        assertThat(licenseState.isSecurityEnabled(), is(true));
     }
 
     public void testSecurityDefaultBasicExpired() {
@@ -254,7 +254,7 @@ public void testNewTrialDefaultsSecurityOff() {
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
         licenseState.update(TRIAL, true, VersionUtils.randomCompatibleVersion(random(), Version.CURRENT));
 
-        assertThat(licenseState.isSecurityDisabledByLicenseDefaults(), is(true));
+        assertThat(licenseState.isSecurityEnabled(), is(false));
         assertSecurityNotAllowed(licenseState);
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java
index 62b1a43e4ef95..443b4630740a4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java
@@ -102,7 +102,8 @@ public void testSourceIncomplete() throws IOException {
             IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing("-1");
             final PlainActionFuture<String> future = PlainActionFuture.newFuture();
             runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId,
-                snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future));
+                snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT,
+                Collections.emptyMap(), future));
             IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, future::actionGet);
             assertEquals(
                 "Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source",
@@ -128,7 +129,8 @@ public void testIncrementalSnapshot() throws IOException {
             SnapshotId snapshotId = new SnapshotId("test", "test");
             final PlainActionFuture<String> future = PlainActionFuture.newFuture();
             runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId,
-                snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future));
+                snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT,
+                Collections.emptyMap(), future));
             shardGeneration = future.actionGet();
             IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy();
             assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount());
@@ -144,7 +146,8 @@ public void testIncrementalSnapshot() throws IOException {
             IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration);
             final PlainActionFuture<String> future = PlainActionFuture.newFuture();
             runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId,
-                snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future));
+                snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT,
+                Collections.emptyMap(), future));
             shardGeneration = future.actionGet();
             IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy();
             // we processed the segments_N file plus _1.si, _1.fnm, _1.fdx, _1.fdt, _1.fdm
@@ -160,7 +163,8 @@ public void testIncrementalSnapshot() throws IOException {
             IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration);
             final PlainActionFuture<String> future = PlainActionFuture.newFuture();
             runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId,
-                snapshotRef.getIndexCommit(), indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future));
+                snapshotRef.getIndexCommit(), null, indexShardSnapshotStatus, Version.CURRENT,
+                Collections.emptyMap(), future));
             future.actionGet();
             IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy();
             // we processed the segments_N file plus _1_1.liv
@@ -208,7 +212,7 @@ public void testRestoreMinmal() throws IOException {
             final PlainActionFuture<String> future = PlainActionFuture.newFuture();
             runAsSnapshot(shard.getThreadPool(), () -> {
                 repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, snapshotRef.getIndexCommit(),
-                    indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future);
+                    null, indexShardSnapshotStatus, Version.CURRENT, Collections.emptyMap(), future);
                 future.actionGet();
                 final PlainActionFuture<SnapshotInfo> finFuture = PlainActionFuture.newFuture();
                 repository.finalizeSnapshot(snapshotId,
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ClientHelperTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ClientHelperTests.java
index 9641f09711995..d7fc6918c8e71 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ClientHelperTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ClientHelperTests.java
@@ -235,7 +235,7 @@ public void testExecuteWithHeadersNoHeaders() {
 
         PlainActionFuture<SearchResponse> searchFuture = PlainActionFuture.newFuture();
         searchFuture.onResponse(new SearchResponse(InternalSearchResponse.empty(), null, 0, 0, 0, 0L, ShardSearchFailure.EMPTY_ARRAY,
-            SearchResponse.Clusters.EMPTY));
+            SearchResponse.Clusters.EMPTY, null));
         when(client.search(any())).thenReturn(searchFuture);
         assertExecutionWithOrigin(Collections.emptyMap(), client);
     }
@@ -249,7 +249,7 @@ public void testExecuteWithHeaders() {
 
         PlainActionFuture<SearchResponse> searchFuture = PlainActionFuture.newFuture();
         searchFuture.onResponse(new SearchResponse(InternalSearchResponse.empty(), null, 0, 0, 0, 0L, ShardSearchFailure.EMPTY_ARRAY,
-            SearchResponse.Clusters.EMPTY));
+            SearchResponse.Clusters.EMPTY, null));
         when(client.search(any())).thenReturn(searchFuture);
         Map<String, String> headers = MapBuilder.<String, String> newMapBuilder().put(AuthenticationField.AUTHENTICATION_KEY, "anything")
                 .put(AuthenticationServiceField.RUN_AS_USER_HEADER, "anything").map();
@@ -270,7 +270,7 @@ public void testExecuteWithHeadersNoSecurityHeaders() {
 
         PlainActionFuture<SearchResponse> searchFuture = PlainActionFuture.newFuture();
         searchFuture.onResponse(new SearchResponse(InternalSearchResponse.empty(), null, 0, 0, 0, 0L, ShardSearchFailure.EMPTY_ARRAY,
-            SearchResponse.Clusters.EMPTY));
+            SearchResponse.Clusters.EMPTY, null));
         when(client.search(any())).thenReturn(searchFuture);
         Map<String, String> unrelatedHeaders = MapBuilder.<String, String> newMapBuilder().put(randomAlphaOfLength(10), "anything").map();
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java
index 487bf5a7a434e..a81cd106d99e8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java
@@ -104,7 +104,7 @@ protected void doNextSearch(SearchRequest request, ActionListener<SearchResponse
 
             // block till latch has been counted down, simulating network latency
             awaitForLatch();
-            nextPhase.onResponse(new SearchResponse(sections, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null));
+            nextPhase.onResponse(new SearchResponse(sections, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null, null));
         }
 
         @Override
@@ -198,7 +198,7 @@ protected void doNextSearch(SearchRequest request, ActionListener<SearchResponse
                 new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0), null,
                 null, false, null, null, 1);
 
-            nextPhase.onResponse(new SearchResponse(sections, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null));
+            nextPhase.onResponse(new SearchResponse(sections, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null, null));
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java
index 902df8cd44fa2..a0b7d6a035f75 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java
@@ -55,7 +55,8 @@ public void testGetDatefeedState() {
         tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), MlTasks.DATAFEED_TASK_NAME,
                 new StartDatafeedAction.DatafeedParams("foo", 0L),
                 new PersistentTasksCustomMetaData.Assignment("bar", "test assignment"));
-        assertEquals(DatafeedState.STOPPED, MlTasks.getDatafeedState("foo", tasksBuilder.build()));
+        // A task with no state means the datafeed is starting
+        assertEquals(DatafeedState.STARTING, MlTasks.getDatafeedState("foo", tasksBuilder.build()));
 
         tasksBuilder.updateTaskState(MlTasks.datafeedTaskId("foo"), DatafeedState.STARTED);
         assertEquals(DatafeedState.STARTED, MlTasks.getDatafeedState("foo", tasksBuilder.build()));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
index 5cb2b3fef5450..3f957f95a902c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStatsNamedWriteablesProvider;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCountsTests;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsageTests;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStatsTests;
@@ -42,6 +44,7 @@ public static Response randomResponse(int listSize) {
             List<PhaseProgress> progress = new ArrayList<>(progressSize);
             IntStream.of(progressSize).forEach(progressIndex -> progress.add(
                 new PhaseProgress(randomAlphaOfLength(10), randomIntBetween(0, 100))));
+            DataCounts dataCounts = randomBoolean() ? null : DataCountsTests.createRandom();
             MemoryUsage memoryUsage = randomBoolean() ? null : MemoryUsageTests.createRandom();
             AnalysisStats analysisStats = randomBoolean() ? null :
                 randomFrom(
@@ -50,7 +53,7 @@ public static Response randomResponse(int listSize) {
                     RegressionStatsTests.createRandom()
                 );
             Response.Stats stats = new Response.Stats(DataFrameAnalyticsConfigTests.randomValidId(),
-                randomFrom(DataFrameAnalyticsState.values()), failureReason, progress, memoryUsage, analysisStats, null,
+                randomFrom(DataFrameAnalyticsState.values()), failureReason, progress, dataCounts, memoryUsage, analysisStats, null,
                 randomAlphaOfLength(20));
             analytics.add(stats);
         }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
index 96961706aa4ca..2185260c01f99 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
@@ -26,7 +26,6 @@
 import java.util.Set;
 
 import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -244,31 +243,37 @@ public void testFieldCardinalityLimitsIsNonEmpty() {
     }
 
     public void testGetExplicitlyMappedFields() {
-        assertThat(new Classification("foo").getExplicitlyMappedFields(null, "results"), is(anEmptyMap()));
-        assertThat(new Classification("foo").getExplicitlyMappedFields(Collections.emptyMap(), "results"), is(anEmptyMap()));
+        assertThat(new Classification("foo").getExplicitlyMappedFields(null, "results"),
+            equalTo(Collections.singletonMap("results.feature_importance", MapUtils.featureImportanceMapping())));
+        assertThat(new Classification("foo").getExplicitlyMappedFields(Collections.emptyMap(), "results"),
+            equalTo(Collections.singletonMap("results.feature_importance", MapUtils.featureImportanceMapping())));
         assertThat(
             new Classification("foo").getExplicitlyMappedFields(Collections.singletonMap("foo", "not_a_map"), "results"),
-            is(anEmptyMap()));
-        assertThat(
-            new Classification("foo").getExplicitlyMappedFields(
-                Collections.singletonMap("foo", Collections.singletonMap("bar", "baz")),
-                "results"),
+            equalTo(Collections.singletonMap("results.feature_importance", MapUtils.featureImportanceMapping())));
+        Map<String, Object> explicitlyMappedFields = new Classification("foo").getExplicitlyMappedFields(
+            Collections.singletonMap("foo", Collections.singletonMap("bar", "baz")),
+            "results");
+        assertThat(explicitlyMappedFields,
             allOf(
                 hasEntry("results.foo_prediction", Collections.singletonMap("bar", "baz")),
                 hasEntry("results.top_classes.class_name", Collections.singletonMap("bar", "baz"))));
-        assertThat(
-            new Classification("foo").getExplicitlyMappedFields(
-                new HashMap<>() {{
-                    put("foo", new HashMap<>() {{
-                        put("type", "alias");
-                        put("path", "bar");
-                    }});
-                    put("bar", Collections.singletonMap("type", "long"));
-                }},
-                "results"),
+        assertThat(explicitlyMappedFields, hasEntry("results.feature_importance", MapUtils.featureImportanceMapping()));
+
+        explicitlyMappedFields = new Classification("foo").getExplicitlyMappedFields(
+            new HashMap<>() {{
+                put("foo", new HashMap<>() {{
+                    put("type", "alias");
+                    put("path", "bar");
+                }});
+                put("bar", Collections.singletonMap("type", "long"));
+            }},
+            "results");
+        assertThat(explicitlyMappedFields,
             allOf(
                 hasEntry("results.foo_prediction", Collections.singletonMap("type", "long")),
                 hasEntry("results.top_classes.class_name", Collections.singletonMap("type", "long"))));
+        assertThat(explicitlyMappedFields, hasEntry("results.feature_importance", MapUtils.featureImportanceMapping()));
+
         assertThat(
             new Classification("foo").getExplicitlyMappedFields(
                 Collections.singletonMap("foo", new HashMap<>() {{
@@ -276,7 +281,7 @@ public void testGetExplicitlyMappedFields() {
                     put("path", "missing");
                 }}),
                 "results"),
-            is(anEmptyMap()));
+            equalTo(Collections.singletonMap("results.feature_importance", MapUtils.featureImportanceMapping())));
     }
 
     public void testToXContent_GivenVersionBeforeRandomizeSeedWasIntroduced() throws IOException {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java
index d171a2844f0d5..4a1b14e4db676 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java
@@ -143,9 +143,9 @@ public void testFieldCardinalityLimitsIsEmpty() {
     }
 
     public void testGetExplicitlyMappedFields() {
-        assertThat(
-            new Regression("foo").getExplicitlyMappedFields(null, "results"),
-            hasEntry("results.foo_prediction", Collections.singletonMap("type", "double")));
+        Map<String, Object> explicitlyMappedFields = new Regression("foo").getExplicitlyMappedFields(null, "results");
+        assertThat(explicitlyMappedFields, hasEntry("results.foo_prediction", Collections.singletonMap("type", "double")));
+        assertThat(explicitlyMappedFields, hasEntry("results.feature_importance", MapUtils.featureImportanceMapping()));
     }
 
     public void testGetStateDocId() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java
new file mode 100644
index 0000000000000..84033d49de655
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.stats.common;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collections;
+
+public class DataCountsTests extends AbstractBWCSerializationTestCase<DataCounts> {
+
+    private boolean lenient;
+
+    @Before
+    public void chooseLenient() {
+        lenient = randomBoolean();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return lenient;
+    }
+
+    @Override
+    protected DataCounts mutateInstanceForVersion(DataCounts instance, Version version) {
+        return instance;
+    }
+
+    @Override
+    protected DataCounts doParseInstance(XContentParser parser) throws IOException {
+        return lenient ? DataCounts.LENIENT_PARSER.apply(parser, null) : DataCounts.STRICT_PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected ToXContent.Params getToXContentParams() {
+        return new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"));
+    }
+
+    @Override
+    protected Writeable.Reader<DataCounts> instanceReader() {
+        return DataCounts::new;
+    }
+
+    @Override
+    protected DataCounts createTestInstance() {
+        return createRandom();
+    }
+
+    public static DataCounts createRandom() {
+        return new DataCounts(
+            randomAlphaOfLength(10),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong()
+        );
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java
index 3ccfb54bd6924..e6ea0398dbc83 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java
@@ -16,20 +16,30 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 
 public class ClassificationInferenceResultsTests extends AbstractWireSerializingTestCase<ClassificationInferenceResults> {
 
     public static ClassificationInferenceResults createRandomResults() {
+        Supplier<FeatureImportance> featureImportanceCtor = randomBoolean() ?
+            FeatureImportanceTests::randomClassification :
+            FeatureImportanceTests::randomRegression;
+
         return new ClassificationInferenceResults(randomDouble(),
             randomBoolean() ? null : randomAlphaOfLength(10),
             randomBoolean() ? null :
                 Stream.generate(ClassificationInferenceResultsTests::createRandomClassEntry)
                     .limit(randomIntBetween(0, 10))
                     .collect(Collectors.toList()),
+            randomBoolean() ? null :
+                Stream.generate(featureImportanceCtor)
+                    .limit(randomIntBetween(1, 10))
+                    .collect(Collectors.toList()),
             ClassificationConfigTests.randomClassificationConfig());
     }
 
@@ -81,6 +91,40 @@ public void testWriteResultsWithTopClasses() {
         assertThat(document.getFieldValue("result_field.my_results", String.class), equalTo("foo"));
     }
 
+    public void testWriteResultsWithImportance() {
+        Supplier<FeatureImportance> featureImportanceCtor = randomBoolean() ?
+            FeatureImportanceTests::randomClassification :
+            FeatureImportanceTests::randomRegression;
+
+        List<FeatureImportance> importanceList = Stream.generate(featureImportanceCtor)
+            .limit(5)
+            .collect(Collectors.toList());
+        ClassificationInferenceResults result = new ClassificationInferenceResults(0.0,
+            "foo",
+            Collections.emptyList(),
+            importanceList,
+            new ClassificationConfig(0, "predicted_value", "top_classes", 3));
+        IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>());
+        result.writeResult(document, "result_field");
+
+        assertThat(document.getFieldValue("result_field.predicted_value", String.class), equalTo("foo"));
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> writtenImportance = (List<Map<String, Object>>)document.getFieldValue(
+            "result_field.feature_importance",
+            List.class);
+        assertThat(writtenImportance, hasSize(3));
+        importanceList.sort((l, r)-> Double.compare(Math.abs(r.getImportance()), Math.abs(l.getImportance())));
+        for (int i = 0; i < 3; i++) {
+            Map<String, Object> objectMap = writtenImportance.get(i);
+            FeatureImportance importance = importanceList.get(i);
+            assertThat(objectMap.get("feature_name"), equalTo(importance.getFeatureName()));
+            assertThat(objectMap.get("importance"), equalTo(importance.getImportance()));
+            if (importance.getClassImportance() != null) {
+                importance.getClassImportance().forEach((k, v) -> assertThat(objectMap.get(k), equalTo(v)));
+            }
+        }
+    }
+
     @Override
     protected ClassificationInferenceResults createTestInstance() {
         return createRandomResults();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportanceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportanceTests.java
new file mode 100644
index 0000000000000..d58171d479b58
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FeatureImportanceTests.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.inference.results;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+
+public class FeatureImportanceTests extends AbstractWireSerializingTestCase<FeatureImportance> {
+
+    public static FeatureImportance createRandomInstance() {
+        return randomBoolean() ? randomClassification() : randomRegression();
+    }
+
+    static FeatureImportance randomRegression() {
+        return FeatureImportance.forRegression(randomAlphaOfLength(10), randomDoubleBetween(-10.0, 10.0, false));
+    }
+
+    static FeatureImportance randomClassification() {
+        return FeatureImportance.forClassification(
+            randomAlphaOfLength(10),
+            Stream.generate(() -> randomAlphaOfLength(10))
+                .limit(randomLongBetween(2, 10))
+                .collect(Collectors.toMap(Function.identity(), (k) -> randomDoubleBetween(-10, 10, false))));
+
+    }
+
+    @Override
+    protected FeatureImportance createTestInstance() {
+        return createRandomInstance();
+    }
+
+    @Override
+    protected Writeable.Reader<FeatureImportance> instanceReader() {
+        return FeatureImportance::new;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResultsTests.java
index 325768054d0b0..07a89a5bd63f7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResultsTests.java
@@ -22,7 +22,8 @@ public static RawInferenceResults createRandomResults() {
         for (int i = 0; i < n; i++) {
             results[i] = randomDouble();
         }
-        return new RawInferenceResults(results, randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("foo", 1.08));
+        return new RawInferenceResults(results,
+            randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("foo", new double[]{1.08}));
     }
 
     public void testEqualityAndHashcode() {
@@ -31,7 +32,9 @@ public void testEqualityAndHashcode() {
         for (int i = 0; i < n; i++) {
             results[i] = randomDouble();
         }
-        Map<String, Double> importance = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("foo", 1.08);
+        Map<String, double[]> importance = randomBoolean() ?
+            Collections.emptyMap() :
+            Collections.singletonMap("foo", new double[]{1.08, 42.0});
         RawInferenceResults lft = new RawInferenceResults(results, new HashMap<>(importance));
         RawInferenceResults rgt = new RawInferenceResults(Arrays.copyOf(results, n), new HashMap<>(importance));
         assertThat(lft, equalTo(rgt));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java
index 5414f97e08783..dbd6a309e8c74 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java
@@ -8,19 +8,28 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.core.ml.inference.results.RegressionInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfigTests;
 
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 
 
 public class RegressionInferenceResultsTests extends AbstractWireSerializingTestCase<RegressionInferenceResults> {
 
     public static RegressionInferenceResults createRandomResults() {
-        return new RegressionInferenceResults(randomDouble(), RegressionConfigTests.randomRegressionConfig());
+        return new RegressionInferenceResults(randomDouble(),
+            RegressionConfigTests.randomRegressionConfig(),
+            randomBoolean() ? null :
+                Stream.generate(FeatureImportanceTests::randomRegression)
+                    .limit(randomIntBetween(1, 10))
+                    .collect(Collectors.toList()));
     }
 
     public void testWriteResults() {
@@ -31,6 +40,32 @@ public void testWriteResults() {
         assertThat(document.getFieldValue("result_field.predicted_value", Double.class), equalTo(0.3));
     }
 
+    public void testWriteResultsWithImportance() {
+        List<FeatureImportance> importanceList = Stream.generate(FeatureImportanceTests::randomRegression)
+            .limit(5)
+            .collect(Collectors.toList());
+        RegressionInferenceResults result = new RegressionInferenceResults(0.3,
+            new RegressionConfig("predicted_value", 3),
+            importanceList);
+        IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>());
+        result.writeResult(document, "result_field");
+
+        assertThat(document.getFieldValue("result_field.predicted_value", Double.class), equalTo(0.3));
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> writtenImportance = (List<Map<String, Object>>)document.getFieldValue(
+            "result_field.feature_importance",
+            List.class);
+        assertThat(writtenImportance, hasSize(3));
+        importanceList.sort((l, r)-> Double.compare(Math.abs(r.getImportance()), Math.abs(l.getImportance())));
+        for (int i = 0; i < 3; i++) {
+            Map<String, Object> objectMap = writtenImportance.get(i);
+            FeatureImportance importance = importanceList.get(i);
+            assertThat(objectMap.get("feature_name"), equalTo(importance.getFeatureName()));
+            assertThat(objectMap.get("importance"), equalTo(importance.getImportance()));
+            assertThat(objectMap.size(), equalTo(2));
+        }
+    }
+
     @Override
     protected RegressionInferenceResults createTestInstance() {
         return createRandomResults();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java
index f9345fd6e78dd..da84341c01902 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java
@@ -684,45 +684,45 @@ public void testFeatureImportance() {
             .build();
 
 
-        Map<String, Double> featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.0, 0.9)));
-        assertThat(featureImportance.get("foo"), closeTo(-1.653200025, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        Map<String, double[]> featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.0, 0.9)));
+        assertThat(featureImportance.get("foo")[0], closeTo(-1.653200025, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.1, 0.8)));
-        assertThat(featureImportance.get("foo"), closeTo(-1.653200025, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-1.653200025, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.2, 0.7)));
-        assertThat(featureImportance.get("foo"), closeTo(-1.653200025, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-1.653200025, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.3, 0.6)));
-        assertThat(featureImportance.get("foo"), closeTo(-1.16997162, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-1.16997162, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.4, 0.5)));
-        assertThat(featureImportance.get("foo"), closeTo(-1.16997162, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-1.16997162, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.5, 0.4)));
-        assertThat(featureImportance.get("foo"), closeTo(0.0798679, eps));
-        assertThat(featureImportance.get("bar"), closeTo( -0.12444978, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(0.0798679, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo( -0.12444978, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.6, 0.3)));
-        assertThat(featureImportance.get("foo"), closeTo(1.80491886, eps));
-        assertThat(featureImportance.get("bar"), closeTo(-0.4355742, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(1.80491886, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(-0.4355742, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.7, 0.2)));
-        assertThat(featureImportance.get("foo"), closeTo(2.0538184, eps));
-        assertThat(featureImportance.get("bar"), closeTo(0.1451914, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(2.0538184, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(0.1451914, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.8, 0.1)));
-        assertThat(featureImportance.get("foo"), closeTo(2.0538184, eps));
-        assertThat(featureImportance.get("bar"), closeTo(0.1451914, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(2.0538184, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(0.1451914, eps));
 
         featureImportance = ensemble.featureImportance(zipObjMap(featureNames, Arrays.asList(0.9, 0.0)));
-        assertThat(featureImportance.get("foo"), closeTo(2.0538184, eps));
-        assertThat(featureImportance.get("bar"), closeTo(0.1451914, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(2.0538184, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(0.1451914, eps));
     }
 
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java
index c86f1c38a08cf..75ac3bb917e0a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java
@@ -390,22 +390,22 @@ public void testFeatureImportance() {
                 TreeNode.builder(5).setLeafValue(13.0).setNumberSamples(1L),
                 TreeNode.builder(6).setLeafValue(18.0).setNumberSamples(1L)).build();
 
-        Map<String, Double> featureImportance = tree.featureImportance(zipObjMap(featureNames, Arrays.asList(0.25, 0.25)),
+        Map<String, double[]> featureImportance = tree.featureImportance(zipObjMap(featureNames, Arrays.asList(0.25, 0.25)),
             Collections.emptyMap());
-        assertThat(featureImportance.get("foo"), closeTo(-5.0, eps));
-        assertThat(featureImportance.get("bar"), closeTo(-2.5, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-5.0, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(-2.5, eps));
 
         featureImportance = tree.featureImportance(zipObjMap(featureNames, Arrays.asList(0.25, 0.75)), Collections.emptyMap());
-        assertThat(featureImportance.get("foo"), closeTo(-5.0, eps));
-        assertThat(featureImportance.get("bar"), closeTo(2.5, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(-5.0, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(2.5, eps));
 
         featureImportance = tree.featureImportance(zipObjMap(featureNames, Arrays.asList(0.75, 0.25)), Collections.emptyMap());
-        assertThat(featureImportance.get("foo"), closeTo(5.0, eps));
-        assertThat(featureImportance.get("bar"), closeTo(-2.5, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(5.0, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(-2.5, eps));
 
         featureImportance = tree.featureImportance(zipObjMap(featureNames, Arrays.asList(0.75, 0.75)), Collections.emptyMap());
-        assertThat(featureImportance.get("foo"), closeTo(5.0, eps));
-        assertThat(featureImportance.get("bar"), closeTo(2.5, eps));
+        assertThat(featureImportance.get("foo")[0], closeTo(5.0, eps));
+        assertThat(featureImportance.get("bar")[0], closeTo(2.5, eps));
     }
 
     public void testMaxFeatureIndex() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
index 70105b48533bd..94536abfc86c5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
@@ -43,7 +43,8 @@ protected GetRollupJobsAction.JobWrapper createTestInstance() {
         return new GetRollupJobsAction.JobWrapper(ConfigTestHelpers.randomRollupJobConfig(random()),
                 new RollupIndexerJobStats(randomNonNegativeLong(), randomNonNegativeLong(),
                     randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-                    randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()),
+                    randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                    randomNonNegativeLong(), randomNonNegativeLong()),
                 new RollupJobStatus(state, Collections.emptyMap()));
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java
index 9cb57c3ecdc60..90d639c024787 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java
@@ -29,7 +29,8 @@ protected RollupIndexerJobStats doParseInstance(XContentParser parser) {
     public static RollupIndexerJobStats randomStats() {
         return new RollupIndexerJobStats(randomNonNegativeLong(), randomNonNegativeLong(),
             randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+            randomNonNegativeLong(), randomNonNegativeLong());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java
index cab10ca728323..19433d132f1ed 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java
@@ -6,6 +6,9 @@
 
 package org.elasticsearch.xpack.core.security.authc.support.mapper;
 
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -17,8 +20,11 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.ScriptMetaData;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.StoredScriptSource;
 import org.elasticsearch.script.mustache.MustacheScriptEngine;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
@@ -31,8 +37,11 @@
 import java.util.Collections;
 
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class TemplateRoleNameTests extends ESTestCase {
 
@@ -116,4 +125,112 @@ public void tryEquals(TemplateRoleName original) {
         };
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(original, copy, mutate);
     }
+
+    public void testValidate() {
+        final ScriptService scriptService = new ScriptService(Settings.EMPTY,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+
+        final TemplateRoleName plainString = new TemplateRoleName(new BytesArray("{ \"source\":\"heroes\" }"), Format.STRING);
+        plainString.validate(scriptService);
+
+        final TemplateRoleName user = new TemplateRoleName(new BytesArray("{ \"source\":\"_user_{{username}}\" }"), Format.STRING);
+        user.validate(scriptService);
+
+        final TemplateRoleName groups = new TemplateRoleName(new BytesArray("{ \"source\":\"{{#tojson}}groups{{/tojson}}\" }"),
+            Format.JSON);
+        groups.validate(scriptService);
+
+        final TemplateRoleName notObject = new TemplateRoleName(new BytesArray("heroes"), Format.STRING);
+        expectThrows(IllegalArgumentException.class, () -> notObject.validate(scriptService));
+
+        final TemplateRoleName invalidField = new TemplateRoleName(new BytesArray("{ \"foo\":\"heroes\" }"), Format.STRING);
+        expectThrows(IllegalArgumentException.class, () -> invalidField.validate(scriptService));
+    }
+
+    public void testValidateWillPassWithEmptyContext() {
+        final ScriptService scriptService = new ScriptService(Settings.EMPTY,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+
+        final BytesReference template = new BytesArray("{ \"source\":\"" +
+            "{{username}}/{{dn}}/{{realm}}/{{metadata}}" +
+            "{{#realm}}" +
+            "  {{name}}/{{type}}" +
+            "{{/realm}}" +
+            "{{#toJson}}groups{{/toJson}}" +
+            "{{^groups}}{{.}}{{/groups}}" +
+            "{{#metadata}}" +
+            "  {{#first}}" +
+            "    <li><strong>{{name}}</strong></li>" +
+            "  {{/first}}" +
+            "  {{#link}}" +
+            "    <li><a href=\\\"{{url}}\\\">{{name}}</a></li>" +
+            "  {{/link}}" +
+            "  {{#toJson}}subgroups{{/toJson}}" +
+            "  {{something-else}}" +
+            "{{/metadata}}\" }");
+        final TemplateRoleName templateRoleName = new TemplateRoleName(template, Format.STRING);
+        templateRoleName.validate(scriptService);
+    }
+
+    public void testValidateWillFailForSyntaxError() {
+        final ScriptService scriptService = new ScriptService(Settings.EMPTY,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+
+        final BytesReference template = new BytesArray("{ \"source\":\" {{#not-closed}} {{other-variable}} \" }");
+
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new TemplateRoleName(template, Format.STRING).validate(scriptService));
+        assertTrue(e.getCause() instanceof ScriptException);
+    }
+
+    public void testValidationWillFailWhenInlineScriptIsNotEnabled() {
+        final Settings settings = Settings.builder().put("script.allowed_types", ScriptService.ALLOW_NONE).build();
+        final ScriptService scriptService = new ScriptService(settings,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+        final BytesReference inlineScript = new BytesArray("{ \"source\":\"\" }");
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new TemplateRoleName(inlineScript, Format.STRING).validate(scriptService));
+        assertThat(e.getMessage(), containsString("[inline]"));
+    }
+
+    public void testValidateWillFailWhenStoredScriptIsNotEnabled() {
+        final Settings settings = Settings.builder().put("script.allowed_types", ScriptService.ALLOW_NONE).build();
+        final ScriptService scriptService = new ScriptService(settings,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+        final ClusterChangedEvent clusterChangedEvent = mock(ClusterChangedEvent.class);
+        final ClusterState clusterState = mock(ClusterState.class);
+        final MetaData metaData = mock(MetaData.class);
+        final StoredScriptSource storedScriptSource = mock(StoredScriptSource.class);
+        final ScriptMetaData scriptMetaData = new ScriptMetaData.Builder(null).storeScript("foo", storedScriptSource).build();
+        when(clusterChangedEvent.state()).thenReturn(clusterState);
+        when(clusterState.metaData()).thenReturn(metaData);
+        when(metaData.custom(ScriptMetaData.TYPE)).thenReturn(scriptMetaData);
+        when(storedScriptSource.getLang()).thenReturn("mustache");
+        when(storedScriptSource.getSource()).thenReturn("");
+        when(storedScriptSource.getOptions()).thenReturn(Collections.emptyMap());
+        scriptService.applyClusterState(clusterChangedEvent);
+
+        final BytesReference storedScript = new BytesArray("{ \"id\":\"foo\" }");
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new TemplateRoleName(storedScript, Format.STRING).validate(scriptService));
+        assertThat(e.getMessage(), containsString("[stored]"));
+    }
+
+    public void testValidateWillFailWhenStoredScriptIsNotFound() {
+        final ScriptService scriptService = new ScriptService(Settings.EMPTY,
+            Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS);
+        final ClusterChangedEvent clusterChangedEvent = mock(ClusterChangedEvent.class);
+        final ClusterState clusterState = mock(ClusterState.class);
+        final MetaData metaData = mock(MetaData.class);
+        final ScriptMetaData scriptMetaData = new ScriptMetaData.Builder(null).build();
+        when(clusterChangedEvent.state()).thenReturn(clusterState);
+        when(clusterState.metaData()).thenReturn(metaData);
+        when(metaData.custom(ScriptMetaData.TYPE)).thenReturn(scriptMetaData);
+        scriptService.applyClusterState(clusterChangedEvent);
+
+        final BytesReference storedScript = new BytesArray("{ \"id\":\"foo\" }");
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new TemplateRoleName(storedScript, Format.STRING).validate(scriptService));
+        assertThat(e.getMessage(), containsString("unable to find script"));
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
index a66be9616600c..422e5dd0128ba 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
@@ -161,8 +161,9 @@ protected IndicesAccessControl getIndicesAccessControl() {
             when(queryShardContext.toQuery(new TermsQueryBuilder("field", values[i]))).thenReturn(parsedQuery);
 
             DirectoryReader wrappedDirectoryReader = wrapper.apply(directoryReader);
-            IndexSearcher indexSearcher = new ContextIndexSearcher(wrappedDirectoryReader,
-                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+            IndexSearcher indexSearcher = new ContextIndexSearcher(
+                    wrappedDirectoryReader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
+                    IndexSearcher.getDefaultQueryCachingPolicy(), true);
 
             int expectedHitCount = valuesHitCount[i];
             logger.info("Going to verify hit count with query [{}] with expected total hits [{}]", parsedQuery.query(), expectedHitCount);
@@ -270,8 +271,9 @@ protected IndicesAccessControl getIndicesAccessControl() {
 
         DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId);
         DirectoryReader wrappedDirectoryReader = wrapper.apply(directoryReader);
-        IndexSearcher indexSearcher = new ContextIndexSearcher(wrappedDirectoryReader,
-            IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+        IndexSearcher indexSearcher = new ContextIndexSearcher(
+                wrappedDirectoryReader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
+                IndexSearcher.getDefaultQueryCachingPolicy(), true);
 
         ScoreDoc[] hits = indexSearcher.search(new MatchAllDocsQuery(), 1000).scoreDocs;
         Set<Integer> actualDocIds = new HashSet<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java
index 0f043035b79c6..64b040e247b2a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilegeTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
 import org.elasticsearch.xpack.core.security.authz.permission.ClusterPermission;
 import org.elasticsearch.xpack.core.security.user.User;
 
@@ -28,8 +29,8 @@ public void testAuthenticationWithApiKeyAllowsAccessToApiKeyActionsWhenItIsOwner
             ManageOwnApiKeyClusterPrivilege.INSTANCE.buildPermission(ClusterPermission.builder()).build();
 
         final String apiKeyId = randomAlphaOfLengthBetween(4, 7);
-        final Authentication authentication = createMockAuthentication("joe","_es_api_key", "_es_api_key",
-            Map.of("_security_api_key_id", apiKeyId));
+        final Authentication authentication = createMockAuthentication("joe","_es_api_key",
+            AuthenticationType.API_KEY, Map.of("_security_api_key_id", apiKeyId));
         final TransportRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean());
         final TransportRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean());
 
@@ -43,8 +44,8 @@ public void testAuthenticationWithApiKeyDeniesAccessToApiKeyActionsWhenItIsNotOw
             ManageOwnApiKeyClusterPrivilege.INSTANCE.buildPermission(ClusterPermission.builder()).build();
 
         final String apiKeyId = randomAlphaOfLengthBetween(4, 7);
-        final Authentication authentication = createMockAuthentication("joe","_es_api_key", "_es_api_key",
-            Map.of("_security_api_key_id", randomAlphaOfLength(7)));
+        final Authentication authentication = createMockAuthentication("joe","_es_api_key",
+            AuthenticationType.API_KEY, Map.of("_security_api_key_id", randomAlphaOfLength(7)));
         final TransportRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean());
         final TransportRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(apiKeyId, randomBoolean());
 
@@ -56,7 +57,8 @@ public void testAuthenticationWithUserAllowsAccessToApiKeyActionsWhenItIsOwner()
         final ClusterPermission clusterPermission =
             ManageOwnApiKeyClusterPrivilege.INSTANCE.buildPermission(ClusterPermission.builder()).build();
 
-        final Authentication authentication = createMockAuthentication("joe","realm1", "native", Map.of());
+        final Authentication authentication = createMockAuthentication("joe","realm1",
+            AuthenticationType.REALM, Map.of());
         final TransportRequest getApiKeyRequest = GetApiKeyRequest.usingRealmAndUserName("realm1", "joe");
         final TransportRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmAndUserName("realm1", "joe");
 
@@ -69,7 +71,8 @@ public void testAuthenticationWithUserAllowsAccessToApiKeyActionsWhenItIsOwner_W
         final ClusterPermission clusterPermission =
             ManageOwnApiKeyClusterPrivilege.INSTANCE.buildPermission(ClusterPermission.builder()).build();
 
-        final Authentication authentication = createMockAuthentication("joe","realm1", "native", Map.of());
+        final Authentication authentication = createMockAuthentication("joe","realm1",
+            AuthenticationType.REALM, Map.of());
         final TransportRequest getApiKeyRequest = GetApiKeyRequest.forOwnedApiKeys();
         final TransportRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.forOwnedApiKeys();
 
@@ -82,7 +85,8 @@ public void testAuthenticationWithUserDeniesAccessToApiKeyActionsWhenItIsNotOwne
         final ClusterPermission clusterPermission =
             ManageOwnApiKeyClusterPrivilege.INSTANCE.buildPermission(ClusterPermission.builder()).build();
 
-        final Authentication authentication = createMockAuthentication("joe", "realm1", "native", Map.of());
+        final Authentication authentication = createMockAuthentication("joe", "realm1",
+            AuthenticationType.REALM, Map.of());
         final TransportRequest getApiKeyRequest = randomFrom(
             GetApiKeyRequest.usingRealmAndUserName("realm1", randomAlphaOfLength(7)),
             GetApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), "joe"),
@@ -110,14 +114,15 @@ public void testGetAndInvalidateApiKeyWillRespectRunAsUser() {
             InvalidateApiKeyRequest.usingRealmAndUserName("realm_b", "user_b"), authentication));
     }
 
-    private Authentication createMockAuthentication(String username, String realmName, String realmType, Map<String, Object> metadata) {
+    private Authentication createMockAuthentication(String username, String realmName,
+                                                    AuthenticationType authenticationType, Map<String, Object> metadata) {
         final User user = new User(username);
         final Authentication authentication = mock(Authentication.class);
         final Authentication.RealmRef authenticatedBy = mock(Authentication.RealmRef.class);
         when(authentication.getUser()).thenReturn(user);
         when(authentication.getSourceRealm()).thenReturn(authenticatedBy);
+        when(authentication.getAuthenticationType()).thenReturn(authenticationType);
         when(authenticatedBy.getName()).thenReturn(realmName);
-        when(authenticatedBy.getType()).thenReturn(realmType);
         when(authentication.getMetadata()).thenReturn(metadata);
         return authentication;
     }
@@ -136,6 +141,7 @@ private Authentication createMockRunAsAuthentication(String username, String rea
         when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
         when(authentication.getSourceRealm()).thenReturn(lookedUpBy);
         when(authentication.getMetadata()).thenReturn(Map.of());
+        when(authentication.getAuthenticationType()).thenReturn(AuthenticationType.REALM);
         return authentication;
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
index d2cdabd3eb212..dc9be3beaf732 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
@@ -402,6 +402,22 @@ public void testKibanaSystemRole() {
             assertThat(kibanaRole.indices().allowedIndicesMatcher(READ_CROSS_CLUSTER_NAME).test(index), is(true));
         });
 
+        // read-only indices for APM telemetry
+        Arrays.asList("apm-*").forEach((index) -> {
+            assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(index), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(index), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(DeleteIndexAction.NAME).test(index), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(index), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(CreateIndexAction.NAME).test(index), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(index), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(DeleteAction.NAME).test(index), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(UpdateSettingsAction.NAME).test(index), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(SearchAction.NAME).test(index), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(MultiSearchAction.NAME).test(index), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetAction.NAME).test(index), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(READ_CROSS_CLUSTER_NAME).test(index), is(true));
+        });
+
         // Beats management index
         final String index = ".management-beats";
         assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(index), is(false));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java
index f83f820e19433..89f026835bd89 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java
@@ -7,14 +7,43 @@
 package org.elasticsearch.xpack.core.transform.transforms;
 
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 
+import java.io.IOException;
+
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 
 public class TransformIndexerStatsTests extends AbstractSerializingTestCase<TransformIndexerStats> {
 
+    public static TransformIndexerStats randomStats() {
+        return new TransformIndexerStats(
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomLongBetween(0L, 10000L),
+            randomDouble(),
+            randomDouble(),
+            randomDouble()
+        );
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
     @Override
     protected TransformIndexerStats createTestInstance() {
         return randomStats();
@@ -30,24 +59,6 @@ protected TransformIndexerStats doParseInstance(XContentParser parser) {
         return TransformIndexerStats.fromXContent(parser);
     }
 
-    public static TransformIndexerStats randomStats() {
-        return new TransformIndexerStats(
-            randomLongBetween(10L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomLongBetween(0L, 10000L),
-            randomBoolean() ? randomDouble() : null,
-            randomBoolean() ? randomDouble() : null,
-            randomBoolean() ? randomDouble() : null
-        );
-    }
-
     public void testExpAvgIncrement() {
         TransformIndexerStats stats = new TransformIndexerStats();
 
@@ -67,4 +78,66 @@ public void testExpAvgIncrement() {
         assertThat(stats.getExpAvgDocumentsIndexed(), closeTo(20.54545454, 0.0000001));
         assertThat(stats.getExpAvgDocumentsProcessed(), closeTo(59.0909090, 0.0000001));
     }
+
+    public void testXContentLeniencyForMissingFields() throws IOException {
+        // this is essentially the same test as done in the super class, but with the difference of a custom toXContent method that leaves
+        // out fields if the value is 0, this allow us to test successful parsing if fields are not available, e.g. on older versions
+        xContentTester(this::createParser, this::createTestInstance, TransformIndexerStatsTests::toXContentIfNotZero, this::doParseInstance)
+            .numberOfTestRuns(NUMBER_OF_TEST_RUNS)
+            .supportsUnknownFields(supportsUnknownFields())
+            .shuffleFieldsExceptions(getShuffleFieldsExceptions())
+            .randomFieldsExcludeFilter(getRandomFieldsExcludeFilter())
+            .assertEqualsConsumer(this::assertEqualInstances)
+            .assertToXContentEquivalence(assertToXContentEquivalence())
+            .test();
+    }
+
+    public static void toXContentIfNotZero(TransformIndexerStats stats, XContentBuilder builder) throws IOException {
+        // a toXContent version which leaves out field with value 0
+        builder.startObject();
+        xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_PAGES.getPreferredName(), stats.getNumPages());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
+        xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
+        xContentFieldIfNotZero(
+            builder,
+            TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
+            stats.getExpAvgCheckpointDurationMs()
+        );
+        xContentFieldIfNotZero(
+            builder,
+            TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(),
+            stats.getExpAvgDocumentsIndexed()
+        );
+        xContentFieldIfNotZero(
+            builder,
+            TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(),
+            stats.getExpAvgDocumentsProcessed()
+        );
+        builder.endObject();
+    }
+
+    private static XContentBuilder xContentFieldIfNotZero(XContentBuilder builder, String name, long value) throws IOException {
+        if (value > 0) {
+            builder.field(name, value);
+        }
+
+        return builder;
+    }
+
+    private static XContentBuilder xContentFieldIfNotZero(XContentBuilder builder, String name, double value) throws IOException {
+        if (value > 0.0) {
+            builder.field(name, value);
+        }
+
+        return builder;
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java
index ee4bd9a47b71b..cbaacf85277a6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java
@@ -69,7 +69,7 @@ public void testBwcWith73() throws IOException {
                 STARTED,
                 randomBoolean() ? null : randomAlphaOfLength(100),
                 randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(),
-                new TransformIndexerStats(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0.0, 0.0, 0.0),
+                new TransformIndexerStats(1, 2, 3, 4, 5, 6, 0, 8, 9, 0, 11, 12, 0.0, 0.0, 0.0),
                 new TransformCheckpointingInfo(
                     new TransformCheckpointStats(0, null, null, 10, 100),
                     new TransformCheckpointStats(0, null, null, 100, 1000),
@@ -89,4 +89,32 @@ public void testBwcWith73() throws IOException {
             }
         }
     }
+
+    public void testBwcWith76() throws IOException {
+        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
+            TransformStats stats = new TransformStats(
+                "bwc-id",
+                STARTED,
+                randomBoolean() ? null : randomAlphaOfLength(100),
+                randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(),
+                new TransformIndexerStats(1, 2, 3, 4, 5, 6, 0, 8, 9, 0, 11, 12, 13.0, 14.0, 15.0),
+                new TransformCheckpointingInfo(
+                    new TransformCheckpointStats(0, null, null, 10, 100),
+                    new TransformCheckpointStats(0, null, null, 100, 1000),
+                    // changesLastDetectedAt aren't serialized back
+                    100,
+                    null
+                )
+            );
+            try (BytesStreamOutput output = new BytesStreamOutput()) {
+                output.setVersion(Version.V_7_6_0);
+                stats.writeTo(output);
+                try (StreamInput in = output.bytes().streamInput()) {
+                    in.setVersion(Version.V_7_6_0);
+                    TransformStats statsFromOld = new TransformStats(in);
+                    assertThat(statsFromOld, equalTo(stats));
+                }
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
index 30cce2da306da..6b472bec35a43 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
@@ -246,7 +246,7 @@ public List<Setting<?>> getSettings() {
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.singletonList(
             new SystemIndexDescriptor(ENRICH_INDEX_PATTERN, "Contains data to support enrich ingest processors.")
         );
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java
index e4e2a450d10f5..efda81082ce8d 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java
@@ -304,7 +304,8 @@ private static SearchResponse createSearchResponse(TopDocs topDocs, SearchHit[]
             0,
             1L,
             ShardSearchFailure.EMPTY_ARRAY,
-            SearchResponse.Clusters.EMPTY
+            SearchResponse.Clusters.EMPTY,
+            null
         );
     }
 
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java
index b2459f1b5aa7a..1bbf228aec575 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java
@@ -194,7 +194,8 @@ public SearchResponse mockResponse(Map<String, Map<String, ?>> documents) {
             0,
             1,
             ShardSearchFailure.EMPTY_ARRAY,
-            new SearchResponse.Clusters(1, 1, 0)
+            new SearchResponse.Clusters(1, 1, 0),
+            null
         );
     }
 }
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java
index ec8a5819d786a..454cde5a26df9 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java
@@ -394,7 +394,8 @@ public SearchResponse mockResponse(Map<?, Map<String, ?>> documents) {
             0,
             1,
             ShardSearchFailure.EMPTY_ARRAY,
-            new SearchResponse.Clusters(1, 1, 0)
+            new SearchResponse.Clusters(1, 1, 0),
+            null
         );
     }
 
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/CoordinatorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/CoordinatorTests.java
index 926fdd2c502f5..4bfa906a2ae99 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/CoordinatorTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/CoordinatorTests.java
@@ -305,7 +305,7 @@ private static SearchResponse emptySearchResponse() {
             null,
             1
         );
-        return new SearchResponse(response, null, 1, 1, 0, 100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+        return new SearchResponse(response, null, 1, 1, 0, 100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
     }
 
     private class MockLookupFunction implements BiConsumer<MultiSearchRequest, BiConsumer<MultiSearchResponse, Exception>> {
diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml
index b510f80d37f9f..f9a869f9e25f3 100644
--- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml
+++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml
@@ -881,34 +881,6 @@ file where opcode=0 and indexOf(file_name, 'explorer.', 0) == 0'''
 expected_event_ids  = [88, 92]
 description = "check substring ranges"
 
-[[queries]]
-query = '''
-file where serial_event_id=88 and substring(file_name, 0, 4) == 'expl'
-'''
-expected_event_ids  = [88]
-description = "check substring ranges"
-
-[[queries]]
-query = '''
-file where serial_event_id=88 and substring(file_name, 1, 3) == 'xp'
-'''
-expected_event_ids  = [88]
-description = "chaeck substring ranges"
-
-[[queries]]
-query = '''
-file where serial_event_id=88 and substring(file_name, -4) == '.exe'
-'''
-expected_event_ids  = [88]
-description = "check substring ranges"
-
-[[queries]]
-query = '''
-file where serial_event_id=88 and substring(file_name, -4, -1) == '.ex'
-'''
-expected_event_ids  = [88]
-description = "check substring ranges"
-
 [[queries]]
 query = '''
 process where add(serial_event_id, 0) == 1 and add(0, 1) == serial_event_id'''
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java
index 5828a74532522..5aeec120218c3 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
@@ -108,7 +109,7 @@ protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplat
 
     @Override
     public ScriptTemplate scriptWithField(FieldAttribute field) {
-        return new ScriptTemplate(processScript("doc[{}].value"),
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
                 paramsBuilder().variable(field.exactAttribute().name()).build(),
                 dataType());
     }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java
index 21bcfdbd2a195..81f5eeecfb368 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java
@@ -43,7 +43,7 @@ public class EqlParser {
     public LogicalPlan createStatement(String eql) {
         return createStatement(eql, new ParserParams());
     }
-    
+
     public LogicalPlan createStatement(String eql, ParserParams params) {
         if (log.isDebugEnabled()) {
             log.debug("Parsing as statement: {}", eql);
@@ -133,51 +133,6 @@ private class PostProcessor extends EqlBaseBaseListener {
             this.ruleNames = ruleNames;
         }
 
-
-        @Override
-        public void exitFunctionExpression(EqlBaseParser.FunctionExpressionContext context) {
-            Token token = context.name;
-            String functionName = token.getText();
-
-            switch (functionName) {
-                case "add":
-                case "between":
-                case "cidrMatch":
-                case "concat":
-                case "divide":
-                case "endsWith":
-                case "indexOf":
-                case "length":
-                case "match":
-                case "modulo":
-                case "multiply":
-                case "number":
-                case "startsWith":
-                case "string":
-                case "stringContains":
-                case "substring":
-                case "subtract":
-                case "wildcard":
-                    break;
-
-                case "arrayContains":
-                case "arrayCount":
-                case "arraySearch":
-                    throw new ParsingException(
-                        "Unsupported function [" + functionName + "]",
-                        null,
-                        token.getLine(),
-                        token.getCharPositionInLine());
-
-                default:
-                    throw new ParsingException(
-                        "Unknown function [" + functionName + "]",
-                        null,
-                        token.getLine(),
-                        token.getCharPositionInLine());
-            }
-        }
-
         @Override
         public void exitJoin(EqlBaseParser.JoinContext context) {
             Token token = context.JOIN().getSymbol();
diff --git a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt
index 8fb4a57958299..6839583d14c1e 100644
--- a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt
+++ b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt
@@ -9,9 +9,7 @@
 #### Classes
 
 class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils {
-}
 
-class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalEqlScriptUtils {
 #
 # Utilities
 # 
@@ -46,6 +44,14 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
 #
   Boolean regex(String, String)
 
+#
+# Math
+#
+  Number neg(Number)
+}
+
+class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalEqlScriptUtils {
+
 #
 # ASCII Functions
 # 
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
index 1570184ced1e0..23763bec6ec7e 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
@@ -113,20 +113,20 @@ public void testJoinUnsupported() {
 
     // Some functions fail with "Unsupported" message at the parse stage
     public void testArrayFunctionsUnsupported() {
-        assertEquals("1:16: Unsupported function [arrayContains]",
-                errorParsing("registry where arrayContains(bytes_written_string_list, 'En')"));
-        assertEquals("1:16: Unsupported function [arraySearch]",
-                errorParsing("registry where arraySearch(bytes_written_string_list, a, a == 'en-us')"));
-        assertEquals("1:16: Unsupported function [arrayCount]",
-                errorParsing("registry where arrayCount(bytes_written_string_list, s, s == '*-us') == 1"));
+        assertEquals("1:16: Unknown function [arrayContains]",
+                error("registry where arrayContains(bytes_written_string_list, 'En')"));
+        assertEquals("1:16: Unknown function [arraySearch]",
+            error("registry where arraySearch(bytes_written_string_list, bytes_written_string, true)"));
+        assertEquals("1:16: Unknown function [arrayCount]",
+            error("registry where arrayCount(bytes_written_string_list, bytes_written_string, true) == 1"));
     }
 
     // Some functions fail with "Unknown" message at the parse stage
     public void testFunctionParsingUnknown() {
         assertEquals("1:15: Unknown function [matchLite]",
-                errorParsing("process where matchLite(?'.*?net1\\s+localgroup\\s+.*?', command_line)"));
+                error("process where matchLite(?'.*?net1\\s+localgroup\\s+.*?', command_line)"));
         assertEquals("1:15: Unknown function [safe]",
-                errorParsing("network where safe(divide(process_name, process_name))"));
+                error("network where safe(process_name)"));
     }
 
     // Test the known EQL functions that are not supported
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
index 0867c6e77c5c8..2beb44a269b20 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java
@@ -94,7 +94,7 @@ public void test() {
         PhysicalPlan p = plan(query);
         assertEquals(EsQueryExec.class, p.getClass());
         EsQueryExec eqe = (EsQueryExec) p;
-        assertEquals(23, eqe.output().size());
+        assertEquals(25, eqe.output().size());
         assertEquals(KEYWORD, eqe.output().get(0).dataType());
 
         final String query = eqe.queryContainer().toString().replaceAll("\\s+", "");
diff --git a/x-pack/plugin/eql/src/test/resources/mapping-default.json b/x-pack/plugin/eql/src/test/resources/mapping-default.json
index 73a16abf380e5..6c9ff933705e2 100644
--- a/x-pack/plugin/eql/src/test/resources/mapping-default.json
+++ b/x-pack/plugin/eql/src/test/resources/mapping-default.json
@@ -1,5 +1,11 @@
 {
     "properties" : {
+        "bytes_written_string" : {
+            "type" : "keyword"
+        },
+        "bytes_written_string_list" : {
+            "type" : "keyword"
+        },
         "command_line" : {
             "type" : "keyword"
         },
diff --git a/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql b/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql
index 08d0d1582750d..e9c20febfedb8 100644
--- a/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql
+++ b/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql
@@ -626,58 +626,44 @@ any where process_name == "svchost.exe"
 ;
 
 
-// array functions
-registry where arrayContains(bytes_written_string_list, 'En-uS');
-registry where arrayContains(bytes_written_string_list, 'En');
+// Array functions
+// * parser will recognize as valid, but functions will fail to resolve in verifier
 
 
-network where mysterious_field
-  and arraySearch(mysterious_field.subarray, s, true)
-;
+// registry where arrayContains(bytes_written_string_list, 'En-uS');
+// registry where arrayContains(bytes_written_string_list, 'En');
 
-registry where arraySearch(bytes_written_string_list, a, a == 'en-us');
+// network where mysterious_field
+//  and arraySearch(mysterious_field.subarray, s, true);
 
-registry where arraySearch(bytes_written_string_list, a, endsWith(a, '-us'));
+// registry where arraySearch(bytes_written_string_list, a, a == 'en-us');
+// registry where arraySearch(bytes_written_string_list, a, endsWith(a, '-us'));
+// network where mysterious_field and arraySearch(mysterious_field.subarray, s, false);
+// network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a == 's0-*');
+// network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a != 's0-*');
 
+// network where mysterious_field
+//   and arraySearch(mysterious_field.subarray, sub1,
+//     arraySearch(sub1.c, nested, nested.x.y == '*'))
+// ;
 
-network where mysterious_field and arraySearch(mysterious_field.subarray, s, false)
-;
+// network where mysterious_field
+//   and arraySearch(mysterious_field.subarray, sub1,
+//     sub1.a == 's0-a' and arraySearch(sub1.c, nested, nested.z == 's0-c1-x-z'))
+// ;
 
-network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a == 's0-*')
-;
+// network where mysterious_field
+//   and arraySearch(mysterious_field.subarray, sub1,
+//     sub1.a == 's0-a' and arraySearch(sub1.c, nested, nested.z == sub1.cross_match));
 
-network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a != 's0-*')
-;
+// network where mysterious_field
+//   and arraySearch(mysterious_field.subarray, sub1,
+//     arraySearch(sub1.c, nested, nested.x.y == mysterious_field.outer_cross_match));
 
-network where mysterious_field
-  and arraySearch(mysterious_field.subarray, sub1,
-    arraySearch(sub1.c, nested, nested.x.y == '*'))
-;
-
-network where mysterious_field
-  and arraySearch(mysterious_field.subarray, sub1,
-    sub1.a == 's0-a' and arraySearch(sub1.c, nested, nested.z == 's0-c1-x-z'))
-;
-
-network where mysterious_field
-  and arraySearch(mysterious_field.subarray, sub1,
-    sub1.a == 's0-a' and arraySearch(sub1.c, nested, nested.z == sub1.cross_match))
-;
+// registry where arrayCount(bytes_written_string_list, s, s == '*-us') == 1;
+// registry where arrayCount(bytes_written_string_list, s, s == '*en*') == 2;
+// registry where arrayContains(bytes_written_string_list, "missing", "en-US");
 
-network where mysterious_field
-  and arraySearch(mysterious_field.subarray, sub1,
-    arraySearch(sub1.c, nested, nested.x.y == mysterious_field.outer_cross_match))
-;
-
-
-registry where arrayCount(bytes_written_string_list, s, s == '*-us') == 1
-;
-
-registry where arrayCount(bytes_written_string_list, s, s == '*en*') == 2
-;
-
-registry where arrayContains(bytes_written_string_list, "missing", "en-US")
-;
 
 // array fields
 
diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt
index 8a2af74ab84a2..a4f73f1b19602 100644
--- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt
+++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt
@@ -64,3 +64,11 @@ process where process_path == "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3)
 "term":{"opcode":{"value":1
 "term":{"opcode":{"value":2
 "term":{"opcode":{"value":3
+
+
+substringFunction
+process where substring(file_name, -4) == '.exe'
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalEqlScriptUtils.substring(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))",
+"params":{"v0":"file_name.keyword","v1":-4,"v2":null,"v3":".exe"}
+
diff --git a/x-pack/plugin/identity-provider/build.gradle b/x-pack/plugin/identity-provider/build.gradle
new file mode 100644
index 0000000000000..4dab2c984b529
--- /dev/null
+++ b/x-pack/plugin/identity-provider/build.gradle
@@ -0,0 +1,368 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+apply plugin: 'nebula.maven-scm'
+esplugin {
+  name 'x-pack-identity-provider'
+  description 'Elasticsearch Expanded Pack Plugin - Identity Provider'
+  classname 'org.elasticsearch.xpack.idp.IdentityProviderPlugin'
+  extendedPlugins = ['x-pack-core']
+}
+
+archivesBaseName = 'x-pack-identity-provider'
+
+dependencies {
+  compileOnly project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+  // So that we can extend LocalStateCompositeXPackPlugin
+  testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
+
+  // the following are all SAML dependencies - might as well download the whole internet
+  compile "org.opensaml:opensaml-core:3.4.5"
+  compile "org.opensaml:opensaml-saml-api:3.4.5"
+  compile "org.opensaml:opensaml-saml-impl:3.4.5"
+  compile "org.opensaml:opensaml-messaging-api:3.4.5"
+  compile "org.opensaml:opensaml-messaging-impl:3.4.5"
+  compile "org.opensaml:opensaml-security-api:3.4.5"
+  compile "org.opensaml:opensaml-security-impl:3.4.5"
+  compile "org.opensaml:opensaml-profile-api:3.4.5"
+  compile "org.opensaml:opensaml-profile-impl:3.4.5"
+  compile "org.opensaml:opensaml-xmlsec-api:3.4.5"
+  compile "org.opensaml:opensaml-xmlsec-impl:3.4.5"
+  compile "org.opensaml:opensaml-soap-api:3.4.5"
+  compile "org.opensaml:opensaml-soap-impl:3.4.5"
+  compile "org.opensaml:opensaml-storage-api:3.4.5"
+  compile "org.opensaml:opensaml-storage-impl:3.4.5"
+  compile "net.shibboleth.utilities:java-support:7.5.1"
+  compile "org.apache.santuario:xmlsec:2.1.4"
+  compile "io.dropwizard.metrics:metrics-core:3.2.2"
+  compile ("org.cryptacular:cryptacular:1.2.4") {
+      exclude group: 'org.bouncycastle'
+  }
+  compile "org.slf4j:slf4j-api:${versions.slf4j}"
+  compile "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
+  compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+  compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+  compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
+  compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
+  compile "org.apache.httpcomponents:httpclient-cache:${versions.httpclient}"
+  compile 'com.google.guava:guava:19.0'
+
+  testCompile 'org.elasticsearch:securemock:1.2'
+  testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
+}
+
+compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
+compileTestJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
+
+dependencyLicenses {
+  mapping from: /java-support|opensaml-.*/, to: 'shibboleth'
+  mapping from: /http.*/, to: 'httpclient'
+}
+
+forbiddenPatterns {
+  exclude '**/*.key'
+  exclude '**/*.p12'
+  exclude '**/*.der'
+  exclude '**/*.zip'
+}
+
+forbiddenApisMain {
+  signaturesFiles += files('forbidden/xml-signatures.txt')
+}
+
+// classes are missing, e.g. com.ibm.icu.lang.UCharacter
+thirdPartyAudit {
+    ignoreMissingClasses (
+        // SAML dependencies
+        // [missing classes] Some cli utilities that we don't use depend on these missing JCommander classes
+        'com.beust.jcommander.JCommander',
+        'com.beust.jcommander.converters.BaseConverter',
+        // [missing classes] Shibboleth + OpenSAML have servlet support that we don't use
+        'javax.servlet.AsyncContext',
+        'javax.servlet.DispatcherType',
+        'javax.servlet.Filter',
+        'javax.servlet.FilterChain',
+        'javax.servlet.FilterConfig',
+        'javax.servlet.RequestDispatcher',
+        'javax.servlet.ServletContext',
+        'javax.servlet.ServletException',
+        'javax.servlet.ServletInputStream',
+        'javax.servlet.ServletOutputStream',
+        'javax.servlet.ServletRequest',
+        'javax.servlet.ServletResponse',
+        'javax.servlet.http.Cookie',
+        'javax.servlet.http.HttpServletRequest',
+        'javax.servlet.http.HttpServletResponse',
+        'javax.servlet.http.HttpServletResponseWrapper',
+        'javax.servlet.http.HttpSession',
+        'javax.servlet.http.Part',
+        // [missing classes] Shibboleth + OpenSAML have velocity support that we don't use
+        'org.apache.velocity.VelocityContext',
+        'org.apache.velocity.app.VelocityEngine',
+        'org.apache.velocity.context.Context',
+        'org.apache.velocity.exception.VelocityException',
+        'org.apache.velocity.runtime.RuntimeServices',
+        'org.apache.velocity.runtime.log.LogChute',
+        'org.apache.velocity.runtime.resource.loader.StringResourceLoader',
+        'org.apache.velocity.runtime.resource.util.StringResourceRepository',
+        // [missing classes] OpenSAML depends on Apache XML security which depends on Xalan, but only for functionality that OpenSAML doesn't use
+        'org.apache.xml.dtm.DTM',
+        'org.apache.xml.utils.PrefixResolver',
+        'org.apache.xml.utils.PrefixResolverDefault',
+        'org.apache.xpath.Expression',
+        'org.apache.xpath.NodeSetDTM',
+        'org.apache.xpath.XPath',
+        'org.apache.xpath.XPathContext',
+        'org.apache.xpath.compiler.FunctionTable',
+        'org.apache.xpath.functions.Function',
+        'org.apache.xpath.objects.XNodeSet',
+        'org.apache.xpath.objects.XObject',
+        // [missing classes] OpenSAML storage has an optional LDAP storage impl
+        'org.ldaptive.AttributeModification',
+        'org.ldaptive.AttributeModificationType',
+        'org.ldaptive.Connection',
+        'org.ldaptive.DeleteOperation',
+        'org.ldaptive.DeleteRequest',
+        'org.ldaptive.LdapAttribute',
+        'org.ldaptive.LdapEntry',
+        'org.ldaptive.LdapException',
+        'org.ldaptive.ModifyOperation',
+        'org.ldaptive.ModifyRequest',
+        'org.ldaptive.Response',
+        'org.ldaptive.ResultCode',
+        'org.ldaptive.SearchOperation',
+        'org.ldaptive.SearchRequest',
+        'org.ldaptive.SearchResult',
+        'org.ldaptive.ext.MergeOperation',
+        'org.ldaptive.ext.MergeRequest',
+        'org.ldaptive.pool.ConnectionPool',
+        'org.ldaptive.pool.PooledConnectionFactory',
+        // [missing classes] OpenSAML storage has an optional JSON-backed storage impl
+        'javax.json.Json',
+        'javax.json.JsonException',
+        'javax.json.JsonNumber',
+        'javax.json.JsonObject',
+        'javax.json.JsonReader',
+        'javax.json.JsonValue$ValueType',
+        'javax.json.JsonValue',
+        'javax.json.stream.JsonGenerator',
+        // [missing classes] OpenSAML storage has an optional JPA storage impl
+        'javax.persistence.EntityManager',
+        'javax.persistence.EntityManagerFactory',
+        'javax.persistence.EntityTransaction',
+        'javax.persistence.LockModeType',
+        'javax.persistence.Query',
+        // [missing classes] OpenSAML storage and HttpClient cache have optional memcache support
+        'net.spy.memcached.CASResponse',
+        'net.spy.memcached.CASValue',
+        'net.spy.memcached.MemcachedClient',
+        'net.spy.memcached.MemcachedClientIF',
+        'net.spy.memcached.CachedData',
+        'net.spy.memcached.internal.OperationFuture',
+        'net.spy.memcached.transcoders.Transcoder',
+        // [missing classes] Http Client cache has optional ehcache support
+        'net.sf.ehcache.Ehcache',
+        'net.sf.ehcache.Element',
+        // [missing classes] SLF4j includes an optional class that depends on an extension class (!)
+        'org.slf4j.ext.EventData',
+        // Bouncycastle is an optional dependency for apache directory, cryptacular and opensaml packages. We
+        // acknowledge them here instead of adding bouncy castle as a compileOnly dependency
+        'org.bouncycastle.asn1.ASN1Encodable',
+        'org.bouncycastle.asn1.ASN1InputStream',
+        'org.bouncycastle.asn1.ASN1Integer',
+        'org.bouncycastle.asn1.ASN1ObjectIdentifier',
+        'org.bouncycastle.asn1.ASN1OctetString',
+        'org.bouncycastle.asn1.ASN1Primitive',
+        'org.bouncycastle.asn1.ASN1Sequence',
+        'org.bouncycastle.asn1.ASN1TaggedObject',
+        'org.bouncycastle.asn1.DEROctetString',
+        'org.bouncycastle.asn1.DERSequence',
+        'org.bouncycastle.asn1.pkcs.EncryptedPrivateKeyInfo',
+        'org.bouncycastle.asn1.pkcs.EncryptionScheme',
+        'org.bouncycastle.asn1.pkcs.KeyDerivationFunc',
+        'org.bouncycastle.asn1.pkcs.PBEParameter',
+        'org.bouncycastle.asn1.pkcs.PBES2Parameters',
+        'org.bouncycastle.asn1.pkcs.PBKDF2Params',
+        'org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers',
+        'org.bouncycastle.asn1.pkcs.PrivateKeyInfo',
+        'org.bouncycastle.asn1.x500.AttributeTypeAndValue',
+        'org.bouncycastle.asn1.x500.RDN',
+        'org.bouncycastle.asn1.x500.X500Name',
+        'org.bouncycastle.asn1.x509.AccessDescription',
+        'org.bouncycastle.asn1.x509.AlgorithmIdentifier',
+        'org.bouncycastle.asn1.x509.AuthorityKeyIdentifier',
+        'org.bouncycastle.asn1.x509.BasicConstraints',
+        'org.bouncycastle.asn1.x509.DistributionPoint',
+        'org.bouncycastle.asn1.x509.Extension',
+        'org.bouncycastle.asn1.x509.GeneralName',
+        'org.bouncycastle.asn1.x509.GeneralNames',
+        'org.bouncycastle.asn1.x509.GeneralNamesBuilder',
+        'org.bouncycastle.asn1.x509.KeyPurposeId',
+        'org.bouncycastle.asn1.x509.KeyUsage',
+        'org.bouncycastle.asn1.x509.PolicyInformation',
+        'org.bouncycastle.asn1.x509.SubjectKeyIdentifier',
+        'org.bouncycastle.asn1.x509.SubjectPublicKeyInfo',
+        'org.bouncycastle.asn1.x9.X9ECParameters',
+        'org.bouncycastle.cert.X509v3CertificateBuilder',
+        'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
+        'org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils',
+        'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
+        'org.bouncycastle.crypto.BlockCipher',
+        'org.bouncycastle.crypto.BufferedBlockCipher',
+        'org.bouncycastle.crypto.CipherParameters',
+        'org.bouncycastle.crypto.Digest',
+        'org.bouncycastle.crypto.PBEParametersGenerator',
+        'org.bouncycastle.crypto.StreamCipher',
+        'org.bouncycastle.crypto.digests.GOST3411Digest',
+        'org.bouncycastle.crypto.digests.MD2Digest',
+        'org.bouncycastle.crypto.digests.MD4Digest',
+        'org.bouncycastle.crypto.digests.MD5Digest',
+        'org.bouncycastle.crypto.digests.RIPEMD128Digest',
+        'org.bouncycastle.crypto.digests.RIPEMD160Digest',
+        'org.bouncycastle.crypto.digests.RIPEMD256Digest',
+        'org.bouncycastle.crypto.digests.RIPEMD320Digest',
+        'org.bouncycastle.crypto.digests.SHA1Digest',
+        'org.bouncycastle.crypto.digests.SHA224Digest',
+        'org.bouncycastle.crypto.digests.SHA256Digest',
+        'org.bouncycastle.crypto.digests.SHA384Digest',
+        'org.bouncycastle.crypto.digests.SHA3Digest',
+        'org.bouncycastle.crypto.digests.SHA512Digest',
+        'org.bouncycastle.crypto.digests.TigerDigest',
+        'org.bouncycastle.crypto.digests.WhirlpoolDigest',
+        'org.bouncycastle.crypto.engines.AESEngine',
+        'org.bouncycastle.crypto.engines.BlowfishEngine',
+        'org.bouncycastle.crypto.engines.CAST5Engine',
+        'org.bouncycastle.crypto.engines.CAST6Engine',
+        'org.bouncycastle.crypto.engines.CamelliaEngine',
+        'org.bouncycastle.crypto.engines.DESEngine',
+        'org.bouncycastle.crypto.engines.DESedeEngine',
+        'org.bouncycastle.crypto.engines.GOST28147Engine',
+        'org.bouncycastle.crypto.engines.Grain128Engine',
+        'org.bouncycastle.crypto.engines.HC128Engine',
+        'org.bouncycastle.crypto.engines.HC256Engine',
+        'org.bouncycastle.crypto.engines.ISAACEngine',
+        'org.bouncycastle.crypto.engines.NoekeonEngine',
+        'org.bouncycastle.crypto.engines.RC2Engine',
+        'org.bouncycastle.crypto.engines.RC4Engine',
+        'org.bouncycastle.crypto.engines.RC532Engine',
+        'org.bouncycastle.crypto.engines.RC564Engine',
+        'org.bouncycastle.crypto.engines.RC6Engine',
+        'org.bouncycastle.crypto.engines.SEEDEngine',
+        'org.bouncycastle.crypto.engines.Salsa20Engine',
+        'org.bouncycastle.crypto.engines.SerpentEngine',
+        'org.bouncycastle.crypto.engines.SkipjackEngine',
+        'org.bouncycastle.crypto.engines.TEAEngine',
+        'org.bouncycastle.crypto.engines.TwofishEngine',
+        'org.bouncycastle.crypto.engines.VMPCEngine',
+        'org.bouncycastle.crypto.engines.XTEAEngine',
+        'org.bouncycastle.crypto.generators.BCrypt',
+        'org.bouncycastle.crypto.generators.OpenSSLPBEParametersGenerator',
+        'org.bouncycastle.crypto.generators.PKCS5S1ParametersGenerator',
+        'org.bouncycastle.crypto.generators.PKCS5S2ParametersGenerator',
+        'org.bouncycastle.crypto.io.CipherInputStream',
+        'org.bouncycastle.crypto.io.CipherOutputStream',
+        'org.bouncycastle.crypto.macs.HMac',
+        'org.bouncycastle.crypto.modes.AEADBlockCipher',
+        'org.bouncycastle.crypto.modes.CBCBlockCipher',
+        'org.bouncycastle.crypto.modes.CCMBlockCipher',
+        'org.bouncycastle.crypto.modes.CFBBlockCipher',
+        'org.bouncycastle.crypto.modes.EAXBlockCipher',
+        'org.bouncycastle.crypto.modes.GCMBlockCipher',
+        'org.bouncycastle.crypto.modes.OCBBlockCipher',
+        'org.bouncycastle.crypto.modes.OFBBlockCipher',
+        'org.bouncycastle.crypto.paddings.BlockCipherPadding',
+        'org.bouncycastle.crypto.paddings.ISO10126d2Padding',
+        'org.bouncycastle.crypto.paddings.ISO7816d4Padding',
+        'org.bouncycastle.crypto.paddings.PKCS7Padding',
+        'org.bouncycastle.crypto.paddings.PaddedBufferedBlockCipher',
+        'org.bouncycastle.crypto.paddings.TBCPadding',
+        'org.bouncycastle.crypto.paddings.X923Padding',
+        'org.bouncycastle.crypto.paddings.ZeroBytePadding',
+        'org.bouncycastle.crypto.params.AEADParameters',
+        'org.bouncycastle.crypto.params.AsymmetricKeyParameter',
+        'org.bouncycastle.crypto.params.DSAKeyParameters',
+        'org.bouncycastle.crypto.params.DSAParameters',
+        'org.bouncycastle.crypto.params.DSAPrivateKeyParameters',
+        'org.bouncycastle.crypto.params.DSAPublicKeyParameters',
+        'org.bouncycastle.crypto.params.ECDomainParameters',
+        'org.bouncycastle.crypto.params.ECKeyParameters',
+        'org.bouncycastle.crypto.params.ECPrivateKeyParameters',
+        'org.bouncycastle.crypto.params.ECPublicKeyParameters',
+        'org.bouncycastle.crypto.params.KeyParameter',
+        'org.bouncycastle.crypto.params.ParametersWithIV',
+        'org.bouncycastle.crypto.params.RC2Parameters',
+        'org.bouncycastle.crypto.params.RC5Parameters',
+        'org.bouncycastle.crypto.params.RSAKeyParameters',
+        'org.bouncycastle.crypto.params.RSAPrivateCrtKeyParameters',
+        'org.bouncycastle.crypto.prng.EntropySource',
+        'org.bouncycastle.crypto.prng.SP800SecureRandom',
+        'org.bouncycastle.crypto.prng.SP800SecureRandomBuilder',
+        'org.bouncycastle.crypto.prng.drbg.HashSP800DRBG',
+        'org.bouncycastle.crypto.prng.drbg.SP80090DRBG',
+        'org.bouncycastle.crypto.signers.DSASigner',
+        'org.bouncycastle.crypto.signers.ECDSASigner',
+        'org.bouncycastle.crypto.signers.RSADigestSigner',
+        'org.bouncycastle.crypto.util.PrivateKeyFactory',
+        'org.bouncycastle.crypto.util.PrivateKeyInfoFactory',
+        'org.bouncycastle.crypto.util.PublicKeyFactory',
+        'org.bouncycastle.crypto.util.SubjectPublicKeyInfoFactory',
+        'org.bouncycastle.jcajce.provider.asymmetric.dsa.KeyPairGeneratorSpi',
+        'org.bouncycastle.jcajce.provider.asymmetric.ec.KeyPairGeneratorSpi$EC',
+        'org.bouncycastle.jcajce.provider.asymmetric.rsa.KeyPairGeneratorSpi',
+        'org.bouncycastle.jcajce.provider.asymmetric.util.EC5Util',
+        'org.bouncycastle.jcajce.provider.asymmetric.util.ECUtil',
+        'org.bouncycastle.jce.spec.ECNamedCurveGenParameterSpec',
+        'org.bouncycastle.math.ec.ECFieldElement',
+        'org.bouncycastle.math.ec.ECPoint',
+        'org.bouncycastle.openssl.jcajce.JcaPEMWriter',
+        'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+        'org.bouncycastle.util.Arrays',
+        'org.bouncycastle.util.Strings',
+        'org.bouncycastle.util.io.Streams',
+        'org.bouncycastle.x509.extension.X509ExtensionUtil'
+    )
+
+  ignoreViolations(
+    // Guava uses internal java api: sun.misc.Unsafe
+    'com.google.common.cache.Striped64',
+    'com.google.common.cache.Striped64$1',
+    'com.google.common.cache.Striped64$Cell',
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+    'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+    'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+  )
+}
+
+thirdPartyAudit.ignoreMissingClasses(
+  'javax.xml.bind.JAXBContext',
+  'javax.xml.bind.JAXBElement',
+  'javax.xml.bind.JAXBException',
+  'javax.xml.bind.Unmarshaller',
+  'javax.xml.bind.UnmarshallerHandler',
+)
+
+
+test {
+  /*
+   * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
+   * other if we allow them to set the number of available processors as it's set-once in Netty.
+   */
+  systemProperty 'es.set.netty.runtime.available.processors', 'false'
+}
+
+// xpack modules are installed in real clusters as the meta plugin, so
+// installing them as individual plugins for integ tests doesn't make sense,
+// so we disable integ tests
+integTest.enabled = false
+
+// add all sub-projects of the qa sub-project
+gradle.projectsEvaluated {
+  project.subprojects
+    .find { it.path == project.path + ":qa" }
+    .subprojects
+    .findAll { it.path.startsWith(project.path + ":qa") }
+    .each { check.dependsOn it.check }
+}
+
diff --git a/x-pack/plugin/identity-provider/forbidden/xml-signatures.txt b/x-pack/plugin/identity-provider/forbidden/xml-signatures.txt
new file mode 100644
index 0000000000000..0acb0db2da1a7
--- /dev/null
+++ b/x-pack/plugin/identity-provider/forbidden/xml-signatures.txt
@@ -0,0 +1,8 @@
+@defaultMessage DocumentBuilderFactory should not be used directly. (See x-pack-security SamlUtils#getHardenedDocumentBuilder if you ever need one)
+javax.xml.parsers.DocumentBuilderFactory#newInstance()
+javax.xml.parsers.DocumentBuilderFactory#newInstance(java.lang.String, java.lang.ClassLoader)
+
+@defaultMessage TransformerFactory should not be used directly. Use IdPSamlTestCase#getHardenedXMLTransformer() instead.
+javax.xml.transform.TransformerFactory#newInstance()
+javax.xml.transform.TransformerFactory#newInstance(java.lang.String, java.lang.ClassLoader)
+
diff --git a/x-pack/plugin/identity-provider/licenses/cryptacular-1.2.4.jar.sha1 b/x-pack/plugin/identity-provider/licenses/cryptacular-1.2.4.jar.sha1
new file mode 100644
index 0000000000000..19095bb5dff64
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/cryptacular-1.2.4.jar.sha1
@@ -0,0 +1 @@
+4994c015d87886212683245d13e87f6fb903a760
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/cryptacular-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/cryptacular-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/cryptacular-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/cryptacular-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/cryptacular-NOTICE.txt
new file mode 100644
index 0000000000000..f17da0d736217
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/cryptacular-NOTICE.txt
@@ -0,0 +1,710 @@
+
+
+
+
+
+
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+  <link rel="dns-prefetch" href="https://assets-cdn.github.com">
+  <link rel="dns-prefetch" href="https://avatars0.githubusercontent.com">
+  <link rel="dns-prefetch" href="https://avatars1.githubusercontent.com">
+  <link rel="dns-prefetch" href="https://avatars2.githubusercontent.com">
+  <link rel="dns-prefetch" href="https://avatars3.githubusercontent.com">
+  <link rel="dns-prefetch" href="https://github-cloud.s3.amazonaws.com">
+  <link rel="dns-prefetch" href="https://user-images.githubusercontent.com/">
+
+
+
+  <link crossorigin="anonymous" href="https://assets-cdn.github.com/assets/frameworks-2d2d4c150f7000385741c6b992b302689ecd172246c6428904e0813be9bceca6.css" media="all" rel="stylesheet" />
+  <link crossorigin="anonymous" href="https://assets-cdn.github.com/assets/github-0522ae8d3b3bdc841d2f91f90efd5f1fd9040d910905674cd134ced43a6dfea6.css" media="all" rel="stylesheet" />
+  
+  
+  <link crossorigin="anonymous" href="https://assets-cdn.github.com/assets/site-cfab053e93f0e27f4c63d4ff6b7957bd25f711667fe678e747f8a4d88c47b38d.css" media="all" rel="stylesheet" />
+  
+
+  <meta name="viewport" content="width=device-width">
+  
+  <title>cryptacular/NOTICE at master · vt-middleware/cryptacular · GitHub</title>
+  <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="GitHub">
+  <link rel="fluid-icon" href="https://github.com/fluidicon.png" title="GitHub">
+  <meta property="fb:app_id" content="1401488693436528">
+
+    
+    <meta content="https://avatars7.githubusercontent.com/u/6122907?v=4&amp;s=400" property="og:image" /><meta content="GitHub" property="og:site_name" /><meta content="object" property="og:type" /><meta content="vt-middleware/cryptacular" property="og:title" /><meta content="https://github.com/vt-middleware/cryptacular" property="og:url" /><meta content="cryptacular - The friendly complement to the BouncyCastle crypto API for Java." property="og:description" />
+
+  <link rel="assets" href="https://assets-cdn.github.com/">
+  
+  <meta name="pjax-timeout" content="1000">
+  
+  <meta name="request-id" content="E0E4:26F16:12A5AE6:1D11801:596C7978" data-pjax-transient>
+  
+
+  <meta name="selected-link" value="repo_source" data-pjax-transient>
+
+  <meta name="google-site-verification" content="KT5gs8h0wvaagLKAVWq8bbeNwnZZK1r1XQysX3xurLU">
+<meta name="google-site-verification" content="ZzhVyEFwb7w3e0-uOTltm8Jsck2F5StVihD0exw2fsA">
+    <meta name="google-analytics" content="UA-3769691-2">
+
+<meta content="collector.githubapp.com" name="octolytics-host" /><meta content="github" name="octolytics-app-id" /><meta content="https://collector.githubapp.com/github-external/browser_event" name="octolytics-event-url" /><meta content="E0E4:26F16:12A5AE6:1D11801:596C7978" name="octolytics-dimension-request_id" /><meta content="sea" name="octolytics-dimension-region_edge" /><meta content="iad" name="octolytics-dimension-region_render" />
+<meta content="/&lt;user-name&gt;/&lt;repo-name&gt;/blob/show" data-pjax-transient="true" name="analytics-location" />
+
+
+
+
+  <meta class="js-ga-set" name="dimension1" content="Logged Out">
+
+
+  
+
+      <meta name="hostname" content="github.com">
+  <meta name="user-login" content="">
+
+      <meta name="expected-hostname" content="github.com">
+    <meta name="js-proxy-site-detection-payload" content="N2ZhMjk0NTA4MjI1NmZhYTVlNzM5NzVjZmFkOWY2NGFkNmMxYzcyMGViNzAzZGQxMGMzZmJhZDQ3YWZiZTI0OHx7InJlbW90ZV9hZGRyZXNzIjoiMTEwLjIwLjIyMC4xMzUiLCJyZXF1ZXN0X2lkIjoiRTBFNDoyNkYxNjoxMkE1QUU2OjFEMTE4MDE6NTk2Qzc5NzgiLCJ0aW1lc3RhbXAiOjE1MDAyODEyMDgsImhvc3QiOiJnaXRodWIuY29tIn0=">
+
+
+  <meta name="html-safe-nonce" content="5aa226b80a18dc40894e1d405e4eb31cfca7d616">
+
+  <meta http-equiv="x-pjax-version" content="f682644ce1bb9629b9d9d9bedf64801b">
+  
+
+      <link href="https://github.com/vt-middleware/cryptacular/commits/master.atom" rel="alternate" title="Recent Commits to cryptacular:master" type="application/atom+xml">
+
+  <meta name="description" content="cryptacular - The friendly complement to the BouncyCastle crypto API for Java.">
+  <meta name="go-import" content="github.com/vt-middleware/cryptacular git https://github.com/vt-middleware/cryptacular.git">
+
+  <meta content="6122907" name="octolytics-dimension-user_id" /><meta content="vt-middleware" name="octolytics-dimension-user_login" /><meta content="15714989" name="octolytics-dimension-repository_id" /><meta content="vt-middleware/cryptacular" name="octolytics-dimension-repository_nwo" /><meta content="true" name="octolytics-dimension-repository_public" /><meta content="false" name="octolytics-dimension-repository_is_fork" /><meta content="15714989" name="octolytics-dimension-repository_network_root_id" /><meta content="vt-middleware/cryptacular" name="octolytics-dimension-repository_network_root_nwo" /><meta content="false" name="octolytics-dimension-repository_explore_github_marketplace_ci_cta_shown" />
+
+
+    <link rel="canonical" href="https://github.com/vt-middleware/cryptacular/blob/master/NOTICE" data-pjax-transient>
+
+
+  <meta name="browser-stats-url" content="https://api.github.com/_private/browser/stats">
+
+  <meta name="browser-errors-url" content="https://api.github.com/_private/browser/errors">
+
+  <link rel="mask-icon" href="https://assets-cdn.github.com/pinned-octocat.svg" color="#000000">
+  <link rel="icon" type="image/x-icon" href="https://assets-cdn.github.com/favicon.ico">
+
+<meta name="theme-color" content="#1e2327">
+
+
+
+  </head>
+
+  <body class="logged-out env-production page-blob">
+    
+
+
+
+  <div class="position-relative js-header-wrapper ">
+    <a href="#start-of-content" tabindex="1" class="px-2 py-4 show-on-focus js-skip-to-content">Skip to content</a>
+    <div id="js-pjax-loader-bar" class="pjax-loader-bar"><div class="progress"></div></div>
+
+    
+    
+    
+
+
+
+          <header class="site-header js-details-container Details" role="banner">
+  <div class="site-nav-container">
+    <a class="header-logo-invertocat" href="https://github.com/" aria-label="Homepage" data-ga-click="(Logged out) Header, go to homepage, icon:logo-wordmark">
+      <svg aria-hidden="true" class="octicon octicon-mark-github" height="32" version="1.1" viewBox="0 0 16 16" width="32"><path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"/></svg>
+    </a>
+
+    <button class="btn-link float-right site-header-toggle js-details-target" type="button" aria-label="Toggle navigation" aria-expanded="false">
+      <svg aria-hidden="true" class="octicon octicon-three-bars" height="24" version="1.1" viewBox="0 0 12 16" width="18"><path fill-rule="evenodd" d="M11.41 9H.59C0 9 0 8.59 0 8c0-.59 0-1 .59-1H11.4c.59 0 .59.41.59 1 0 .59 0 1-.59 1h.01zm0-4H.59C0 5 0 4.59 0 4c0-.59 0-1 .59-1H11.4c.59 0 .59.41.59 1 0 .59 0 1-.59 1h.01zM.59 11H11.4c.59 0 .59.41.59 1 0 .59 0 1-.59 1H.59C0 13 0 12.59 0 12c0-.59 0-1 .59-1z"/></svg>
+    </button>
+
+    <div class="site-header-menu">
+      <nav class="site-header-nav">
+        <a href="/features" class="js-selected-navigation-item nav-item" data-ga-click="Header, click, Nav menu - item:features" data-selected-links="/features /features/code-review /features/project-management /features/integrations /features">
+          Features
+</a>        <a href="/business" class="js-selected-navigation-item nav-item" data-ga-click="Header, click, Nav menu - item:business" data-selected-links="/business /business/security /business/customers /business">
+          Business
+</a>        <a href="/explore" class="js-selected-navigation-item nav-item" data-ga-click="Header, click, Nav menu - item:explore" data-selected-links="/explore /trending /trending/developers /stars /integrations /integrations/feature/code /integrations/feature/collaborate /integrations/feature/ship showcases showcases_search showcases_landing /explore">
+          Explore
+</a>            <a href="/marketplace" class="js-selected-navigation-item nav-item" data-ga-click="Header, click, Nav menu - item:marketplace" data-selected-links=" /marketplace">
+              Marketplace
+</a>        <a href="/pricing" class="js-selected-navigation-item nav-item" data-ga-click="Header, click, Nav menu - item:pricing" data-selected-links="/pricing /pricing/developer /pricing/team /pricing/business-hosted /pricing/business-enterprise /pricing">
+          Pricing
+</a>      </nav>
+
+      <div class="site-header-actions">
+          <div class="header-search scoped-search site-scoped-search js-site-search" role="search">
+  <!-- '"` --><!-- </textarea></xmp> --></option></form><form accept-charset="UTF-8" action="/vt-middleware/cryptacular/search" class="js-site-search-form" data-scoped-search-url="/vt-middleware/cryptacular/search" data-unscoped-search-url="/search" method="get"><div style="margin:0;padding:0;display:inline"><input name="utf8" type="hidden" value="&#x2713;" /></div>
+    <label class="form-control header-search-wrapper js-chromeless-input-container">
+        <a href="/vt-middleware/cryptacular/blob/master/NOTICE" class="header-search-scope no-underline">This repository</a>
+      <input type="text"
+        class="form-control header-search-input js-site-search-focus js-site-search-field is-clearable"
+        data-hotkey="s"
+        name="q"
+        value=""
+        placeholder="Search"
+        aria-label="Search this repository"
+        data-unscoped-placeholder="Search GitHub"
+        data-scoped-placeholder="Search"
+        autocapitalize="off">
+        <input type="hidden" class="js-site-search-type-field" name="type" >
+    </label>
+</form></div>
+
+
+          <a class="text-bold site-header-link" href="/login?return_to=%2Fvt-middleware%2Fcryptacular%2Fblob%2Fmaster%2FNOTICE" data-ga-click="(Logged out) Header, clicked Sign in, text:sign-in">Sign in</a>
+            <span class="text-gray">or</span>
+            <a class="text-bold site-header-link" href="/join?source=header-repo" data-ga-click="(Logged out) Header, clicked Sign up, text:sign-up">Sign up</a>
+      </div>
+    </div>
+  </div>
+</header>
+
+
+  </div>
+
+  <div id="start-of-content" class="show-on-focus"></div>
+
+    <div id="js-flash-container">
+</div>
+
+
+
+  <div role="main">
+        <div itemscope itemtype="http://schema.org/SoftwareSourceCode">
+    <div id="js-repo-pjax-container" data-pjax-container>
+      
+
+
+
+  
+
+
+    <div class="pagehead repohead instapaper_ignore readability-menu experiment-repo-nav">
+      <div class="container repohead-details-container">
+
+        <ul class="pagehead-actions">
+  <li>
+      <a href="/login?return_to=%2Fvt-middleware%2Fcryptacular"
+    class="btn btn-sm btn-with-count tooltipped tooltipped-n"
+    aria-label="You must be signed in to watch a repository" rel="nofollow">
+    <svg aria-hidden="true" class="octicon octicon-eye" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M8.06 2C3 2 0 8 0 8s3 6 8.06 6C13 14 16 8 16 8s-3-6-7.94-6zM8 12c-2.2 0-4-1.78-4-4 0-2.2 1.8-4 4-4 2.22 0 4 1.8 4 4 0 2.22-1.78 4-4 4zm2-4c0 1.11-.89 2-2 2-1.11 0-2-.89-2-2 0-1.11.89-2 2-2 1.11 0 2 .89 2 2z"/></svg>
+    Watch
+  </a>
+  <a class="social-count" href="/vt-middleware/cryptacular/watchers"
+     aria-label="9 users are watching this repository">
+    9
+  </a>
+
+  </li>
+
+  <li>
+      <a href="/login?return_to=%2Fvt-middleware%2Fcryptacular"
+    class="btn btn-sm btn-with-count tooltipped tooltipped-n"
+    aria-label="You must be signed in to star a repository" rel="nofollow">
+    <svg aria-hidden="true" class="octicon octicon-star" height="16" version="1.1" viewBox="0 0 14 16" width="14"><path fill-rule="evenodd" d="M14 6l-4.9-.64L7 1 4.9 5.36 0 6l3.6 3.26L2.67 14 7 11.67 11.33 14l-.93-4.74z"/></svg>
+    Star
+  </a>
+
+    <a class="social-count js-social-count" href="/vt-middleware/cryptacular/stargazers"
+      aria-label="15 users starred this repository">
+      15
+    </a>
+
+  </li>
+
+  <li>
+      <a href="/login?return_to=%2Fvt-middleware%2Fcryptacular"
+        class="btn btn-sm btn-with-count tooltipped tooltipped-n"
+        aria-label="You must be signed in to fork a repository" rel="nofollow">
+        <svg aria-hidden="true" class="octicon octicon-repo-forked" height="16" version="1.1" viewBox="0 0 10 16" width="10"><path fill-rule="evenodd" d="M8 1a1.993 1.993 0 0 0-1 3.72V6L5 8 3 6V4.72A1.993 1.993 0 0 0 2 1a1.993 1.993 0 0 0-1 3.72V6.5l3 3v1.78A1.993 1.993 0 0 0 5 15a1.993 1.993 0 0 0 1-3.72V9.5l3-3V4.72A1.993 1.993 0 0 0 8 1zM2 4.2C1.34 4.2.8 3.65.8 3c0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2zm3 10c-.66 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2zm3-10c-.66 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2z"/></svg>
+        Fork
+      </a>
+
+    <a href="/vt-middleware/cryptacular/network" class="social-count"
+       aria-label="1 user forked this repository">
+      1
+    </a>
+  </li>
+</ul>
+
+        <h1 class="public ">
+  <svg aria-hidden="true" class="octicon octicon-repo" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M4 9H3V8h1v1zm0-3H3v1h1V6zm0-2H3v1h1V4zm0-2H3v1h1V2zm8-1v12c0 .55-.45 1-1 1H6v2l-1.5-1.5L3 16v-2H1c-.55 0-1-.45-1-1V1c0-.55.45-1 1-1h10c.55 0 1 .45 1 1zm-1 10H1v2h2v-1h3v1h5v-2zm0-10H2v9h9V1z"/></svg>
+  <span class="author" itemprop="author"><a href="/vt-middleware" class="url fn" rel="author">vt-middleware</a></span><!--
+--><span class="path-divider">/</span><!--
+--><strong itemprop="name"><a href="/vt-middleware/cryptacular" data-pjax="#js-repo-pjax-container">cryptacular</a></strong>
+
+</h1>
+
+      </div>
+      <div class="container">
+        
+<nav class="reponav js-repo-nav js-sidenav-container-pjax"
+     itemscope
+     itemtype="http://schema.org/BreadcrumbList"
+     role="navigation"
+     data-pjax="#js-repo-pjax-container">
+
+  <span itemscope itemtype="http://schema.org/ListItem" itemprop="itemListElement">
+    <a href="/vt-middleware/cryptacular" class="js-selected-navigation-item selected reponav-item" data-hotkey="g c" data-selected-links="repo_source repo_downloads repo_commits repo_releases repo_tags repo_branches /vt-middleware/cryptacular" itemprop="url">
+      <svg aria-hidden="true" class="octicon octicon-code" height="16" version="1.1" viewBox="0 0 14 16" width="14"><path fill-rule="evenodd" d="M9.5 3L8 4.5 11.5 8 8 11.5 9.5 13 14 8 9.5 3zm-5 0L0 8l4.5 5L6 11.5 2.5 8 6 4.5 4.5 3z"/></svg>
+      <span itemprop="name">Code</span>
+      <meta itemprop="position" content="1">
+</a>  </span>
+
+    <span itemscope itemtype="http://schema.org/ListItem" itemprop="itemListElement">
+      <a href="/vt-middleware/cryptacular/issues" class="js-selected-navigation-item reponav-item" data-hotkey="g i" data-selected-links="repo_issues repo_labels repo_milestones /vt-middleware/cryptacular/issues" itemprop="url">
+        <svg aria-hidden="true" class="octicon octicon-issue-opened" height="16" version="1.1" viewBox="0 0 14 16" width="14"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"/></svg>
+        <span itemprop="name">Issues</span>
+        <span class="Counter">4</span>
+        <meta itemprop="position" content="2">
+</a>    </span>
+
+  <span itemscope itemtype="http://schema.org/ListItem" itemprop="itemListElement">
+    <a href="/vt-middleware/cryptacular/pulls" class="js-selected-navigation-item reponav-item" data-hotkey="g p" data-selected-links="repo_pulls /vt-middleware/cryptacular/pulls" itemprop="url">
+      <svg aria-hidden="true" class="octicon octicon-git-pull-request" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M11 11.28V5c-.03-.78-.34-1.47-.94-2.06C9.46 2.35 8.78 2.03 8 2H7V0L4 3l3 3V4h1c.27.02.48.11.69.31.21.2.3.42.31.69v6.28A1.993 1.993 0 0 0 10 15a1.993 1.993 0 0 0 1-3.72zm-1 2.92c-.66 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2zM4 3c0-1.11-.89-2-2-2a1.993 1.993 0 0 0-1 3.72v6.56A1.993 1.993 0 0 0 2 15a1.993 1.993 0 0 0 1-3.72V4.72c.59-.34 1-.98 1-1.72zm-.8 10c0 .66-.55 1.2-1.2 1.2-.65 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2zM2 4.2C1.34 4.2.8 3.65.8 3c0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2z"/></svg>
+      <span itemprop="name">Pull requests</span>
+      <span class="Counter">1</span>
+      <meta itemprop="position" content="3">
+</a>  </span>
+
+    <a href="/vt-middleware/cryptacular/projects" class="js-selected-navigation-item reponav-item" data-selected-links="repo_projects new_repo_project repo_project /vt-middleware/cryptacular/projects">
+      <svg aria-hidden="true" class="octicon octicon-project" height="16" version="1.1" viewBox="0 0 15 16" width="15"><path fill-rule="evenodd" d="M10 12h3V2h-3v10zm-4-2h3V2H6v8zm-4 4h3V2H2v12zm-1 1h13V1H1v14zM14 0H1a1 1 0 0 0-1 1v14a1 1 0 0 0 1 1h13a1 1 0 0 0 1-1V1a1 1 0 0 0-1-1z"/></svg>
+      Projects
+      <span class="Counter" >0</span>
+</a>
+
+
+    <div class="reponav-dropdown js-menu-container">
+      <button type="button" class="btn-link reponav-item reponav-dropdown js-menu-target " data-no-toggle aria-expanded="false" aria-haspopup="true">
+        Insights
+        <svg aria-hidden="true" class="octicon octicon-triangle-down v-align-middle text-gray" height="11" version="1.1" viewBox="0 0 12 16" width="8"><path fill-rule="evenodd" d="M0 5l6 6 6-6z"/></svg>
+      </button>
+      <div class="dropdown-menu-content js-menu-content">
+        <div class="dropdown-menu dropdown-menu-sw">
+          <a class="dropdown-item" href="/vt-middleware/cryptacular/pulse" data-skip-pjax>
+            <svg aria-hidden="true" class="octicon octicon-pulse" height="16" version="1.1" viewBox="0 0 14 16" width="14"><path fill-rule="evenodd" d="M11.5 8L8.8 5.4 6.6 8.5 5.5 1.6 2.38 8H0v2h3.6l.9-1.8.9 5.4L9 8.5l1.6 1.5H14V8z"/></svg>
+            Pulse
+          </a>
+          <a class="dropdown-item" href="/vt-middleware/cryptacular/graphs" data-skip-pjax>
+            <svg aria-hidden="true" class="octicon octicon-graph" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M16 14v1H0V0h1v14h15zM5 13H3V8h2v5zm4 0H7V3h2v10zm4 0h-2V6h2v7z"/></svg>
+            Graphs
+          </a>
+        </div>
+      </div>
+    </div>
+</nav>
+
+      </div>
+    </div>
+
+<div class="container new-discussion-timeline experiment-repo-nav">
+  <div class="repository-content">
+
+    
+  <a href="/vt-middleware/cryptacular/blob/c26911e3cd28497ce9daa3ce682e09cb2d1d8688/NOTICE" class="d-none js-permalink-shortcut" data-hotkey="y">Permalink</a>
+
+  <!-- blob contrib key: blob_contributors:v21:f5054b9e46039e0ad937c69e6151b7d4 -->
+
+  <div class="file-navigation js-zeroclipboard-container">
+    
+<div class="select-menu branch-select-menu js-menu-container js-select-menu float-left">
+  <button class=" btn btn-sm select-menu-button js-menu-target css-truncate" data-hotkey="w"
+    
+    type="button" aria-label="Switch branches or tags" aria-expanded="false" aria-haspopup="true">
+      <i>Branch:</i>
+      <span class="js-select-button css-truncate-target">master</span>
+  </button>
+
+  <div class="select-menu-modal-holder js-menu-content js-navigation-container" data-pjax>
+
+    <div class="select-menu-modal">
+      <div class="select-menu-header">
+        <svg aria-label="Close" class="octicon octicon-x js-menu-close" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M7.48 8l3.75 3.75-1.48 1.48L6 9.48l-3.75 3.75-1.48-1.48L4.52 8 .77 4.25l1.48-1.48L6 6.52l3.75-3.75 1.48 1.48z"/></svg>
+        <span class="select-menu-title">Switch branches/tags</span>
+      </div>
+
+      <div class="select-menu-filters">
+        <div class="select-menu-text-filter">
+          <input type="text" aria-label="Filter branches/tags" id="context-commitish-filter-field" class="form-control js-filterable-field js-navigation-enable" placeholder="Filter branches/tags">
+        </div>
+        <div class="select-menu-tabs">
+          <ul>
+            <li class="select-menu-tab">
+              <a href="#" data-tab-filter="branches" data-filter-placeholder="Filter branches/tags" class="js-select-menu-tab" role="tab">Branches</a>
+            </li>
+            <li class="select-menu-tab">
+              <a href="#" data-tab-filter="tags" data-filter-placeholder="Find a tag…" class="js-select-menu-tab" role="tab">Tags</a>
+            </li>
+          </ul>
+        </div>
+      </div>
+
+      <div class="select-menu-list select-menu-tab-bucket js-select-menu-tab-bucket" data-tab-filter="branches" role="menu">
+
+        <div data-filterable-for="context-commitish-filter-field" data-filterable-type="substring">
+
+
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/vt-middleware/cryptacular/blob/ISSUE-31+32/NOTICE"
+               data-name="ISSUE-31+32"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text">
+                ISSUE-31+32
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/vt-middleware/cryptacular/blob/gh-pages/NOTICE"
+               data-name="gh-pages"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text">
+                gh-pages
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open selected"
+               href="/vt-middleware/cryptacular/blob/master/NOTICE"
+               data-name="master"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text">
+                master
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/vt-middleware/cryptacular/blob/v1.1/NOTICE"
+               data-name="v1.1"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text">
+                v1.1
+              </span>
+            </a>
+        </div>
+
+          <div class="select-menu-no-results">Nothing to show</div>
+      </div>
+
+      <div class="select-menu-list select-menu-tab-bucket js-select-menu-tab-bucket" data-tab-filter="tags">
+        <div data-filterable-for="context-commitish-filter-field" data-filterable-type="substring">
+
+
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.2.1/NOTICE"
+              data-name="v1.2.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.2.1">
+                v1.2.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.2.0/NOTICE"
+              data-name="v1.2.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.2.0">
+                v1.2.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.1.2/NOTICE"
+              data-name="v1.1.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.1.2">
+                v1.1.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.1.1/NOTICE"
+              data-name="v1.1.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.1.1">
+                v1.1.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.1.0/NOTICE"
+              data-name="v1.1.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.1.0">
+                v1.1.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0/NOTICE"
+              data-name="v1.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0">
+                v1.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0-RC6/NOTICE"
+              data-name="v1.0-RC6"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0-RC6">
+                v1.0-RC6
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0-RC4/NOTICE"
+              data-name="v1.0-RC4"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0-RC4">
+                v1.0-RC4
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0-RC3/NOTICE"
+              data-name="v1.0-RC3"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0-RC3">
+                v1.0-RC3
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0-RC2/NOTICE"
+              data-name="v1.0-RC2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0-RC2">
+                v1.0-RC2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/vt-middleware/cryptacular/tree/v1.0-RC1/NOTICE"
+              data-name="v1.0-RC1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M12 5l-8 8-4-4 1.5-1.5L4 10l6.5-6.5z"/></svg>
+              <span class="select-menu-item-text css-truncate-target" title="v1.0-RC1">
+                v1.0-RC1
+              </span>
+            </a>
+        </div>
+
+        <div class="select-menu-no-results">Nothing to show</div>
+      </div>
+
+    </div>
+  </div>
+</div>
+
+    <div class="BtnGroup float-right">
+      <a href="/vt-middleware/cryptacular/find/master"
+            class="js-pjax-capture-input btn btn-sm BtnGroup-item"
+            data-pjax
+            data-hotkey="t">
+        Find file
+      </a>
+      <button aria-label="Copy file path to clipboard" class="js-zeroclipboard btn btn-sm BtnGroup-item tooltipped tooltipped-s" data-copied-hint="Copied!" type="button">Copy path</button>
+    </div>
+    <div class="breadcrumb js-zeroclipboard-target">
+      <span class="repo-root js-repo-root"><span class="js-path-segment"><a href="/vt-middleware/cryptacular"><span>cryptacular</span></a></span></span><span class="separator">/</span><strong class="final-path">NOTICE</strong>
+    </div>
+  </div>
+
+
+  
+  <div class="commit-tease">
+      <span class="float-right">
+        <a class="commit-tease-sha" href="/vt-middleware/cryptacular/commit/6dd6f199ac3ecc3b4c5aef9e04be3bbe265a30a1" data-pjax>
+          6dd6f19
+        </a>
+        <relative-time datetime="2017-07-06T22:28:36Z">Jul 7, 2017</relative-time>
+      </span>
+      <div>
+        <img alt="@dfish3r" class="avatar" height="20" src="https://avatars6.githubusercontent.com/u/1051499?v=4&amp;s=40" width="20" />
+        <a href="/dfish3r" class="user-mention" rel="contributor">dfish3r</a>
+          <a href="/vt-middleware/cryptacular/commit/6dd6f199ac3ecc3b4c5aef9e04be3bbe265a30a1" class="message" data-pjax="true" title="Update year in notice.">Update year in notice.</a>
+      </div>
+
+    <div class="commit-tease-contributors">
+      <button type="button" class="btn-link muted-link contributors-toggle" data-facebox="#blob_contributors_box">
+        <strong>1</strong>
+         contributor
+      </button>
+      
+    </div>
+
+    <div id="blob_contributors_box" style="display:none">
+      <h2 class="facebox-header" data-facebox-id="facebox-header">Users who have contributed to this file</h2>
+      <ul class="facebox-user-list" data-facebox-id="facebox-description">
+          <li class="facebox-user-list-item">
+            <img alt="@dfish3r" height="24" src="https://avatars4.githubusercontent.com/u/1051499?v=4&amp;s=48" width="24" />
+            <a href="/dfish3r">dfish3r</a>
+          </li>
+      </ul>
+    </div>
+  </div>
+
+  <div class="file">
+    <div class="file-header">
+  <div class="file-actions">
+
+    <div class="BtnGroup">
+      <a href="/vt-middleware/cryptacular/raw/master/NOTICE" class="btn btn-sm BtnGroup-item" id="raw-url">Raw</a>
+        <a href="/vt-middleware/cryptacular/blame/master/NOTICE" class="btn btn-sm js-update-url-with-hash BtnGroup-item" data-hotkey="b">Blame</a>
+      <a href="/vt-middleware/cryptacular/commits/master/NOTICE" class="btn btn-sm BtnGroup-item" rel="nofollow">History</a>
+    </div>
+
+
+        <button type="button" class="btn-octicon disabled tooltipped tooltipped-nw"
+          aria-label="You must be signed in to make or propose changes">
+          <svg aria-hidden="true" class="octicon octicon-pencil" height="16" version="1.1" viewBox="0 0 14 16" width="14"><path fill-rule="evenodd" d="M0 12v3h3l8-8-3-3-8 8zm3 2H1v-2h1v1h1v1zm10.3-9.3L12 6 9 3l1.3-1.3a.996.996 0 0 1 1.41 0l1.59 1.59c.39.39.39 1.02 0 1.41z"/></svg>
+        </button>
+        <button type="button" class="btn-octicon btn-octicon-danger disabled tooltipped tooltipped-nw"
+          aria-label="You must be signed in to make or propose changes">
+          <svg aria-hidden="true" class="octicon octicon-trashcan" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M11 2H9c0-.55-.45-1-1-1H5c-.55 0-1 .45-1 1H2c-.55 0-1 .45-1 1v1c0 .55.45 1 1 1v9c0 .55.45 1 1 1h7c.55 0 1-.45 1-1V5c.55 0 1-.45 1-1V3c0-.55-.45-1-1-1zm-1 12H3V5h1v8h1V5h1v8h1V5h1v8h1V5h1v9zm1-10H2V3h9v1z"/></svg>
+        </button>
+  </div>
+
+  <div class="file-info">
+      7 lines (5 sloc)
+      <span class="file-info-divider"></span>
+    165 Bytes
+  </div>
+</div>
+
+    
+
+  <div itemprop="text" class="blob-wrapper data type-text">
+      <table class="highlight tab-size js-file-line-container" data-tab-size="8">
+      <tr>
+        <td id="L1" class="blob-num js-line-number" data-line-number="1"></td>
+        <td id="LC1" class="blob-code blob-code-inner js-file-line">Cryptacular Java Library</td>
+      </tr>
+      <tr>
+        <td id="L2" class="blob-num js-line-number" data-line-number="2"></td>
+        <td id="LC2" class="blob-code blob-code-inner js-file-line">Copyright (C) 2003-2017 Virginia Tech.</td>
+      </tr>
+      <tr>
+        <td id="L3" class="blob-num js-line-number" data-line-number="3"></td>
+        <td id="LC3" class="blob-code blob-code-inner js-file-line">All rights reserved.</td>
+      </tr>
+      <tr>
+        <td id="L4" class="blob-num js-line-number" data-line-number="4"></td>
+        <td id="LC4" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L5" class="blob-num js-line-number" data-line-number="5"></td>
+        <td id="LC5" class="blob-code blob-code-inner js-file-line">This product includes software developed at</td>
+      </tr>
+      <tr>
+        <td id="L6" class="blob-num js-line-number" data-line-number="6"></td>
+        <td id="LC6" class="blob-code blob-code-inner js-file-line">Virginia Tech (http://www.vt.edu).</td>
+      </tr>
+</table>
+
+
+  </div>
+
+  </div>
+
+  <button type="button" data-facebox="#jump-to-line" data-facebox-class="linejump" data-hotkey="l" class="d-none">Jump to Line</button>
+  <div id="jump-to-line" style="display:none">
+    <!-- '"` --><!-- </textarea></xmp> --></option></form><form accept-charset="UTF-8" action="" class="js-jump-to-line-form" method="get"><div style="margin:0;padding:0;display:inline"><input name="utf8" type="hidden" value="&#x2713;" /></div>
+      <input class="form-control linejump-input js-jump-to-line-field" type="text" placeholder="Jump to line&hellip;" aria-label="Jump to line" autofocus>
+      <button type="submit" class="btn">Go</button>
+</form>  </div>
+
+  </div>
+  <div class="modal-backdrop js-touch-events"></div>
+</div>
+
+    </div>
+  </div>
+
+  </div>
+
+      
+<div class="container site-footer-container">
+  <div class="site-footer " role="contentinfo">
+    <ul class="site-footer-links float-right">
+        <li><a href="https://github.com/contact" data-ga-click="Footer, go to contact, text:contact">Contact GitHub</a></li>
+      <li><a href="https://developer.github.com" data-ga-click="Footer, go to api, text:api">API</a></li>
+      <li><a href="https://training.github.com" data-ga-click="Footer, go to training, text:training">Training</a></li>
+      <li><a href="https://shop.github.com" data-ga-click="Footer, go to shop, text:shop">Shop</a></li>
+        <li><a href="https://github.com/blog" data-ga-click="Footer, go to blog, text:blog">Blog</a></li>
+        <li><a href="https://github.com/about" data-ga-click="Footer, go to about, text:about">About</a></li>
+
+    </ul>
+
+    <a href="https://github.com" aria-label="Homepage" class="site-footer-mark" title="GitHub">
+      <svg aria-hidden="true" class="octicon octicon-mark-github" height="24" version="1.1" viewBox="0 0 16 16" width="24"><path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"/></svg>
+</a>
+    <ul class="site-footer-links">
+      <li>&copy; 2017 <span title="0.10938s from unicorn-2925809464-f2ltq">GitHub</span>, Inc.</li>
+        <li><a href="https://github.com/site/terms" data-ga-click="Footer, go to terms, text:terms">Terms</a></li>
+        <li><a href="https://github.com/site/privacy" data-ga-click="Footer, go to privacy, text:privacy">Privacy</a></li>
+        <li><a href="https://github.com/security" data-ga-click="Footer, go to security, text:security">Security</a></li>
+        <li><a href="https://status.github.com/" data-ga-click="Footer, go to status, text:status">Status</a></li>
+        <li><a href="https://help.github.com" data-ga-click="Footer, go to help, text:help">Help</a></li>
+    </ul>
+  </div>
+</div>
+
+
+
+  <div id="ajax-error-message" class="ajax-error-message flash flash-error">
+    <svg aria-hidden="true" class="octicon octicon-alert" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M8.865 1.52c-.18-.31-.51-.5-.87-.5s-.69.19-.87.5L.275 13.5c-.18.31-.18.69 0 1 .19.31.52.5.87.5h13.7c.36 0 .69-.19.86-.5.17-.31.18-.69.01-1L8.865 1.52zM8.995 13h-2v-2h2v2zm0-3h-2V6h2v4z"/></svg>
+    <button type="button" class="flash-close js-flash-close js-ajax-error-dismiss" aria-label="Dismiss error">
+      <svg aria-hidden="true" class="octicon octicon-x" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M7.48 8l3.75 3.75-1.48 1.48L6 9.48l-3.75 3.75-1.48-1.48L4.52 8 .77 4.25l1.48-1.48L6 6.52l3.75-3.75 1.48 1.48z"/></svg>
+    </button>
+    You can't perform that action at this time.
+  </div>
+
+
+    <script crossorigin="anonymous" src="https://assets-cdn.github.com/assets/compat-91f98c37fc84eac24836eec2567e9912742094369a04c4eba6e3cd1fa18902d9.js"></script>
+    <script crossorigin="anonymous" src="https://assets-cdn.github.com/assets/frameworks-f84bb87b149685d1e6c6f057ee324f2cd496e677f5a359a8b5db853313bb83e6.js"></script>
+    
+    <script async="async" crossorigin="anonymous" src="https://assets-cdn.github.com/assets/github-13fa3aa50ac8f9fa9a7d198f0cd13b0905775d39446ad076d17d8f74a998438a.js"></script>
+    
+    
+    
+    
+  <div class="js-stale-session-flash stale-session-flash flash flash-warn flash-banner d-none">
+    <svg aria-hidden="true" class="octicon octicon-alert" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M8.865 1.52c-.18-.31-.51-.5-.87-.5s-.69.19-.87.5L.275 13.5c-.18.31-.18.69 0 1 .19.31.52.5.87.5h13.7c.36 0 .69-.19.86-.5.17-.31.18-.69.01-1L8.865 1.52zM8.995 13h-2v-2h2v2zm0-3h-2V6h2v4z"/></svg>
+    <span class="signed-in-tab-flash">You signed in with another tab or window. <a href="">Reload</a> to refresh your session.</span>
+    <span class="signed-out-tab-flash">You signed out in another tab or window. <a href="">Reload</a> to refresh your session.</span>
+  </div>
+  <div class="facebox" id="facebox" style="display:none;">
+  <div class="facebox-popup">
+    <div class="facebox-content" role="dialog" aria-labelledby="facebox-header" aria-describedby="facebox-description">
+    </div>
+    <button type="button" class="facebox-close js-facebox-close" aria-label="Close modal">
+      <svg aria-hidden="true" class="octicon octicon-x" height="16" version="1.1" viewBox="0 0 12 16" width="12"><path fill-rule="evenodd" d="M7.48 8l3.75 3.75-1.48 1.48L6 9.48l-3.75 3.75-1.48-1.48L4.52 8 .77 4.25l1.48-1.48L6 6.52l3.75-3.75 1.48 1.48z"/></svg>
+    </button>
+  </div>
+</div>
+
+
+  </body>
+</html>
+
diff --git a/x-pack/plugin/identity-provider/licenses/guava-19.0.jar.sha1 b/x-pack/plugin/identity-provider/licenses/guava-19.0.jar.sha1
new file mode 100644
index 0000000000000..02f937f821f45
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/guava-19.0.jar.sha1
@@ -0,0 +1 @@
+6ce200f6b23222af3d8abb6b6459e6c44f4bb0e9
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/guava-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/guava-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/guava-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/guava-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/guava-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/identity-provider/licenses/httpclient-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/httpclient-LICENSE.txt
new file mode 100644
index 0000000000000..32f01eda18fe9
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/httpclient-LICENSE.txt
@@ -0,0 +1,558 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+=========================================================================
+
+This project includes Public Suffix List copied from
+<https://publicsuffix.org/list/effective_tld_names.dat>
+licensed under the terms of the Mozilla Public License, v. 2.0
+
+Full license text: <http://mozilla.org/MPL/2.0/>
+
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.
diff --git a/x-pack/plugin/identity-provider/licenses/httpclient-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/httpclient-NOTICE.txt
new file mode 100644
index 0000000000000..91e5c40c4c6d3
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/httpclient-NOTICE.txt
@@ -0,0 +1,6 @@
+Apache HttpComponents Client
+Copyright 1999-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
diff --git a/x-pack/plugin/identity-provider/licenses/httpclient-cache-4.5.10.jar.sha1 b/x-pack/plugin/identity-provider/licenses/httpclient-cache-4.5.10.jar.sha1
new file mode 100644
index 0000000000000..3b1d8db643114
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/httpclient-cache-4.5.10.jar.sha1
@@ -0,0 +1 @@
+b195778247a21e980cb9f80c41364dc0c38feaef
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/java-support-7.5.1.jar.sha1 b/x-pack/plugin/identity-provider/licenses/java-support-7.5.1.jar.sha1
new file mode 100644
index 0000000000000..2d138de40005a
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/java-support-7.5.1.jar.sha1
@@ -0,0 +1 @@
+c3fecaa141e8f0fff8a14e6800aefa8155c9b3e8
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-2.11.1.jar.sha1 b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..6178556b31848
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-2.11.1.jar.sha1
@@ -0,0 +1 @@
+4b41b53a3a2d299ce381a69d165381ca19f62912
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-NOTICE.txt
new file mode 100644
index 0000000000000..ea99ef1d4726b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/log4j-slf4j-impl-NOTICE.txt
@@ -0,0 +1,8 @@
+
+Apache Log4j SLF4J Binding
+Copyright 1999-2017 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+
diff --git a/x-pack/plugin/identity-provider/licenses/metrics-core-3.2.2.jar.sha1 b/x-pack/plugin/identity-provider/licenses/metrics-core-3.2.2.jar.sha1
new file mode 100644
index 0000000000000..dedc1dfa8d0e4
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/metrics-core-3.2.2.jar.sha1
@@ -0,0 +1 @@
+cd9886f498ee2ab2d994f0c779e5553b2c450416
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/metrics-core-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/metrics-core-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/metrics-core-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/metrics-core-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/metrics-core-NOTICE.txt
new file mode 100644
index 0000000000000..255f6a193ad0f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/metrics-core-NOTICE.txt
@@ -0,0 +1,4 @@
+Dropwizard
+Copyright 2010-2013 Coda Hale and Yammer, Inc., 2014-2016 Dropwizard Team
+
+This product includes software developed by Coda Hale and Yammer, Inc.
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-core-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-core-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..be9ed5be9296b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-core-3.4.5.jar.sha1
@@ -0,0 +1 @@
+0958fae127de9e8b0296e6f089c7451b6d5f0846
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-messaging-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-messaging-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..c370c3bd6369c
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-messaging-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+e3ec93dfbf90c451e9f7fb34a3e33a6ac60edd31
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-messaging-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-messaging-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..9131bc2ac23c1
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-messaging-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+beaca9bd69ad861dbb55f1694853a02cb6988ae7
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-profile-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-profile-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..409c3b9b6db30
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-profile-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+bb0a1f97d38342a5715bad628ee24000b08e821e
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-profile-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-profile-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..38ce75e4c2da3
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-profile-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+6cb4595c7a988d964f6a2d55dcac754b0c68904e
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-saml-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-saml-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..d575629f16277
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-saml-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+bef43d21b2d878baceae291af4a0ad3449c7d7ec
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-saml-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-saml-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..f52b95e499521
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-saml-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+ecf4a9552575d38cffd4dc56d95e7564b7dccfc1
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-security-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-security-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..7db3d5ea3558e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-security-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+15cbb232ae6665edc5df5f260e551e69fdb362e5
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-security-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-security-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..e5bbed6dae0ba
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-security-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+b2bc1aa5b0f400aa50499f3783b10e9f7c216a47
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-soap-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-soap-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..840170214e9f7
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-soap-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+c497df002980c6e482ce7b828924bb24f60f99f7
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-soap-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-soap-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..219b3060af86c
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-soap-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+30ed8d37259e840df5b3fd8daf7b654129a9190c
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-storage-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-storage-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..2290462e05c65
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-storage-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+a984671fd04e50da03f68003d2b062578e63ec86
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-storage-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-storage-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..c511027f9aff4
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-storage-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+a4b828fe1a9d64953ecdd8a9e00ff31b63ad6ef0
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-api-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-api-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..44dea7d989bef
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-api-3.4.5.jar.sha1
@@ -0,0 +1 @@
+a1b10f97deca1e3405f95db5b39697c0d46f5e0d
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-impl-3.4.5.jar.sha1 b/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-impl-3.4.5.jar.sha1
new file mode 100644
index 0000000000000..dcdc61dc2e247
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/opensaml-xmlsec-impl-3.4.5.jar.sha1
@@ -0,0 +1 @@
+d46cb9854a1ff85bea34ece7077bc32dbc2f10da
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/shibboleth-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/shibboleth-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/shibboleth-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/shibboleth-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/shibboleth-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/identity-provider/licenses/slf4j-api-1.6.2.jar.sha1 b/x-pack/plugin/identity-provider/licenses/slf4j-api-1.6.2.jar.sha1
new file mode 100644
index 0000000000000..a2f93ea55802b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/slf4j-api-1.6.2.jar.sha1
@@ -0,0 +1 @@
+8619e95939167fb37245b5670135e4feb0ec7d50
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/slf4j-api-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/slf4j-api-LICENSE.txt
new file mode 100644
index 0000000000000..c8e72c2cd4738
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/slf4j-api-LICENSE.txt
@@ -0,0 +1,21 @@
+ Copyright (c) 2004-2017 QOS.ch
+ All rights reserved.
+
+ Permission is hereby granted, free  of charge, to any person obtaining
+ a  copy  of this  software  and  associated  documentation files  (the
+ "Software"), to  deal in  the Software without  restriction, including
+ without limitation  the rights to  use, copy, modify,  merge, publish,
+ distribute,  sublicense, and/or sell  copies of  the Software,  and to
+ permit persons to whom the Software  is furnished to do so, subject to
+ the following conditions:
+ 
+ The  above  copyright  notice  and  this permission  notice  shall  be
+ included in all copies or substantial portions of the Software.
+ 
+ THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
+ EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
+ MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/x-pack/plugin/identity-provider/licenses/slf4j-api-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/slf4j-api-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/identity-provider/licenses/xmlsec-2.1.4.jar.sha1 b/x-pack/plugin/identity-provider/licenses/xmlsec-2.1.4.jar.sha1
new file mode 100644
index 0000000000000..d85a4194f6a59
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/xmlsec-2.1.4.jar.sha1
@@ -0,0 +1 @@
+cb43326f02e3e77526c24269c8b5d3cc3f7f6653
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/licenses/xmlsec-LICENSE.txt b/x-pack/plugin/identity-provider/licenses/xmlsec-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/xmlsec-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/identity-provider/licenses/xmlsec-NOTICE.txt b/x-pack/plugin/identity-provider/licenses/xmlsec-NOTICE.txt
new file mode 100644
index 0000000000000..ab7c6276cd836
--- /dev/null
+++ b/x-pack/plugin/identity-provider/licenses/xmlsec-NOTICE.txt
@@ -0,0 +1,9 @@
+
+Apache XML Security for Java
+Copyright 2000-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+
+
diff --git a/x-pack/plugin/identity-provider/qa/build.gradle b/x-pack/plugin/identity-provider/qa/build.gradle
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle b/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle
new file mode 100644
index 0000000000000..85884ef2d74bd
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle
@@ -0,0 +1,33 @@
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: xpackModule('core'), configuration: 'default')
+  testCompile project(path: xpackModule('identity-provider'), configuration: 'default')
+  testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
+
+  setting 'xpack.license.self_generated.type', 'trial'
+
+  setting 'xpack.idp.enabled', 'true'
+  setting 'xpack.idp.entity_id', 'https://idp.test.es.elasticsearch.org/'
+  setting 'xpack.idp.sso_endpoint.redirect', 'http://idp.test.es.elasticsearch.org/test/saml/redirect'
+  setting 'xpack.idp.signing.certificate', 'idp-sign.crt'
+  setting 'xpack.idp.signing.key', 'idp-sign.key'
+  setting 'xpack.idp.privileges.application', 'elastic-cloud'
+
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.authc.token.enabled', 'true'
+  setting 'xpack.security.authc.api_key.enabled', 'true'
+
+  extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
+  extraConfigFile 'idp-sign.crt', file('src/test/resources/idp-sign.crt')
+  extraConfigFile 'idp-sign.key', file('src/test/resources/idp-sign.key')
+
+  user username: "admin_user", password: "admin-password"
+  user username: "idp_user", password: "idp-password", role: "idp_role"
+}
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java
new file mode 100644
index 0000000000000..e833d644bf3a5
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+public abstract class IdpRestTestCase extends ESRestTestCase {
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("idp_user", new SecureString("idp-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+}
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/ManageServiceProviderRestIT.java b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/ManageServiceProviderRestIT.java
new file mode 100644
index 0000000000000..4cae5a29bf5d0
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/ManageServiceProviderRestIT.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp;
+
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex.DocumentVersion;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class ManageServiceProviderRestIT extends IdpRestTestCase {
+
+    // From build.gradle
+    private final String IDP_ENTITY_ID = "https://idp.test.es.elasticsearch.org/";
+    // From SAMLConstants
+    private final String REDIRECT_BINDING = "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect";
+
+    public void testCreateAndDeleteServiceProvider() throws Exception {
+        final String entityId = "ec:" + randomAlphaOfLength(8) + ":" + randomAlphaOfLength(12);
+        final Map<String, Object> request = Map.ofEntries(
+            Map.entry("name", "Test SP"),
+            Map.entry("acs", "https://sp1.test.es.elasticsearch.org/saml/acs"),
+            Map.entry("privileges", Map.ofEntries(
+                Map.entry("resource", entityId),
+                Map.entry("roles", Map.of("superuser", "role:superuser", "viewer", "role:viewer"))
+            )),
+            Map.entry("attributes", Map.ofEntries(
+                Map.entry("principal", "https://idp.test.es.elasticsearch.org/attribute/principal"),
+                Map.entry("name", "https://idp.test.es.elasticsearch.org/attribute/name"),
+                Map.entry("email", "https://idp.test.es.elasticsearch.org/attribute/email"),
+                Map.entry("roles", "https://idp.test.es.elasticsearch.org/attribute/roles")
+            )));
+        final DocumentVersion docVersion = createServiceProvider(entityId, request);
+        checkIndexDoc(docVersion);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        getMetaData(entityId);
+        deleteServiceProvider(entityId, docVersion);
+        expectThrows(ResponseException.class, () -> getMetaData(entityId));
+        expectThrows(ResponseException.class, () -> deleteServiceProvider(entityId, docVersion));
+    }
+
+    private DocumentVersion createServiceProvider(String entityId, Map<String, Object> body) throws IOException {
+        final Request request = new Request("PUT", "/_idp/saml/sp/" + encode(entityId) + "?refresh=" + RefreshPolicy.IMMEDIATE.getValue());
+        final String entity = Strings.toString(JsonXContent.contentBuilder().map(body));
+        request.setJsonEntity(entity);
+        final Response response = client().performRequest(request);
+        final Map<String, Object> map = entityAsMap(response);
+        assertThat(ObjectPath.eval("service_provider.entity_id", map), equalTo(entityId));
+        assertThat(ObjectPath.eval("service_provider.enabled", map), equalTo(true));
+
+        final Object docId = ObjectPath.eval("document._id", map);
+        final Object seqNo = ObjectPath.eval("document._seq_no", map);
+        final Object primaryTerm = ObjectPath.eval("document._primary_term", map);
+        assertThat(docId, instanceOf(String.class));
+        assertThat(seqNo, instanceOf(Number.class));
+        assertThat(primaryTerm, instanceOf(Number.class));
+        return new DocumentVersion((String) docId, asLong(primaryTerm), asLong(seqNo));
+    }
+
+    private void checkIndexDoc(DocumentVersion docVersion) throws IOException {
+        final Request request = new Request("GET", SamlServiceProviderIndex.INDEX_NAME + "/_doc/" + docVersion.id);
+        final Response response = adminClient().performRequest(request);
+        final Map<String, Object>
+            map = entityAsMap(response);
+        assertThat(map.get("_index"), equalTo(SamlServiceProviderIndex.INDEX_NAME));
+        assertThat(map.get("_id"), equalTo(docVersion.id));
+        assertThat(asLong(map.get("_seq_no")), equalTo(docVersion.seqNo));
+        assertThat(asLong(map.get("_primary_term")), equalTo(docVersion.primaryTerm));
+    }
+
+    private void deleteServiceProvider(String entityId, DocumentVersion version) throws IOException {
+        final Response response = client().performRequest(new Request("DELETE",
+            "/_idp/saml/sp/" + encode(entityId) + "?refresh=" + RefreshPolicy.IMMEDIATE.getValue()));
+        final Map<String, Object> map = entityAsMap(response);
+
+        assertThat(ObjectPath.eval("document._id", map), equalTo(version.id));
+
+        Long seqNo = asLong(ObjectPath.eval("document._seq_no", map));
+        Long primaryTerm = asLong(ObjectPath.eval("document._primary_term", map));
+        if (primaryTerm == version.primaryTerm) {
+            assertThat(seqNo, greaterThanOrEqualTo(version.seqNo));
+        } else {
+            assertThat(primaryTerm, greaterThanOrEqualTo(version.primaryTerm));
+        }
+
+        assertThat(ObjectPath.eval("service_provider.entity_id", map), equalTo(entityId));
+    }
+
+    private void getMetaData(String entityId) throws IOException {
+        final Map<String, Object> map = getAsMap("/_idp/saml/metadata/" + encode(entityId));
+        assertThat(map, notNullValue());
+        assertThat(map.keySet(), containsInAnyOrder("metadata"));
+        final Object metadata = map.get("metadata");
+        assertThat(metadata, notNullValue());
+        assertThat(metadata, instanceOf(String.class));
+        assertThat((String) metadata, containsString(IDP_ENTITY_ID));
+        assertThat((String) metadata, containsString(REDIRECT_BINDING));
+    }
+
+    private String encode(String param) {
+        return URLEncoder.encode(param, StandardCharsets.UTF_8);
+    }
+
+    private Long asLong(Object val) {
+        if (val == null) {
+            return null;
+        }
+        if (val instanceof Long) {
+            return (Long) val;
+        }
+        if (val instanceof Number) {
+            return ((Number) val).longValue();
+        }
+        if (val instanceof String) {
+            return Long.parseLong((String) val);
+        }
+        throw new IllegalArgumentException("Value [" + val + "] of type [" + val.getClass() + "] is not a Long");
+    }
+}
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.crt b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.crt
new file mode 100644
index 0000000000000..151aec5b336bb
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.crt
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDoDCCAoigAwIBAgIVAPVpXjny5MCPPsCCGvInDfZQu+4JMA0GCSqGSIb3DQEB
+CwUAMF8xEzARBgoJkiaJk/IsZAEZFgNvcmcxHTAbBgoJkiaJk/IsZAEZFg1lbGFz
+dGljc2VhcmNoMQswCQYDVQQLEwJlczENMAsGA1UECxMEdGVzdDENMAsGA1UEAxME
+c2lnbjAeFw0yMDAzMTMwNDAzNDFaFw00NzA3MjkwNDAzNDFaMF8xEzARBgoJkiaJ
+k/IsZAEZFgNvcmcxHTAbBgoJkiaJk/IsZAEZFg1lbGFzdGljc2VhcmNoMQswCQYD
+VQQLEwJlczENMAsGA1UECxMEdGVzdDENMAsGA1UEAxMEc2lnbjCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBAI/JK5IlwBixo2mqX8wDnH2F+tXDJKU8uNez
+oulYFVMp17iK41sOIwCInd6DGDJPkgIIkobnmp3gDgnLckzApC8Ck9xmnjmEQdg+
+gZO1gtgBVU6Lj8haCzPUMVszGodlraeaRMTIBrNG6GPMo/PN5hj2vyC383jiDlE9
+K3I+Y0rxhnPj6Ic3aTO/bXiMG4FiIvag/ViITnvkL1mREwF9JER40dgQoaJJJFTf
+jcWYnd1q9FDlIhylj2KJwXhOkSZ5znGM93b3GZ5VZUm8zzMlq/RzyywDet2jyOSr
+Bn9X3lrp0fPjfhTW+O/+2G/k6bVP29BYJ5Im+JVjjaYbYFI5pQ0CAwEAAaNTMFEw
+HQYDVR0OBBYEFEghxQTreebAJEak+zywt5B/LpaLMB8GA1UdIwQYMBaAFEghxQTr
+eebAJEak+zywt5B/LpaLMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQAD
+ggEBACtGNnaUK4UtKPOSg7fOW3/tIJoPhriuSlAdqQxNRYjkdmPJsY7Gra/kikCm
+0dRUbttMTfOqwZwu5KdKJUo3duXKpRVK3emFDixM8FxFbhtA19+Newzcp6mS0K0T
+4wqAI5jH/a+eHJb6hDWuCrm9RICxXwqpUdn3HfJrYQzsOCuecTAy60jfxjjZeZxA
+MRIARkhaHQeOdX3rg5v9kWzFXRD0Y7nlqqqCa0el2gs9yZqjDYRMFzz8z4qN9q2N
+zjohhxgIwuWK+0R6qOIg3XK53kdh6YjgWqlGn1aX9z7ztqlDmiD7LIsPgWL2BARh
+WOexDVOg4wTpACOt4c84VVAVt98=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.key b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.key
new file mode 100644
index 0000000000000..792de4f855b6f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-sign.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAj8krkiXAGLGjaapfzAOcfYX61cMkpTy417Oi6VgVUynXuIrj
+Ww4jAIid3oMYMk+SAgiShueaneAOCctyTMCkLwKT3GaeOYRB2D6Bk7WC2AFVTouP
+yFoLM9QxWzMah2Wtp5pExMgGs0boY8yj883mGPa/ILfzeOIOUT0rcj5jSvGGc+Po
+hzdpM79teIwbgWIi9qD9WIhOe+QvWZETAX0kRHjR2BChokkkVN+NxZid3Wr0UOUi
+HKWPYonBeE6RJnnOcYz3dvcZnlVlSbzPMyWr9HPLLAN63aPI5KsGf1feWunR8+N+
+FNb47/7Yb+TptU/b0Fgnkib4lWONphtgUjmlDQIDAQABAoIBAFxba5FEjk9OSxjk
+RwRUjYBTPvtvjWpVxqVGza301j3E8rum+sLSIcmF92Gl8zTdx3tm5jehLL+b22iP
+Nav3UX6guTko8RkgyuqLu8+VF6aHtLPvETwwGmIAOnPKL5Aoz5KIlQORODyPn1X7
+bJR9/P6r/7iUb9+F1Ix6Lh4D1OOdqDKuEyIJBOjMuxtECq15tth6Ufmvp5mupT4x
+J5z9RqX36hoOWf4NvAv2YochDSvjYHZMVXsYgu2445ws36I3MGnMuIixDdvVbbtx
+NkUup+tNOZA2669OKwH+D71MGVKKiPq1W5R6J0LuBKdai1LgNLfFcNB27mcOXwaL
+BdRd6EECgYEA1lZYvobU27TdQkA/s+PaUAKw/3/tF/9sVe5V2vA4n15Fgd8KY3BO
+LzPSehBL1EOnyLccKi1K7PHOC/3NrM5Hxon9R5SIY5L8hfK1hVVONdUFgAsElzlf
+OOhsmRXs3Qq1qVfzW7ViipOUTGOYbTo7wSFZr9+rdh2WolhestmCFgMCgYEAq7wb
+VFOyyL267r9CvFsMB8Yy3xMTFJGLadVZJw0N0Em/nt9JEHxFhrr3lSJaYjAW+BHG
+82n3TUODdejwCSFGpI3E/nibhvgopmnQZm8uE8mtULDN388eKbh3iGR7He2dLJEM
+5bG8p5eLGM99god2hu54VYIsgf5jT2WcsAFOM68CgYAjPHK+b8ASlntBZvqSAkVj
+ne2nM0qxBUa0Ichvg8prOL5IiXhVvKK909EoTAGLVcwBjUjODkAhD+eFxSXI4Oif
+1ROUbvC1HfbxtmLtFocTBoAu+qC4k6/51Qv1ZstX02jl/BV/4CPhED3zCPSIEGi+
+aVMrPKQdeOPIsKpw0J04LQKBgCbRqGPRX5JcwMqC1TT6Z6fCN3GRQDjgBWFw5mwb
+WUoBwZzJ4Bwn2xdvX9OIJmIXeLmuWwhepZYDcs3OT6Pgr7U2jpbu8Ej8A0RKmt7s
+tr+mUNTygjba/Hh1yB8+h03mjiaqyv2IxZokeT6seDRvJm2trem2ORVRSWWDFH38
+bY7lAoGAGTsziUtoWbuly66fBjT7XTYDiKgYqn7xz4vyH+KoQ1meqpRqvuC8jkEn
+Ai2ne441btCrsZsH44msc/ss543XHv+Ybqp4aNK0VU72lsYcSMGYtsWCpDAGkZU0
+4zjkGDa+AzdAf2XDAsL+SiaMuDJ8wKovfLpgCkoryQ0Iwdfjkh0=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/roles.yml b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/roles.yml
new file mode 100644
index 0000000000000..d5e00352b3e73
--- /dev/null
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/roles.yml
@@ -0,0 +1,6 @@
+# A basic role that is used to call the IdP's APIs
+idp_role:
+  cluster:
+    - monitor
+    - "cluster:admin/idp/*"
+
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java
new file mode 100644
index 0000000000000..b11a1e806c630
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.ssl.X509KeyPairSettings;
+import org.elasticsearch.xpack.idp.action.DeleteSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.PutSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.SamlInitiateSingleSignOnAction;
+import org.elasticsearch.xpack.idp.action.SamlMetadataAction;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestAction;
+import org.elasticsearch.xpack.idp.action.TransportDeleteSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.TransportPutSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.TransportSamlInitiateSingleSignOnAction;
+import org.elasticsearch.xpack.idp.action.TransportSamlMetadataAction;
+import org.elasticsearch.xpack.idp.action.TransportSamlValidateAuthnRequestAction;
+import org.elasticsearch.xpack.idp.privileges.UserPrivilegeResolver;
+import org.elasticsearch.xpack.idp.saml.rest.action.RestDeleteSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder;
+import org.elasticsearch.xpack.idp.saml.rest.action.RestSamlMetadataAction;
+import org.elasticsearch.xpack.idp.saml.rest.action.RestSamlInitiateSingleSignOnAction;
+import org.elasticsearch.xpack.idp.saml.rest.action.RestSamlValidateAuthenticationRequestAction;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.rest.action.RestPutSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+/**
+ * This plugin provides the backend for an IdP built on top of Elasticsearch security features.
+ * It is used internally within Elastic and is not intended for general use.
+ */
+public class IdentityProviderPlugin extends Plugin implements ActionPlugin {
+
+    private static final Setting<Boolean> ENABLED_SETTING = Setting.boolSetting("xpack.idp.enabled", false, Setting.Property.NodeScope);
+
+    private final Logger logger = LogManager.getLogger();
+    private boolean enabled;
+    private Settings settings;
+
+    @Override
+    public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
+                                               ResourceWatcherService resourceWatcherService, ScriptService scriptService,
+                                               NamedXContentRegistry xContentRegistry, Environment environment,
+                                               NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
+                                               IndexNameExpressionResolver indexNameExpressionResolver) {
+        settings = environment.settings();
+        enabled = ENABLED_SETTING.get(settings);
+        if (enabled == false) {
+            return List.of();
+        }
+
+        SamlInit.initialize();
+        final SamlServiceProviderIndex index = new SamlServiceProviderIndex(client, clusterService);
+        final SecurityContext securityContext = new SecurityContext(settings, threadPool.getThreadContext());
+        final UserPrivilegeResolver userPrivilegeResolver = new UserPrivilegeResolver(client, securityContext);
+
+
+        final ServiceProviderDefaults serviceProviderDefaults = ServiceProviderDefaults.forSettings(settings);
+        final SamlServiceProviderResolver resolver = new SamlServiceProviderResolver(settings, index, serviceProviderDefaults);
+        final SamlIdentityProvider idp = SamlIdentityProvider.builder(resolver)
+            .fromSettings(environment)
+            .serviceProviderDefaults(serviceProviderDefaults)
+            .build();
+
+        final SamlFactory factory = new SamlFactory();
+
+        return List.of(
+            index,
+            idp,
+            factory,
+            userPrivilegeResolver
+        );
+    }
+
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        if (enabled == false) {
+            return List.of();
+        }
+        return List.of(
+            new ActionHandler<>(SamlInitiateSingleSignOnAction.INSTANCE, TransportSamlInitiateSingleSignOnAction.class),
+            new ActionHandler<>(SamlValidateAuthnRequestAction.INSTANCE, TransportSamlValidateAuthnRequestAction.class),
+            new ActionHandler<>(SamlMetadataAction.INSTANCE, TransportSamlMetadataAction.class),
+            new ActionHandler<>(PutSamlServiceProviderAction.INSTANCE, TransportPutSamlServiceProviderAction.class),
+            new ActionHandler<>(DeleteSamlServiceProviderAction.INSTANCE, TransportDeleteSamlServiceProviderAction.class)
+        );
+    }
+
+    @Override
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                                             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                                             IndexNameExpressionResolver indexNameExpressionResolver,
+                                             Supplier<DiscoveryNodes> nodesInCluster) {
+        if (enabled == false) {
+            return List.of();
+        }
+        return List.of(
+            new RestSamlInitiateSingleSignOnAction(getLicenseState()),
+            new RestSamlValidateAuthenticationRequestAction(getLicenseState()),
+            new RestSamlMetadataAction(getLicenseState()),
+            new RestPutSamlServiceProviderAction(getLicenseState()),
+            new RestDeleteSamlServiceProviderAction(getLicenseState())
+        );
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        List<Setting<?>> settings = new ArrayList<>();
+        settings.add(ENABLED_SETTING);
+        settings.addAll(SamlIdentityProviderBuilder.getSettings());
+        settings.addAll(ServiceProviderDefaults.getSettings());
+        settings.addAll(X509KeyPairSettings.withPrefix("xpack.idp.signing.", false).getAllSettings());
+        settings.addAll(X509KeyPairSettings.withPrefix("xpack.idp.metadata_signing.", false).getAllSettings());
+        return Collections.unmodifiableList(settings);
+    }
+
+    protected XPackLicenseState getLicenseState() {
+        return XPackPlugin.getSharedLicenseState();
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..99ac5a6773534
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderAction.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionType;
+
+/**
+ * Action to remove a service provider from the IdP.
+ */
+public class DeleteSamlServiceProviderAction extends ActionType<DeleteSamlServiceProviderResponse> {
+
+    public static final String NAME = "cluster:admin/idp/saml/sp/delete";
+    public static final DeleteSamlServiceProviderAction INSTANCE = new DeleteSamlServiceProviderAction(NAME);
+
+    public DeleteSamlServiceProviderAction(String name) {
+        super(name, DeleteSamlServiceProviderResponse::new);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequest.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequest.java
new file mode 100644
index 0000000000000..d76b678f5652e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request object to remove a service provider (by Entity ID) from the IdP.
+ */
+public class DeleteSamlServiceProviderRequest extends ActionRequest {
+
+    private final String entityId;
+    private final WriteRequest.RefreshPolicy refreshPolicy;
+
+    public DeleteSamlServiceProviderRequest(String entityId, WriteRequest.RefreshPolicy refreshPolicy) {
+        this.entityId = entityId;
+        this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "Refresh policy may not be null");
+    }
+
+    public DeleteSamlServiceProviderRequest(StreamInput in) throws IOException {
+        this.entityId = in.readString();
+        this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(entityId);
+        refreshPolicy.writeTo(out);
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    public WriteRequest.RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(entityId)) {
+            validationException = addValidationError("The Service Provider Entity ID is required", validationException);
+        }
+        return validationException;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final DeleteSamlServiceProviderRequest that = (DeleteSamlServiceProviderRequest) o;
+        return Objects.equals(entityId, that.entityId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(entityId, refreshPolicy);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{" + entityId + "," + refreshPolicy + "}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponse.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponse.java
new file mode 100644
index 0000000000000..957b464d2a145
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponse.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
+
+/**
+ * Response object after removing a service provider from the IdP.
+ */
+public class DeleteSamlServiceProviderResponse extends ActionResponse implements ToXContentObject {
+
+    private final String docId;
+    private final long seqNo;
+    private final long primaryTerm;
+    private final String entityId;
+
+    public DeleteSamlServiceProviderResponse(String docId, long seqNo, long primaryTerm, String entityId) {
+        this.docId = docId;
+        this.seqNo = seqNo;
+        this.primaryTerm = primaryTerm;
+        this.entityId = Objects.requireNonNull(entityId, "Entity Id cannot be null");
+    }
+
+    public DeleteSamlServiceProviderResponse(DeleteResponse deleteResponse, String entityId) {
+        this(deleteResponse == null ? null : deleteResponse.getId(),
+            deleteResponse == null ? UNASSIGNED_SEQ_NO : deleteResponse.getSeqNo(),
+            deleteResponse == null ? UNASSIGNED_PRIMARY_TERM : deleteResponse.getPrimaryTerm(),
+            entityId);
+    }
+
+    public DeleteSamlServiceProviderResponse(StreamInput in) throws IOException {
+        docId = in.readString();
+        seqNo = in.readZLong();
+        primaryTerm = in.readVLong();
+        entityId = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(docId);
+        out.writeZLong(seqNo);
+        out.writeVLong(primaryTerm);
+        out.writeString(entityId);
+    }
+
+    @Nullable
+    public String getDocId() {
+        return docId;
+    }
+
+    public long getSeqNo() {
+        return seqNo;
+    }
+
+    public long getPrimaryTerm() {
+        return primaryTerm;
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.startObject("document");
+        builder.field("_id", docId);
+        builder.field("_seq_no", seqNo);
+        builder.field("_primary_term", primaryTerm);
+        builder.endObject();
+
+        builder.startObject("service_provider");
+        builder.field("entity_id", entityId);
+        builder.endObject();
+
+        return builder.endObject();
+    }
+
+    public boolean found() {
+        return docId != null;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..6ffc8c14c1d1a
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderAction.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionType;
+
+public class PutSamlServiceProviderAction extends ActionType<PutSamlServiceProviderResponse> {
+
+    public static final String NAME = "cluster:admin/idp/saml/sp/put";
+    public static final PutSamlServiceProviderAction INSTANCE = new PutSamlServiceProviderAction(NAME);
+
+    public PutSamlServiceProviderAction(String name) {
+        super(name, PutSamlServiceProviderResponse::new);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequest.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequest.java
new file mode 100644
index 0000000000000..ae7dd1485b8fe
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequest.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+public class PutSamlServiceProviderRequest extends ActionRequest {
+    public static final WriteRequest.RefreshPolicy DEFAULT_REFRESH_POLICY = WriteRequest.RefreshPolicy.NONE;
+
+    private final SamlServiceProviderDocument document;
+    private final WriteRequest.RefreshPolicy refreshPolicy;
+
+    public static PutSamlServiceProviderRequest fromXContent(String entityId, WriteRequest.RefreshPolicy refreshPolicy,
+                                                             XContentParser parser) throws IOException {
+        final SamlServiceProviderDocument document = SamlServiceProviderDocument.fromXContent(null, parser);
+        if (document.entityId == null) {
+            document.setEntityId(entityId);
+        } else if (entityId != null) {
+            if (entityId.equals(document.entityId) == false) {
+                throw new ElasticsearchParseException(
+                    "Entity id [{}] inside request body and entity id [{}] from parameter do not match", document.entityId, entityId);
+            }
+        }
+        if (document.created != null) {
+            throw new ElasticsearchParseException(
+                "Field [{}] may not be specified in a request", SamlServiceProviderDocument.Fields.CREATED_DATE);
+        }
+        if (document.lastModified != null) {
+            throw new ElasticsearchParseException(
+                "Field [{}] may not be specified in a request", SamlServiceProviderDocument.Fields.LAST_MODIFIED);
+        }
+        document.setCreatedMillis(System.currentTimeMillis());
+        document.setLastModifiedMillis(System.currentTimeMillis());
+        return new PutSamlServiceProviderRequest(document, refreshPolicy);
+    }
+
+    public PutSamlServiceProviderRequest(SamlServiceProviderDocument document, WriteRequest.RefreshPolicy refreshPolicy) {
+        this.document = document;
+        this.refreshPolicy = refreshPolicy;
+    }
+
+    public PutSamlServiceProviderRequest(StreamInput in) throws IOException {
+        this.document = new SamlServiceProviderDocument(in);
+        this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in);
+    }
+
+    public WriteRequest.RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        document.writeTo(out);
+        refreshPolicy.writeTo(out);
+    }
+
+    public SamlServiceProviderDocument getDocument() {
+        return document;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        final ValidationException docException = document.validate();
+        ActionRequestValidationException validationException = null;
+        if (docException != null) {
+            validationException = new ActionRequestValidationException();
+            validationException.addValidationErrors(docException.validationErrors());
+        }
+
+        if (Strings.hasText(document.acs)) { // if this is blank the document validation will fail
+            try {
+                final URL url = new URL(document.acs);
+                if (url.getProtocol().equals("https") == false) {
+                    validationException = addValidationError(
+                        "[" + SamlServiceProviderDocument.Fields.ACS + "] must use the [https] protocol", validationException);
+                }
+            } catch (MalformedURLException e) {
+                String error = "[" + SamlServiceProviderDocument.Fields.ACS + "] must be a valid URL";
+                if (e.getMessage() != null) {
+                    error += " - " + e.getMessage();
+                }
+                validationException = addValidationError(error, validationException);
+            }
+        }
+
+        if (document.certificates.identityProviderSigning.isEmpty() == false) {
+            validationException = addValidationError(
+                "[" + SamlServiceProviderDocument.Fields.Certificates.IDP_SIGNING + "] certificates may not be specified",
+                validationException);
+        }
+
+        if (document.certificates.identityProviderMetadataSigning.isEmpty() == false) {
+            validationException = addValidationError(
+                "[" + SamlServiceProviderDocument.Fields.Certificates.IDP_METADATA + "] certificates may not be specified",
+                validationException);
+        }
+
+        return validationException;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final PutSamlServiceProviderRequest that = (PutSamlServiceProviderRequest) o;
+        return Objects.equals(document, that.document);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(document);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{" + document + "}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderResponse.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderResponse.java
new file mode 100644
index 0000000000000..f6a89a9e4804f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderResponse.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class PutSamlServiceProviderResponse extends ActionResponse implements ToXContentObject {
+
+    private final String docId;
+    private final boolean created;
+    private final long seqNo;
+    private final long primaryTerm;
+    private final String entityId;
+    private final boolean enabled;
+
+    public PutSamlServiceProviderResponse(String docId, boolean created, long seqNo, long primaryTerm, String entityId, boolean enabled) {
+        this.docId = Objects.requireNonNull(docId, "Document Id cannot be null");
+        this.created = created;
+        this.seqNo = seqNo;
+        this.primaryTerm = primaryTerm;
+        this.entityId = Objects.requireNonNull(entityId, "Entity Id cannot be null");
+        this.enabled = enabled;
+    }
+
+    public PutSamlServiceProviderResponse(StreamInput in) throws IOException {
+        docId = in.readString();
+        created = in.readBoolean();
+        seqNo = in.readZLong();
+        primaryTerm = in.readVLong();
+        entityId = in.readString();
+        enabled = in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(docId);
+        out.writeBoolean(created);
+        out.writeZLong(seqNo);
+        out.writeVLong(primaryTerm);
+        out.writeString(entityId);
+        out.writeBoolean(enabled);
+    }
+
+    public String getDocId() {
+        return docId;
+    }
+
+    public boolean isCreated() {
+        return created;
+    }
+
+    public long getSeqNo() {
+        return seqNo;
+    }
+
+    public long getPrimaryTerm() {
+        return primaryTerm;
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.startObject("document");
+        builder.field("_id", docId);
+        builder.field("_created", created);
+        builder.field("_seq_no", seqNo);
+        builder.field("_primary_term", primaryTerm);
+        builder.endObject();
+
+        builder.startObject("service_provider");
+        builder.field("entity_id", entityId);
+        builder.field("enabled", enabled);
+        builder.endObject();
+
+        return builder.endObject();
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnAction.java
new file mode 100644
index 0000000000000..92c492842ee34
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnAction.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionType;
+
+/**
+ * ActionType to create a SAML Response in the context of IDP initiated SSO for a given SP
+ */
+public class SamlInitiateSingleSignOnAction extends ActionType<SamlInitiateSingleSignOnResponse> {
+
+    public static final String NAME = "cluster:admin/idp/saml/init";
+    public static final SamlInitiateSingleSignOnAction INSTANCE = new SamlInitiateSingleSignOnAction();
+
+    private SamlInitiateSingleSignOnAction() {
+        super(NAME, SamlInitiateSingleSignOnResponse::new);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequest.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequest.java
new file mode 100644
index 0000000000000..f29f9cf066f38
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequest.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.idp.saml.support.SamlAuthenticationState;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+import java.io.IOException;
+
+public class SamlInitiateSingleSignOnRequest extends ActionRequest {
+
+    private String spEntityId;
+    private SamlAuthenticationState samlAuthenticationState;
+
+    public SamlInitiateSingleSignOnRequest(StreamInput in) throws IOException {
+        super(in);
+        spEntityId = in.readString();
+        samlAuthenticationState = in.readOptionalWriteable(SamlAuthenticationState::new);
+    }
+
+    public SamlInitiateSingleSignOnRequest() {
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(spEntityId)) {
+            validationException = addValidationError("entity_id is missing", validationException);
+        }
+        if (samlAuthenticationState != null) {
+            final ValidationException authnStateException = samlAuthenticationState.validate();
+            if (validationException != null) {
+                ActionRequestValidationException actionRequestValidationException = new ActionRequestValidationException();
+                actionRequestValidationException.addValidationErrors(authnStateException.validationErrors());
+                validationException = addValidationError("entity_id is missing", actionRequestValidationException);
+            }
+        }
+        return validationException;
+    }
+
+    public String getSpEntityId() {
+        return spEntityId;
+    }
+
+    public void setSpEntityId(String spEntityId) {
+        this.spEntityId = spEntityId;
+    }
+
+    public SamlAuthenticationState getSamlAuthenticationState() {
+        return samlAuthenticationState;
+    }
+
+    public void setSamlAuthenticationState(SamlAuthenticationState samlAuthenticationState) {
+        this.samlAuthenticationState = samlAuthenticationState;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(spEntityId);
+        out.writeOptionalWriteable(samlAuthenticationState);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{spEntityId='" + spEntityId + "'}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnResponse.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnResponse.java
new file mode 100644
index 0000000000000..7d978152d0377
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnResponse.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+public class SamlInitiateSingleSignOnResponse extends ActionResponse {
+
+    private String postUrl;
+    private String samlResponse;
+    private String entityId;
+
+    public SamlInitiateSingleSignOnResponse(StreamInput in) throws IOException {
+        super(in);
+        this.postUrl = in.readString();
+        this.samlResponse = in.readString();
+        this.entityId = in.readString();
+    }
+
+    public SamlInitiateSingleSignOnResponse(String postUrl, String samlResponse, String entityId) {
+        this.postUrl = postUrl;
+        this.samlResponse = samlResponse;
+        this.entityId = entityId;
+    }
+
+    public String getPostUrl() {
+        return postUrl;
+    }
+
+    public String getSamlResponse() {
+        return samlResponse;
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(postUrl);
+        out.writeString(samlResponse);
+        out.writeString(entityId);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataAction.java
new file mode 100644
index 0000000000000..4c816ef122a97
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataAction.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionType;
+
+public class SamlMetadataAction extends ActionType<SamlMetadataResponse> {
+
+    public static final String NAME = "cluster:admin/idp/saml/metadata";
+    public static final SamlMetadataAction INSTANCE = new SamlMetadataAction();
+
+    private SamlMetadataAction() {
+        super(NAME, SamlMetadataResponse::new);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataRequest.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataRequest.java
new file mode 100644
index 0000000000000..86ba297629a3f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataRequest.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class SamlMetadataRequest extends ActionRequest {
+
+    private String spEntityId;
+
+    public SamlMetadataRequest(StreamInput in) throws IOException {
+        super(in);
+        spEntityId = in.readString();
+    }
+
+    public SamlMetadataRequest(String spEntityId) {
+        this.spEntityId = Objects.requireNonNull(spEntityId, "Service Provider entity id must be provided");
+    }
+
+    public SamlMetadataRequest() {
+        this.spEntityId = null;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    public String getSpEntityId() {
+        return spEntityId;
+    }
+
+    public void setSpEntityId(String spEntityId) {
+        this.spEntityId = spEntityId;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{spEntityId='" + spEntityId + "'}";
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataResponse.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataResponse.java
new file mode 100644
index 0000000000000..0839dc70b4803
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlMetadataResponse.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class SamlMetadataResponse extends ActionResponse {
+
+    private final String xmlString;
+
+    public SamlMetadataResponse(StreamInput in) throws IOException {
+        super(in);
+        this.xmlString = in.readString();
+    }
+
+    public SamlMetadataResponse(String xmlString) {
+        this.xmlString = Objects.requireNonNull(xmlString, "Metadata XML string must be provided");
+    }
+
+    public String getXmlString() {
+        return xmlString;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(xmlString);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestAction.java
new file mode 100644
index 0000000000000..bd246d0f9ff7f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestAction.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionType;
+
+public class SamlValidateAuthnRequestAction extends ActionType<SamlValidateAuthnRequestResponse> {
+
+    public static final String NAME = "cluster:admin/idp/saml/validate";
+    public static final SamlValidateAuthnRequestAction INSTANCE = new SamlValidateAuthnRequestAction();
+
+    private SamlValidateAuthnRequestAction() {
+        super(NAME, SamlValidateAuthnRequestResponse::new);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequest.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequest.java
new file mode 100644
index 0000000000000..1fffcf23871d0
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequest.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+public class SamlValidateAuthnRequestRequest extends ActionRequest {
+
+    private String queryString;
+
+    public SamlValidateAuthnRequestRequest(StreamInput in) throws IOException {
+        super(in);
+        queryString = in.readString();
+    }
+
+    public SamlValidateAuthnRequestRequest() {
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(queryString)) {
+            validationException = addValidationError("Authentication request query string must be provided", validationException);
+        }
+        return validationException;
+    }
+
+    public String getQueryString() {
+        return queryString;
+    }
+
+    public void setQueryString(String queryString) {
+        this.queryString = queryString;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(queryString);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{queryString='" + queryString + "'}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestResponse.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestResponse.java
new file mode 100644
index 0000000000000..cf06efc53084f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestResponse.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class SamlValidateAuthnRequestResponse extends ActionResponse {
+
+    private final String spEntityId;
+    private final boolean forceAuthn;
+    private final Map<String, Object> authnState;
+
+    public SamlValidateAuthnRequestResponse(StreamInput in) throws IOException {
+        super(in);
+        this.spEntityId = in.readString();
+        this.forceAuthn = in.readBoolean();
+        this.authnState = in.readMap();
+    }
+
+    public SamlValidateAuthnRequestResponse(String spEntityId, boolean forceAuthn, Map<String, Object> authnState) {
+        this.spEntityId = Objects.requireNonNull(spEntityId, "spEntityId is required for successful responses");
+        this.forceAuthn = forceAuthn;
+        this.authnState = Map.copyOf(Objects.requireNonNull(authnState));
+    }
+
+    public String getSpEntityId() {
+        return spEntityId;
+    }
+
+    public boolean isForceAuthn() {
+        return forceAuthn;
+    }
+
+    public Map<String, Object> getAuthnState() {
+        return authnState;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(spEntityId);
+        out.writeBoolean(forceAuthn);
+        out.writeMap(authnState);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{ spEntityId='" + getSpEntityId() + "',\n" +
+            " forceAuthn='" + isForceAuthn() + "',\n" +
+            " authnState='" + getAuthnState() + "' }";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..f32505046c1ad
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+
+import java.util.stream.Collectors;
+
+/**
+ * Transport action to remove a service provider from the IdP
+ */
+public class TransportDeleteSamlServiceProviderAction
+    extends HandledTransportAction<DeleteSamlServiceProviderRequest, DeleteSamlServiceProviderResponse> {
+
+    private final Logger logger = LogManager.getLogger();
+    private final SamlServiceProviderIndex index;
+
+    @Inject
+    public TransportDeleteSamlServiceProviderAction(TransportService transportService, ActionFilters actionFilters,
+                                                    SamlServiceProviderIndex index) {
+        super(DeleteSamlServiceProviderAction.NAME, transportService, actionFilters, DeleteSamlServiceProviderRequest::new);
+        this.index = index;
+    }
+
+    @Override
+    protected void doExecute(Task task, final DeleteSamlServiceProviderRequest request,
+                             final ActionListener<DeleteSamlServiceProviderResponse> listener) {
+        final String entityId = request.getEntityId();
+        index.findByEntityId(entityId, ActionListener.wrap(matchingDocuments -> {
+            if (matchingDocuments.isEmpty()) {
+                listener.onResponse(new DeleteSamlServiceProviderResponse(null, entityId));
+            } else if (matchingDocuments.size() == 1) {
+                final SamlServiceProviderIndex.DocumentSupplier docInfo = Iterables.get(matchingDocuments, 0);
+                final SamlServiceProviderDocument existingDoc = docInfo.getDocument();
+                assert existingDoc.docId != null : "Loaded document with no doc id";
+                assert existingDoc.entityId.equals(entityId) : "Loaded document with non-matching entity-id";
+                logger.info("Deleting Service Provider [{}]", existingDoc);
+                index.deleteDocument(docInfo.version, request.getRefreshPolicy(), ActionListener.wrap(
+                    deleteResponse -> listener.onResponse(new DeleteSamlServiceProviderResponse(deleteResponse, entityId)),
+                    listener::onFailure
+                ));
+            } else {
+                logger.warn("Found multiple existing service providers in [{}] with entity id [{}] - [{}]",
+                    index, entityId, matchingDocuments.stream().map(d -> d.getDocument().docId).collect(Collectors.joining(",")));
+                listener.onFailure(new IllegalStateException("Multiple service providers exist with entity id [" + entityId + "]"));
+            }
+        }, listener::onFailure));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..e2556240b5755
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Base64;
+import java.util.stream.Collectors;
+
+public class TransportPutSamlServiceProviderAction
+    extends HandledTransportAction<PutSamlServiceProviderRequest, PutSamlServiceProviderResponse> {
+
+    private final Logger logger = LogManager.getLogger();
+    private final SamlServiceProviderIndex index;
+    private final SamlIdentityProvider identityProvider;
+    private final Clock clock;
+
+    @Inject
+    public TransportPutSamlServiceProviderAction(TransportService transportService, ActionFilters actionFilters,
+                                                 SamlServiceProviderIndex index, SamlIdentityProvider identityProvider) {
+        this(transportService, actionFilters, index, identityProvider, Clock.systemUTC());
+    }
+
+    TransportPutSamlServiceProviderAction(TransportService transportService, ActionFilters actionFilters,
+                                          SamlServiceProviderIndex index, SamlIdentityProvider identityProvider, Clock clock) {
+        super(PutSamlServiceProviderAction.NAME, transportService, actionFilters, PutSamlServiceProviderRequest::new);
+        this.index = index;
+        this.identityProvider = identityProvider;
+        this.clock = clock;
+    }
+
+    @Override
+    protected void doExecute(Task task, final PutSamlServiceProviderRequest request,
+                             final ActionListener<PutSamlServiceProviderResponse> listener) {
+        final SamlServiceProviderDocument document = request.getDocument();
+        if (document.docId != null) {
+            listener.onFailure(new IllegalArgumentException("request document must not have an id [" + document.docId + "]"));
+            return;
+        }
+        if (document.nameIdFormat != null && identityProvider.getAllowedNameIdFormats().contains(document.nameIdFormat) == false) {
+            listener.onFailure(new IllegalArgumentException("NameID format [" + document.nameIdFormat + "] is not supported."));
+            return;
+        }
+        index.findByEntityId(document.entityId, ActionListener.wrap(matchingDocuments -> {
+            if (matchingDocuments.isEmpty()) {
+                // derive a document id from the entity id so that don't accidentally create duplicate entities due to a race condition
+                document.docId = deriveDocumentId(document);
+                // force a create in case there are concurrent requests. This way, if two nodes/threads are trying to create the SP at
+                // the same time, one will fail. That's not ideal, but it's better than having 1 silently overwrite the other.
+                writeDocument(document, DocWriteRequest.OpType.CREATE, request.getRefreshPolicy(), listener);
+            } else if (matchingDocuments.size() == 1) {
+                final SamlServiceProviderDocument existingDoc = Iterables.get(matchingDocuments, 0).getDocument();
+                assert existingDoc.docId != null : "Loaded document with no doc id";
+                assert existingDoc.entityId.equals(document.entityId) : "Loaded document with non-matching entity-id";
+                document.setDocId(existingDoc.docId);
+                document.setCreated(existingDoc.created);
+                writeDocument(document, DocWriteRequest.OpType.INDEX, request.getRefreshPolicy(), listener);
+            } else {
+                logger.warn("Found multiple existing service providers in [{}] with entity id [{}] - [{}]",
+                    index, document.entityId, matchingDocuments.stream().map(d -> d.getDocument().docId).collect(Collectors.joining(",")));
+                listener.onFailure(new IllegalStateException(
+                    "Multiple service providers already exist with entity id [" + document.entityId + "]"));
+            }
+        }, listener::onFailure));
+    }
+
+    private void writeDocument(SamlServiceProviderDocument document, DocWriteRequest.OpType opType,
+                               WriteRequest.RefreshPolicy refreshPolicy, ActionListener<PutSamlServiceProviderResponse> listener) {
+
+        final Instant now = clock.instant();
+        if (document.created == null || opType == DocWriteRequest.OpType.CREATE) {
+            document.created = now;
+        }
+        document.lastModified = now;
+        final ValidationException validationException = document.validate();
+        if (validationException != null) {
+            listener.onFailure(validationException);
+            return;
+        }
+        logger.debug("[{}] service provider [{}] in document [{}] of [{}]", opType, document.entityId, document.docId, index);
+        index.writeDocument(document, opType, refreshPolicy, ActionListener.wrap(
+            response -> listener.onResponse(new PutSamlServiceProviderResponse(
+                response.getId(),
+                response.getResult() == DocWriteResponse.Result.CREATED,
+                response.getSeqNo(),
+                response.getPrimaryTerm(),
+                document.entityId,
+                document.enabled)),
+            listener::onFailure
+        ));
+    }
+
+    private String deriveDocumentId(SamlServiceProviderDocument document) {
+        final byte[] sha256 = MessageDigests.sha256().digest(document.entityId.getBytes(StandardCharsets.UTF_8));
+        return Base64.getUrlEncoder().withoutPadding().encodeToString(sha256);
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java
new file mode 100644
index 0000000000000..11aff7bf81d1b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.authc.support.SecondaryAuthentication;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.idp.privileges.UserPrivilegeResolver;
+import org.elasticsearch.xpack.idp.saml.authn.FailedAuthenticationResponseMessageBuilder;
+import org.elasticsearch.xpack.idp.saml.authn.SuccessfulAuthenticationResponseMessageBuilder;
+import org.elasticsearch.xpack.idp.saml.authn.UserServiceAuthentication;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlAuthenticationState;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.opensaml.saml.saml2.core.Response;
+import org.opensaml.saml.saml2.core.StatusCode;
+
+import java.time.Clock;
+
+public class TransportSamlInitiateSingleSignOnAction
+    extends HandledTransportAction<SamlInitiateSingleSignOnRequest, SamlInitiateSingleSignOnResponse> {
+
+    private final Logger logger = LogManager.getLogger(TransportSamlInitiateSingleSignOnAction.class);
+
+    private final SecurityContext securityContext;
+    private final SamlIdentityProvider identityProvider;
+    private final SamlFactory samlFactory;
+    private final UserPrivilegeResolver privilegeResolver;
+
+    @Inject
+    public TransportSamlInitiateSingleSignOnAction(TransportService transportService, ActionFilters actionFilters,
+                                                   SecurityContext securityContext, SamlIdentityProvider idp, SamlFactory factory,
+                                                   UserPrivilegeResolver privilegeResolver) {
+        super(SamlInitiateSingleSignOnAction.NAME, transportService, actionFilters, SamlInitiateSingleSignOnRequest::new);
+        this.securityContext = securityContext;
+        this.identityProvider = idp;
+        this.samlFactory = factory;
+        this.privilegeResolver = privilegeResolver;
+    }
+
+    @Override
+    protected void doExecute(Task task, SamlInitiateSingleSignOnRequest request,
+                             ActionListener<SamlInitiateSingleSignOnResponse> listener) {
+        final SamlAuthenticationState authenticationState = request.getSamlAuthenticationState();
+        identityProvider.getRegisteredServiceProvider(request.getSpEntityId(), false, ActionListener.wrap(
+            sp -> {
+                if (null == sp) {
+                    final String message = "Service Provider with Entity ID [" + request.getSpEntityId()
+                        + "] is not registered with this Identity Provider";
+                    logger.debug(message);
+                    possiblyReplyWithSamlFailure(authenticationState, StatusCode.RESPONDER, new IllegalArgumentException(message),
+                        listener);
+                    return;
+                }
+                final SecondaryAuthentication secondaryAuthentication = SecondaryAuthentication.readFromContext(securityContext);
+                if (secondaryAuthentication == null) {
+                    possiblyReplyWithSamlFailure(authenticationState,
+                        StatusCode.REQUESTER,
+                        new ElasticsearchSecurityException("Request is missing secondary authentication", RestStatus.FORBIDDEN),
+                        listener);
+                    return;
+                }
+                buildUserFromAuthentication(secondaryAuthentication, sp, ActionListener.wrap(
+                    user -> {
+                        if (user == null) {
+                            possiblyReplyWithSamlFailure(authenticationState,
+                                StatusCode.REQUESTER,
+                                new ElasticsearchSecurityException("User [{}] is not permitted to access service [{}]",
+                                    RestStatus.FORBIDDEN, secondaryAuthentication.getUser(), sp),
+                                listener);
+                            return;
+                        }
+                        final SuccessfulAuthenticationResponseMessageBuilder builder =
+                            new SuccessfulAuthenticationResponseMessageBuilder(samlFactory, Clock.systemUTC(), identityProvider);
+                        try {
+                            final Response response = builder.build(user, authenticationState);
+                            listener.onResponse(new SamlInitiateSingleSignOnResponse(
+                                user.getServiceProvider().getAssertionConsumerService().toString(),
+                                samlFactory.getXmlContent(response),
+                                user.getServiceProvider().getEntityId()));
+                        } catch (ElasticsearchException e) {
+                            listener.onFailure(e);
+                        }
+                    },
+                    e -> possiblyReplyWithSamlFailure(authenticationState, StatusCode.RESPONDER, e, listener)
+                ));
+            },
+            e -> possiblyReplyWithSamlFailure(authenticationState, StatusCode.RESPONDER, e, listener)
+        ));
+    }
+
+    private void buildUserFromAuthentication(SecondaryAuthentication secondaryAuthentication, SamlServiceProvider serviceProvider,
+                                             ActionListener<UserServiceAuthentication> listener) {
+        User user = secondaryAuthentication.getUser();
+        secondaryAuthentication.execute(ignore -> {
+                privilegeResolver.resolve(serviceProvider.getPrivileges(), ActionListener.wrap(
+                    userPrivileges -> {
+                        if (userPrivileges.hasAccess == false) {
+                            listener.onResponse(null);
+                        } else {
+                            logger.debug("Resolved [{}] for [{}]", userPrivileges, user);
+                            listener.onResponse(new UserServiceAuthentication(user.principal(), user.fullName(), user.email(),
+                                userPrivileges.roles, serviceProvider));
+                        }
+                    },
+                    listener::onFailure
+                ));
+                return null;
+            }
+        );
+    }
+
+    private void possiblyReplyWithSamlFailure(SamlAuthenticationState authenticationState, String statusCode, Exception e,
+                                              ActionListener<SamlInitiateSingleSignOnResponse> listener) {
+        if (authenticationState != null) {
+            final FailedAuthenticationResponseMessageBuilder builder =
+                new FailedAuthenticationResponseMessageBuilder(samlFactory, Clock.systemUTC(), identityProvider)
+                    .setInResponseTo(authenticationState.getAuthnRequestId())
+                    .setAcsUrl(authenticationState.getRequestedAcsUrl())
+                    .setPrimaryStatusCode(statusCode);
+            final Response response = builder.build();
+            //TODO: Log and indicate SAML Response status is failure in the response
+            listener.onResponse(new SamlInitiateSingleSignOnResponse(
+                authenticationState.getRequestedAcsUrl(),
+                samlFactory.getXmlContent(response),
+                authenticationState.getEntityId()));
+        } else {
+            listener.onFailure(e);
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java
new file mode 100644
index 0000000000000..9d17fae407a7e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.idp.SamlMetadataGenerator;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+
+public class TransportSamlMetadataAction extends HandledTransportAction<SamlMetadataRequest, SamlMetadataResponse> {
+
+    private final SamlIdentityProvider identityProvider;
+    private final SamlFactory samlFactory;
+
+    @Inject
+    public TransportSamlMetadataAction(TransportService transportService, ActionFilters actionFilters,
+                                       SamlIdentityProvider idp, SamlFactory factory) {
+        super(SamlMetadataAction.NAME, transportService, actionFilters, SamlMetadataRequest::new);
+        this.identityProvider = idp;
+        this.samlFactory = factory;
+    }
+
+    @Override
+    protected void doExecute(Task task, SamlMetadataRequest request, ActionListener<SamlMetadataResponse> listener) {
+        final String spEntityId = request.getSpEntityId();
+        final SamlMetadataGenerator generator = new SamlMetadataGenerator(samlFactory, identityProvider);
+        generator.generateMetadata(spEntityId, listener);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java
new file mode 100644
index 0000000000000..9b2b1d3a819ab
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.idp.saml.authn.SamlAuthnRequestValidator;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+
+public class TransportSamlValidateAuthnRequestAction
+    extends HandledTransportAction<SamlValidateAuthnRequestRequest, SamlValidateAuthnRequestResponse> {
+
+    private final SamlIdentityProvider identityProvider;
+    private final SamlFactory samlFactory;
+
+    @Inject
+    public TransportSamlValidateAuthnRequestAction(TransportService transportService, ActionFilters actionFilters,
+                                                   SamlIdentityProvider idp, SamlFactory factory) {
+        super(SamlValidateAuthnRequestAction.NAME, transportService, actionFilters, SamlValidateAuthnRequestRequest::new);
+        this.identityProvider = idp;
+        this.samlFactory = factory;
+    }
+
+    @Override
+    protected void doExecute(Task task, SamlValidateAuthnRequestRequest request,
+                             ActionListener<SamlValidateAuthnRequestResponse> listener) {
+        final SamlAuthnRequestValidator validator = new SamlAuthnRequestValidator(samlFactory, identityProvider);
+        try {
+            validator.processQueryString(request.getQueryString(), listener);
+        } catch (Exception e) {
+            listener.onFailure(e);
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/AuthenticationMethod.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/AuthenticationMethod.java
new file mode 100644
index 0000000000000..32d93261f0cf2
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/AuthenticationMethod.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.authc;
+
+/**
+ * Denotes support authentication methods for users
+ */
+public enum AuthenticationMethod {
+    PASSWORD,
+    KERBEROS,
+    TLS_CLIENT_AUTH,
+    PRIOR_SESSION,
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/NetworkControl.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/NetworkControl.java
new file mode 100644
index 0000000000000..00f53dff6f032
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/authc/NetworkControl.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.authc;
+
+/**
+ * Denotes network based controls that were applied during authentication of a user
+ */
+public enum NetworkControl {
+    IP_FILTER,
+    TLS,
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/ServiceProviderPrivileges.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/ServiceProviderPrivileges.java
new file mode 100644
index 0000000000000..794cdc7b72ac1
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/ServiceProviderPrivileges.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.privileges;
+
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+
+import java.util.Map;
+import java.util.Objects;
+
+public class ServiceProviderPrivileges {
+
+    private final String applicationName;
+    private final String resource;
+    private final Map<String, String> roles;
+
+    public ServiceProviderPrivileges(String applicationName, String resource, Map<String, String> roles) {
+        this.applicationName = Objects.requireNonNull(applicationName, "Application name cannot be null");
+        this.resource = Objects.requireNonNull(resource, "Resource cannot be null");
+        this.roles = Map.copyOf(roles);
+    }
+
+    /**
+     * Returns the "application" (see {@link RoleDescriptor.ApplicationResourcePrivileges#getApplication()}) in the IdP's security cluster
+     * under which this service provider is defined.
+     */
+    public String getApplicationName() {
+        return applicationName;
+    }
+
+    /**
+     * Returns the "resource" (see {@link RoleDescriptor.ApplicationResourcePrivileges#getResources()}) that represents this
+     * Service Provider in the IdP's security cluster.
+     */
+    public String getResource() {
+        return resource;
+    }
+
+    /**
+     * Returns a mapping from "role name" (key) to "{@link ApplicationPrivilegeDescriptor#getActions() action name}" (value)
+     * that represents the roles that should be exposed to this Service Provider.
+     * The "role name" (but not the action name) will be provided to the service provider.
+     * These roles have no semantic meaning within the IdP, they are simply metadata that we pass to the Service Provider. They may not
+     * have any relationship to the roles that the user has in this cluster, and the service provider may refer to them using a different
+     * terminology (e.g. "groups").
+     * The actions will be resolved as application privileges from the IdP's security cluster.
+     */
+    public Map<String, String> getRoleActions() {
+        return roles;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolver.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolver.java
new file mode 100644
index 0000000000000..8c0e67dbbe166
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolver.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.privileges;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Determines what privileges a user has within a given {@link ServiceProviderPrivileges service}.
+ */
+public class UserPrivilegeResolver {
+
+    public static class UserPrivileges {
+        public final String principal;
+        public final boolean hasAccess;
+        public final Set<String> roles;
+
+        public UserPrivileges(String principal, boolean hasAccess, Set<String> roles) {
+            this.principal = Objects.requireNonNull(principal, "principal may not be null");
+            if (hasAccess == false && roles.isEmpty() == false) {
+                throw new IllegalArgumentException("a user without access ([" + hasAccess + "]) may not have roles ([" + roles + "])");
+            }
+            this.hasAccess = hasAccess;
+            this.roles = Set.copyOf(Objects.requireNonNull(roles, "roles may not be null"));
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder str = new StringBuilder()
+                .append(getClass().getSimpleName())
+                .append("{")
+                .append(principal)
+                .append(", ")
+                .append(hasAccess);
+            if (hasAccess) {
+                str.append(", ").append(roles);
+            }
+            str.append("}");
+            return str.toString();
+        }
+
+        public static UserPrivileges noAccess(String principal) {
+            return new UserPrivileges(principal, false, Set.of());
+        }
+    }
+
+    private final Logger logger = LogManager.getLogger();
+    private final Client client;
+    private final SecurityContext securityContext;
+
+    public UserPrivilegeResolver(Client client, SecurityContext securityContext) {
+        this.client = client;
+        this.securityContext = securityContext;
+    }
+
+    /**
+     * Resolves the user's privileges for the specified service.
+     * Requires that the active user is set in the {@link org.elasticsearch.xpack.core.security.SecurityContext}.
+     */
+    public void resolve(ServiceProviderPrivileges service, ActionListener<UserPrivileges> listener) {
+        HasPrivilegesRequest request = new HasPrivilegesRequest();
+        final String username = securityContext.requireUser().principal();
+        request.username(username);
+        request.applicationPrivileges(buildResourcePrivilege(service));
+        request.clusterPrivileges(Strings.EMPTY_ARRAY);
+        request.indexPrivileges(new RoleDescriptor.IndicesPrivileges[0]);
+        client.execute(HasPrivilegesAction.INSTANCE, request, ActionListener.wrap(
+            response -> {
+                logger.debug("Checking access for user [{}] to application [{}] resource [{}]",
+                    username, service.getApplicationName(), service.getResource());
+                UserPrivileges privileges = buildResult(response, service);
+                logger.debug("Resolved service privileges [{}]", privileges);
+                listener.onResponse(privileges);
+            },
+            listener::onFailure
+        ));
+    }
+
+    private UserPrivileges buildResult(HasPrivilegesResponse response, ServiceProviderPrivileges service) {
+        final Set<ResourcePrivileges> appPrivileges = response.getApplicationPrivileges().get(service.getApplicationName());
+        if (appPrivileges == null || appPrivileges.isEmpty()) {
+            return UserPrivileges.noAccess(response.getUsername());
+        }
+        final Set<String> roles = service.getRoleActions().entrySet().stream()
+            .filter(entry -> checkAccess(appPrivileges, entry.getValue(), service.getResource()))
+            .map(Map.Entry::getKey)
+            .collect(Collectors.toUnmodifiableSet());
+        final boolean hasAccess = roles.isEmpty() == false;
+        return new UserPrivileges(response.getUsername(), hasAccess, roles);
+    }
+
+    private boolean checkAccess(Set<ResourcePrivileges> userPrivileges, String action, String resource) {
+        final Optional<ResourcePrivileges> match = userPrivileges.stream()
+            .filter(rp -> rp.getResource().equals(resource))
+            .filter(rp -> rp.isAllowed(action))
+            .findAny();
+        match.ifPresent(rp -> logger.debug("User has access to [{} on {}] via [{}]", action, resource, rp));
+        return match.isPresent();
+    }
+
+    private RoleDescriptor.ApplicationResourcePrivileges buildResourcePrivilege(ServiceProviderPrivileges service) {
+        final RoleDescriptor.ApplicationResourcePrivileges.Builder builder = RoleDescriptor.ApplicationResourcePrivileges.builder();
+        builder.application(service.getApplicationName());
+        builder.resources(service.getResource());
+        builder.privileges(service.getRoleActions().values());
+        return builder.build();
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseMessageBuilder.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseMessageBuilder.java
new file mode 100644
index 0000000000000..60781526492ab
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseMessageBuilder.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.Response;
+import org.opensaml.saml.saml2.core.Status;
+import org.opensaml.saml.saml2.core.StatusCode;
+import org.opensaml.saml.saml2.core.StatusMessage;
+
+import java.time.Clock;
+
+public class FailedAuthenticationResponseMessageBuilder {
+
+    private final Clock clock;
+    private final SamlFactory samlFactory;
+    private final SamlIdentityProvider idp;
+    private String primaryStatusCode;
+    private String statusMessage;
+    private String acsUrl;
+    private String inResponseTo;
+    private String secondaryStatusCode;
+
+    public FailedAuthenticationResponseMessageBuilder(SamlFactory samlFactory, Clock clock, SamlIdentityProvider idp) {
+        SamlInit.initialize();
+        this.samlFactory = samlFactory;
+        this.clock = clock;
+        this.idp = idp;
+        this.primaryStatusCode = StatusCode.REQUESTER;
+    }
+
+    public FailedAuthenticationResponseMessageBuilder setAcsUrl(String acsUrl) {
+        this.acsUrl = acsUrl;
+        return this;
+    }
+
+    public FailedAuthenticationResponseMessageBuilder setPrimaryStatusCode(String primaryStatusCode) {
+        this.primaryStatusCode = primaryStatusCode;
+        return this;
+    }
+
+    public FailedAuthenticationResponseMessageBuilder setStatusMessage(String statusMessage) {
+        this.statusMessage = statusMessage;
+        return this;
+    }
+
+    public FailedAuthenticationResponseMessageBuilder setSecondaryStatusCode(String secondaryStatusCode) {
+        this.secondaryStatusCode = secondaryStatusCode;
+        return this;
+    }
+
+    public FailedAuthenticationResponseMessageBuilder setInResponseTo(String inResponseTo) {
+        this.inResponseTo = inResponseTo;
+        return this;
+    }
+
+    public Response build() {
+        final DateTime now = now();
+        final Response response = samlFactory.object(Response.class, Response.DEFAULT_ELEMENT_NAME);
+        response.setID(samlFactory.secureIdentifier());
+        response.setInResponseTo(inResponseTo);
+        response.setIssuer(buildIssuer());
+        response.setIssueInstant(now);
+        response.setStatus(buildStatus());
+        response.setDestination(acsUrl);
+        return response;
+    }
+
+    private Issuer buildIssuer() {
+        final Issuer issuer = samlFactory.object(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(this.idp.getEntityId());
+        return issuer;
+    }
+
+    private Status buildStatus() {
+        final StatusCode firstLevelCode = samlFactory.object(StatusCode.class, StatusCode.DEFAULT_ELEMENT_NAME);
+        firstLevelCode.setValue(primaryStatusCode);
+
+        if (Strings.hasText(secondaryStatusCode)) {
+            final StatusCode secondLevelCode = samlFactory.object(StatusCode.class, StatusCode.DEFAULT_ELEMENT_NAME);
+            secondLevelCode.setValue(secondaryStatusCode);
+            firstLevelCode.setStatusCode(secondLevelCode);
+        }
+
+        final Status status = samlFactory.object(Status.class, Status.DEFAULT_ELEMENT_NAME);
+        if (Strings.hasText(statusMessage)) {
+            final StatusMessage firstLevelMessage = samlFactory.object(StatusMessage.class, StatusMessage.DEFAULT_ELEMENT_NAME);
+            firstLevelMessage.setMessage(statusMessage);
+            status.setStatusMessage(firstLevelMessage);
+        }
+        status.setStatusCode(firstLevelCode);
+        return status;
+    }
+
+    private DateTime now() {
+        return new DateTime(clock.millis(), DateTimeZone.UTC);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidator.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidator.java
new file mode 100644
index 0000000000000..eba6fe390e91e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidator.java
@@ -0,0 +1,354 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.internal.io.Streams;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestResponse;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlAuthenticationState;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.opensaml.saml.saml2.core.AuthnRequest;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.NameIDPolicy;
+import org.opensaml.security.x509.X509Credential;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
+
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.UNSPECIFIED;
+
+/**
+ * Processes a SAML AuthnRequest, validates it, extracts necessary information and returns a {@link SamlValidateAuthnRequestResponse}
+ */
+public class SamlAuthnRequestValidator {
+
+    private final SamlFactory samlFactory;
+    private final SamlIdentityProvider idp;
+    private final Logger logger = LogManager.getLogger(SamlAuthnRequestValidator.class);
+    private static final String[] XSD_FILES = new String[]{"/org/elasticsearch/xpack/idp/saml/support/saml-schema-protocol-2.0.xsd",
+        "/org/elasticsearch/xpack/idp/saml/support/saml-schema-assertion-2.0.xsd",
+        "/org/elasticsearch/xpack/idp/saml/support/xenc-schema.xsd",
+        "/org/elasticsearch/xpack/idp/saml/support/xmldsig-core-schema.xsd"};
+
+    private static final ThreadLocal<DocumentBuilder> THREAD_LOCAL_DOCUMENT_BUILDER = ThreadLocal.withInitial(() -> {
+        try {
+            return SamlFactory.getHardenedBuilder(XSD_FILES);
+        } catch (Exception e) {
+            throw new ElasticsearchSecurityException("Could not load XSD schema file", e);
+        }
+    });
+
+    public SamlAuthnRequestValidator(SamlFactory samlFactory, SamlIdentityProvider idp) {
+        SamlInit.initialize();
+        this.samlFactory = samlFactory;
+        this.idp = idp;
+    }
+
+    public void processQueryString(String queryString, ActionListener<SamlValidateAuthnRequestResponse> listener) {
+
+        final ParsedQueryString parsedQueryString;
+        try {
+            parsedQueryString = parseQueryString(queryString);
+        } catch (ElasticsearchSecurityException e) {
+            logger.debug("Failed to parse query string for SAML AuthnRequest", e);
+            listener.onFailure(e);
+            return;
+        }
+
+        try {
+            // We consciously parse the AuthnRequest before we validate its signature as we need to get the Issuer, in order to
+            // verify if we know of this SP and get its credentials for signature verification
+            final Element root = parseSamlMessage(inflate(decodeBase64(parsedQueryString.samlRequest)));
+            if (samlFactory.elementNameMatches(root, "urn:oasis:names:tc:SAML:2.0:protocol", "AuthnRequest") == false) {
+                logAndRespond(new ParameterizedMessage("SAML message [{}] is not an AuthnRequest", samlFactory.text(root, 128)), listener);
+                return;
+            }
+            final AuthnRequest authnRequest = samlFactory.buildXmlObject(root, AuthnRequest.class);
+            getSpFromIssuer(authnRequest.getIssuer(), ActionListener.wrap(
+                sp -> {
+                    try {
+                        validateAuthnRequest(authnRequest, sp, parsedQueryString, listener);
+                    } catch (ElasticsearchSecurityException e) {
+                        logger.debug("Could not validate AuthnRequest", e);
+                        listener.onFailure(e);
+                    } catch (Exception e) {
+                        logAndRespond("Could not validate AuthnRequest", e, listener);
+                    }
+                },
+                listener::onFailure
+            ));
+        } catch (ElasticsearchSecurityException e) {
+            logger.debug("Could not process AuthnRequest", e);
+            listener.onFailure(e);
+        } catch (Exception e) {
+            logAndRespond("Could not process AuthnRequest", e, listener);
+        }
+    }
+
+    private ParsedQueryString parseQueryString(String queryString) throws ElasticsearchSecurityException {
+
+        final Map<String, String> parameters = new HashMap<>();
+        RestUtils.decodeQueryString(queryString, 0, parameters);
+        if (parameters.isEmpty()) {
+            throw new ElasticsearchSecurityException("Invalid Authentication Request query string (zero parameters)");
+        }
+        logger.trace(new ParameterizedMessage("Parsed the following parameters from the query string: {}", parameters));
+        final String samlRequest = parameters.get("SAMLRequest");
+        if (null == samlRequest) {
+            throw new ElasticsearchSecurityException("Query string [{}] does not contain a SAMLRequest parameter",
+                RestStatus.BAD_REQUEST, queryString);
+        }
+        return new ParsedQueryString(
+            queryString,
+            samlRequest,
+            parameters.get("RelayState"),
+            parameters.get("SigAlg"),
+            parameters.get("Signature"));
+    }
+
+    private void validateAuthnRequest(AuthnRequest authnRequest, SamlServiceProvider sp, ParsedQueryString parsedQueryString,
+                                         ActionListener<SamlValidateAuthnRequestResponse> listener) {
+        // If the Service Provider should not sign requests, do not try to handle signatures even if they are added to the request
+        if (sp.shouldSignAuthnRequests()) {
+            if (Strings.hasText(parsedQueryString.signature)) {
+                if (Strings.hasText(parsedQueryString.sigAlg) == false) {
+                    logAndRespond(new ParameterizedMessage("Query string [{}] contains a Signature but SigAlg parameter is missing",
+                        parsedQueryString.queryString), listener);
+                    return;
+                }
+                final Set<X509Credential> spSigningCredentials = sp.getSpSigningCredentials();
+                if (spSigningCredentials == null || spSigningCredentials.isEmpty()) {
+                    logAndRespond(new ParameterizedMessage("Unable to validate signature of authentication request, " +
+                        "Service Provider [{}] hasn't registered signing credentials", sp.getEntityId()), listener);
+                    return;
+                }
+                if (validateSignature(parsedQueryString, spSigningCredentials) == false) {
+                    logAndRespond(
+                        new ParameterizedMessage("Unable to validate signature of authentication request [{}] using credentials [{}]",
+                            parsedQueryString.queryString, samlFactory.describeCredentials(spSigningCredentials)), listener);
+                    return;
+                }
+            } else if (Strings.hasText(parsedQueryString.sigAlg)) {
+                logAndRespond(new ParameterizedMessage("Query string [{}] contains a SigAlg parameter but Signature is missing",
+                    parsedQueryString.queryString), listener);
+                return;
+            } else {
+                logAndRespond(
+                    new ParameterizedMessage(
+                        "The Service Provider [{}] must sign authentication requests but no signature was found", sp.getEntityId()),
+                    listener);
+                return;
+            }
+        }
+        final Map<String, Object> authnState = new HashMap<>();
+        checkDestination(authnRequest);
+        checkAcs(authnRequest, sp, authnState);
+        validateNameIdPolicy(authnRequest, sp, authnState);
+        authnState.put(SamlAuthenticationState.Fields.ENTITY_ID.getPreferredName(), sp.getEntityId());
+        authnState.put(SamlAuthenticationState.Fields.AUTHN_REQUEST_ID.getPreferredName(), authnRequest.getID());
+        final SamlValidateAuthnRequestResponse response = new SamlValidateAuthnRequestResponse(sp.getEntityId(),
+            authnRequest.isForceAuthn(), authnState);
+        logger.trace(new ParameterizedMessage("Validated AuthnResponse from queryString [{}] and extracted [{}]",
+            parsedQueryString.queryString, response));
+        listener.onResponse(response);
+    }
+
+    private void validateNameIdPolicy(AuthnRequest request, SamlServiceProvider sp, Map<String, Object> authnState) {
+        final NameIDPolicy nameIDPolicy = request.getNameIDPolicy();
+        if (null != nameIDPolicy) {
+            final String requestedFormat = nameIDPolicy.getFormat();
+            final String allowedFormat = sp.getAllowedNameIdFormat();
+            if (Strings.hasText(requestedFormat)) {
+                if (allowedFormat != null && requestedFormat.equals(UNSPECIFIED) == false
+                    && requestedFormat.equals(allowedFormat) == false) {
+                    throw new ElasticsearchSecurityException("The requested NameID format [{}] doesn't match the allowed NameID format" +
+                        " for this Service Provider which is [{}]", requestedFormat, sp.getAllowedNameIdFormat());
+                } else {
+                    authnState.put(SamlAuthenticationState.Fields.NAMEID_FORMAT.getPreferredName(), requestedFormat);
+                }
+            }
+        }
+    }
+
+    private boolean validateSignature(ParsedQueryString queryString, Collection<X509Credential> credentials) {
+        final String javaSigAlgorithm = samlFactory.getJavaAlorithmNameFromUri(queryString.sigAlg);
+        final byte[] contentBytes = queryString.reconstructQueryParameters().getBytes(StandardCharsets.UTF_8);
+        final byte[] signatureBytes = Base64.getDecoder().decode(queryString.signature);
+        return credentials.stream().anyMatch(credential -> {
+            try {
+                Signature sig = Signature.getInstance(javaSigAlgorithm);
+                sig.initVerify(credential.getEntityCertificate().getPublicKey());
+                sig.update(contentBytes);
+                return sig.verify(signatureBytes);
+            } catch (NoSuchAlgorithmException e) {
+                throw new ElasticsearchSecurityException("Java signature algorithm [{}] is not available for SAML/XML-Sig algorithm [{}]",
+                    e, javaSigAlgorithm, queryString.sigAlg);
+            } catch (InvalidKeyException | SignatureException e) {
+                logger.warn(new ParameterizedMessage("Signature verification failed for credential [{}]",
+                    samlFactory.describeCredentials(Set.of(credential))), e);
+                return false;
+            }
+        });
+    }
+
+    private void getSpFromIssuer(Issuer issuer, ActionListener<SamlServiceProvider> listener) {
+        if (issuer == null || issuer.getValue() == null) {
+            throw new ElasticsearchSecurityException("SAML authentication request has no issuer", RestStatus.BAD_REQUEST);
+        }
+        final String issuerString = issuer.getValue();
+        idp.getRegisteredServiceProvider(issuerString, false, ActionListener.wrap(
+            serviceProvider -> {
+                if (null == serviceProvider) {
+                    throw new ElasticsearchSecurityException(
+                        "Service Provider with Entity ID [{}] is not registered with this Identity Provider", RestStatus.BAD_REQUEST,
+                        issuerString);
+                }
+                listener.onResponse(serviceProvider);
+            },
+            listener::onFailure
+        ));
+    }
+
+    private void checkDestination(AuthnRequest request) {
+        final String url = idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI).toString();
+        if (url.equals(request.getDestination()) == false) {
+            throw new ElasticsearchSecurityException(
+                "SAML authentication request [{}] is for destination [{}] but the SSO endpoint of this Identity Provider is [{}]",
+                RestStatus.BAD_REQUEST, request.getID(), request.getDestination(), url);
+        }
+    }
+
+    private void checkAcs(AuthnRequest request, SamlServiceProvider sp, Map<String, Object> authnState) {
+        final String acs = request.getAssertionConsumerServiceURL();
+        if (Strings.hasText(acs) == false) {
+            final String message = request.getAssertionConsumerServiceIndex() == null ?
+                "SAML authentication does not contain an AssertionConsumerService URL" :
+                "SAML authentication does not contain an AssertionConsumerService URL. It contains an Assertion Consumer Service Index " +
+                    "but this IDP doesn't support multiple AssertionConsumerService URLs.";
+            throw new ElasticsearchSecurityException(message, RestStatus.BAD_REQUEST);
+        }
+        if (acs.equals(sp.getAssertionConsumerService().toString()) == false) {
+            throw new ElasticsearchSecurityException("The registered ACS URL for this Service Provider is [{}] but the authentication " +
+                "request contained [{}]", RestStatus.BAD_REQUEST, sp.getAssertionConsumerService(), acs);
+        }
+        authnState.put(SamlAuthenticationState.Fields.ACS_URL.getPreferredName(), acs);
+    }
+
+    protected Element parseSamlMessage(byte[] content) {
+        final Element root;
+        try (ByteArrayInputStream input = new ByteArrayInputStream(content)) {
+            // This will parse and validate the input against the schemas
+            final Document doc = THREAD_LOCAL_DOCUMENT_BUILDER.get().parse(input);
+            root = doc.getDocumentElement();
+            if (logger.isTraceEnabled()) {
+                logger.trace("Received SAML Message: {} \n", samlFactory.toString(root, true));
+            }
+        } catch (SAXException | IOException e) {
+            throw new ElasticsearchSecurityException("Failed to parse SAML message", RestStatus.BAD_REQUEST, e);
+        }
+        return root;
+    }
+
+    private byte[] decodeBase64(String content) {
+        try {
+            return Base64.getDecoder().decode(content.replaceAll("\\s+", ""));
+        } catch (IllegalArgumentException e) {
+            logger.info("Failed to decode base64 string [{}] - {}", content, e);
+            throw new ElasticsearchSecurityException("SAML message cannot be Base64 decoded", RestStatus.BAD_REQUEST, e);
+        }
+    }
+
+    private byte[] inflate(byte[] bytes) {
+        Inflater inflater = new Inflater(true);
+        try (ByteArrayInputStream in = new ByteArrayInputStream(bytes);
+             InflaterInputStream inflate = new InflaterInputStream(in, inflater);
+             ByteArrayOutputStream out = new ByteArrayOutputStream(bytes.length * 3 / 2)) {
+            Streams.copy(inflate, out);
+            return out.toByteArray();
+        } catch (IOException e) {
+            throw new ElasticsearchSecurityException("SAML message cannot be inflated", RestStatus.BAD_REQUEST, e);
+        }
+    }
+
+    private String urlEncode(String param) throws UnsupportedEncodingException {
+        return URLEncoder.encode(param, StandardCharsets.UTF_8.name());
+    }
+
+    private void logAndRespond(String message, ActionListener<SamlValidateAuthnRequestResponse> listener) {
+        logger.debug(message);
+        listener.onFailure(new ElasticsearchSecurityException(message));
+    }
+
+    private void logAndRespond(ParameterizedMessage message, ActionListener<SamlValidateAuthnRequestResponse> listener) {
+        logAndRespond(message.getFormattedMessage(), listener);
+    }
+
+    private void logAndRespond(String message, Throwable e, ActionListener<SamlValidateAuthnRequestResponse> listener) {
+        logger.debug(message);
+        listener.onFailure(new ElasticsearchSecurityException(message, e));
+    }
+
+    private class ParsedQueryString {
+        private final String queryString;
+        private final String samlRequest;
+        @Nullable
+        private final String relayState;
+        @Nullable
+        private final String sigAlg;
+        @Nullable
+        private final String signature;
+
+        private ParsedQueryString(String queryString, String samlRequest, String relayState, String sigAlg, String signature) {
+            this.queryString = Objects.requireNonNull(queryString, "Query string may not be null");
+            this.samlRequest = Objects.requireNonNull(samlRequest, "SAML request parameter may not be null");
+            this.relayState = relayState;
+            this.sigAlg = sigAlg;
+            this.signature = signature;
+        }
+
+        public String reconstructQueryParameters() throws ElasticsearchSecurityException {
+            try {
+                return relayState == null ?
+                    "SAMLRequest=" + urlEncode(samlRequest) + "&SigAlg=" + urlEncode(sigAlg) :
+                    "SAMLRequest=" + urlEncode(samlRequest) + "&RelayState=" + urlEncode(relayState) + "&SigAlg=" + urlEncode(sigAlg);
+            } catch (UnsupportedEncodingException e) {
+                throw new ElasticsearchSecurityException("Cannot reconstruct query for signature verification", e);
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilder.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilder.java
new file mode 100644
index 0000000000000..470834b23b177
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilder.java
@@ -0,0 +1,274 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.idp.authc.AuthenticationMethod;
+import org.elasticsearch.xpack.idp.authc.NetworkControl;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlAuthenticationState;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.support.SamlObjectSigner;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.opensaml.core.xml.schema.XSString;
+import org.opensaml.saml.saml2.core.Assertion;
+import org.opensaml.saml.saml2.core.Attribute;
+import org.opensaml.saml.saml2.core.AttributeStatement;
+import org.opensaml.saml.saml2.core.AttributeValue;
+import org.opensaml.saml.saml2.core.Audience;
+import org.opensaml.saml.saml2.core.AudienceRestriction;
+import org.opensaml.saml.saml2.core.AuthnContext;
+import org.opensaml.saml.saml2.core.AuthnContextClassRef;
+import org.opensaml.saml.saml2.core.AuthnStatement;
+import org.opensaml.saml.saml2.core.Conditions;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.NameID;
+import org.opensaml.saml.saml2.core.Response;
+import org.opensaml.saml.saml2.core.Status;
+import org.opensaml.saml.saml2.core.StatusCode;
+import org.opensaml.saml.saml2.core.Subject;
+import org.opensaml.saml.saml2.core.SubjectConfirmation;
+import org.opensaml.saml.saml2.core.SubjectConfirmationData;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+/**
+ * Builds SAML 2.0 {@link Response} objects for successful authentication results.
+ */
+public class SuccessfulAuthenticationResponseMessageBuilder {
+
+    private final Logger logger = LogManager.getLogger();
+
+    private final Clock clock;
+    private final SamlIdentityProvider idp;
+    private final SamlFactory samlFactory;
+
+    public SuccessfulAuthenticationResponseMessageBuilder(SamlFactory samlFactory, Clock clock, SamlIdentityProvider idp) {
+        SamlInit.initialize();
+        this.samlFactory = samlFactory;
+        this.clock = clock;
+        this.idp = idp;
+    }
+
+    public Response build(UserServiceAuthentication user, @Nullable SamlAuthenticationState authnState) {
+        logger.debug("Building success response for [{}] from [{}]", user, authnState);
+        final DateTime now = now();
+        final SamlServiceProvider serviceProvider = user.getServiceProvider();
+
+        final Response response = samlFactory.object(Response.class, Response.DEFAULT_ELEMENT_NAME);
+        response.setID(samlFactory.secureIdentifier());
+        if (authnState != null && authnState.getAuthnRequestId() != null) {
+            response.setInResponseTo(authnState.getAuthnRequestId());
+        }
+        response.setIssuer(buildIssuer());
+        response.setIssueInstant(now);
+        response.setStatus(buildStatus());
+        response.setDestination(serviceProvider.getAssertionConsumerService().toString());
+
+        final Assertion assertion = samlFactory.object(Assertion.class, Assertion.DEFAULT_ELEMENT_NAME);
+        assertion.setID(samlFactory.secureIdentifier());
+        assertion.setIssuer(buildIssuer());
+        assertion.setIssueInstant(now);
+        assertion.setConditions(buildConditions(now, serviceProvider));
+        assertion.setSubject(buildSubject(now, user, authnState));
+        assertion.getAuthnStatements().add(buildAuthnStatement(now, user));
+        final AttributeStatement attributes = buildAttributes(user);
+        if (attributes != null) {
+            assertion.getAttributeStatements().add(attributes);
+        }
+        response.getAssertions().add(assertion);
+        return sign(response);
+    }
+
+    private Response sign(Response response) {
+        final SamlObjectSigner signer = new SamlObjectSigner(samlFactory, idp);
+        return samlFactory.buildXmlObject(signer.sign(response), Response.class);
+    }
+
+    private Conditions buildConditions(DateTime now, SamlServiceProvider serviceProvider) {
+        final Audience spAudience = samlFactory.object(Audience.class, Audience.DEFAULT_ELEMENT_NAME);
+        spAudience.setAudienceURI(serviceProvider.getEntityId());
+
+        final AudienceRestriction restriction = samlFactory.object(AudienceRestriction.class, AudienceRestriction.DEFAULT_ELEMENT_NAME);
+        restriction.getAudiences().add(spAudience);
+
+        final Conditions conditions = samlFactory.object(Conditions.class, Conditions.DEFAULT_ELEMENT_NAME);
+        conditions.setNotBefore(now);
+        conditions.setNotOnOrAfter(now.plus(serviceProvider.getAuthnExpiry()));
+        conditions.getAudienceRestrictions().add(restriction);
+        return conditions;
+    }
+
+    private DateTime now() {
+        return new DateTime(clock.millis(), DateTimeZone.UTC);
+    }
+
+    private Subject buildSubject(DateTime now, UserServiceAuthentication user, SamlAuthenticationState authnState) {
+        final SamlServiceProvider serviceProvider = user.getServiceProvider();
+
+        final NameID nameID = buildNameId(user, authnState);
+
+        final Subject subject = samlFactory.object(Subject.class, Subject.DEFAULT_ELEMENT_NAME);
+        subject.setNameID(nameID);
+
+        final SubjectConfirmationData data = samlFactory.object(SubjectConfirmationData.class,
+            SubjectConfirmationData.DEFAULT_ELEMENT_NAME);
+        if (authnState != null && authnState.getAuthnRequestId() != null) {
+            data.setInResponseTo(authnState.getAuthnRequestId());
+        }
+        data.setNotBefore(now);
+        data.setNotOnOrAfter(now.plus(serviceProvider.getAuthnExpiry()));
+        data.setRecipient(serviceProvider.getAssertionConsumerService().toString());
+
+        final SubjectConfirmation confirmation = samlFactory.object(SubjectConfirmation.class, SubjectConfirmation.DEFAULT_ELEMENT_NAME);
+        confirmation.setMethod(SubjectConfirmation.METHOD_BEARER);
+        confirmation.setSubjectConfirmationData(data);
+
+        subject.getSubjectConfirmations().add(confirmation);
+        return subject;
+    }
+
+    private AuthnStatement buildAuthnStatement(DateTime now, UserServiceAuthentication user) {
+        final SamlServiceProvider serviceProvider = user.getServiceProvider();
+        final AuthnStatement statement = samlFactory.object(AuthnStatement.class, AuthnStatement.DEFAULT_ELEMENT_NAME);
+        statement.setAuthnInstant(now);
+        statement.setSessionNotOnOrAfter(now.plus(serviceProvider.getAuthnExpiry()));
+
+        final AuthnContext context = samlFactory.object(AuthnContext.class, AuthnContext.DEFAULT_ELEMENT_NAME);
+        final AuthnContextClassRef classRef = samlFactory.object(AuthnContextClassRef.class, AuthnContextClassRef.DEFAULT_ELEMENT_NAME);
+        classRef.setAuthnContextClassRef(resolveAuthnClass(user.getAuthenticationMethods(), user.getNetworkControls()));
+        context.setAuthnContextClassRef(classRef);
+        statement.setAuthnContext(context);
+
+        return statement;
+    }
+
+    private String resolveAuthnClass(Set<AuthenticationMethod> authenticationMethods, Set<NetworkControl> networkControls) {
+        if (authenticationMethods.contains(AuthenticationMethod.PASSWORD)) {
+            if (networkControls.contains(NetworkControl.IP_FILTER)) {
+                return AuthnContext.IP_PASSWORD_AUTHN_CTX;
+            } else if (networkControls.contains(NetworkControl.TLS)) {
+                return AuthnContext.PPT_AUTHN_CTX;
+            } else {
+                return AuthnContext.PASSWORD_AUTHN_CTX;
+            }
+        } else if (authenticationMethods.contains(AuthenticationMethod.KERBEROS)) {
+            return AuthnContext.KERBEROS_AUTHN_CTX;
+        } else if (authenticationMethods.contains(AuthenticationMethod.TLS_CLIENT_AUTH) && networkControls.contains(NetworkControl.TLS)) {
+            return AuthnContext.TLS_CLIENT_AUTHN_CTX;
+        } else if (authenticationMethods.contains(AuthenticationMethod.PRIOR_SESSION)) {
+            return AuthnContext.PREVIOUS_SESSION_AUTHN_CTX;
+        } else if (networkControls.contains(NetworkControl.IP_FILTER)) {
+            return AuthnContext.IP_AUTHN_CTX;
+        } else {
+            return AuthnContext.UNSPECIFIED_AUTHN_CTX;
+        }
+    }
+
+    private AttributeStatement buildAttributes(UserServiceAuthentication user) {
+        final SamlServiceProvider serviceProvider = user.getServiceProvider();
+        final AttributeStatement statement = samlFactory.object(AttributeStatement.class, AttributeStatement.DEFAULT_ELEMENT_NAME);
+        final List<Attribute> attributes = new ArrayList<>();
+        final Attribute roles = buildAttribute(serviceProvider.getAttributeNames().roles, "roles", user.getRoles());
+        if (roles != null) {
+            attributes.add(roles);
+        }
+        final Attribute principal = buildAttribute(serviceProvider.getAttributeNames().principal, "principal", user.getPrincipal());
+        if (principal != null) {
+            attributes.add(principal);
+        }
+        final Attribute email = buildAttribute(serviceProvider.getAttributeNames().email, "email", user.getEmail());
+        if (email != null) {
+            attributes.add(email);
+        }
+        final Attribute name = buildAttribute(serviceProvider.getAttributeNames().name, "name", user.getName());
+        if (name != null) {
+            attributes.add(name);
+        }
+        if (attributes.isEmpty()) {
+            return null;
+        }
+        statement.getAttributes().addAll(attributes);
+        return statement;
+    }
+
+    private Attribute buildAttribute(String formalName, String friendlyName, String value) {
+        if (Strings.isNullOrEmpty(value)) {
+            return null;
+        }
+        return buildAttribute(formalName, friendlyName, List.of(value));
+    }
+
+    private Attribute buildAttribute(String formalName, String friendlyName, Collection<String> values) {
+        if (values.isEmpty() || Strings.isNullOrEmpty(formalName)) {
+            return null;
+        }
+        final Attribute attribute = samlFactory.object(Attribute.class, Attribute.DEFAULT_ELEMENT_NAME);
+        attribute.setName(formalName);
+        attribute.setFriendlyName(friendlyName);
+        attribute.setNameFormat(Attribute.URI_REFERENCE);
+        for (String val : values) {
+            final XSString string = samlFactory.object(XSString.class, AttributeValue.DEFAULT_ELEMENT_NAME, XSString.TYPE_NAME);
+            string.setValue(val);
+            attribute.getAttributeValues().add(string);
+        }
+        return attribute;
+    }
+
+    private Issuer buildIssuer() {
+        final Issuer issuer = samlFactory.object(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(this.idp.getEntityId());
+        return issuer;
+    }
+
+    private Status buildStatus() {
+        final StatusCode code = samlFactory.object(StatusCode.class, StatusCode.DEFAULT_ELEMENT_NAME);
+        code.setValue(StatusCode.SUCCESS);
+
+        final Status status = samlFactory.object(Status.class, Status.DEFAULT_ELEMENT_NAME);
+        status.setStatusCode(code);
+
+        return status;
+    }
+
+    private NameID buildNameId(UserServiceAuthentication user, @Nullable SamlAuthenticationState authnState) {
+        final SamlServiceProvider serviceProvider = user.getServiceProvider();
+        final NameID nameID = samlFactory.object(NameID.class, NameID.DEFAULT_ELEMENT_NAME);
+        final String nameIdFormat;
+        if (authnState != null && authnState.getRequestedNameidFormat() != null) {
+            nameIdFormat = authnState.getRequestedNameidFormat();
+        } else {
+            nameIdFormat = serviceProvider.getAllowedNameIdFormat() != null ? serviceProvider.getAllowedNameIdFormat() :
+                idp.getServiceProviderDefaults().nameIdFormat;
+        }
+        nameID.setFormat(nameIdFormat);
+        nameID.setValue(getNameIdValueForFormat(nameIdFormat, user));
+        return nameID;
+    }
+
+    private String getNameIdValueForFormat(String format, UserServiceAuthentication user) {
+        switch (format) {
+            case TRANSIENT:
+                // See SAML 2.0 Core 8.3.8 & 1.3.4
+                return samlFactory.secureIdentifier();
+            default:
+                throw new IllegalStateException("Unsupported NameID Format: " + format);
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/UserServiceAuthentication.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/UserServiceAuthentication.java
new file mode 100644
index 0000000000000..4223b26810d55
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/authn/UserServiceAuthentication.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.elasticsearch.xpack.idp.authc.AuthenticationMethod;
+import org.elasticsearch.xpack.idp.authc.NetworkControl;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+
+import java.util.Set;
+
+/**
+ * Lightweight representation of a user that has authenticated to the IdP in the context of a specific service provider
+ */
+public class UserServiceAuthentication {
+
+    private final String principal;
+    private final String name;
+    private final String email;
+    private final Set<String> roles;
+
+    private final SamlServiceProvider serviceProvider;
+    private final Set<AuthenticationMethod> authenticationMethods;
+    private final Set<NetworkControl> networkControls;
+
+    public UserServiceAuthentication(String principal, String name, String email, Set<String> roles,
+                                     SamlServiceProvider serviceProvider,
+                                     Set<AuthenticationMethod> authenticationMethods, Set<NetworkControl> networkControls) {
+        this.principal = principal;
+        this.name = name;
+        this.email = email;
+        this.roles = Set.copyOf(roles);
+        this.serviceProvider = serviceProvider;
+        this.authenticationMethods = authenticationMethods;
+        this.networkControls = networkControls;
+    }
+
+    public UserServiceAuthentication(String principal, String name, String email, Set<String> roles, SamlServiceProvider serviceProvider) {
+        this(principal, name, email, roles, serviceProvider, Set.of(AuthenticationMethod.PASSWORD), Set.of(NetworkControl.TLS));
+    }
+
+    public String getPrincipal() {
+        return principal;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getEmail() {
+        return email;
+    }
+
+    public Set<String> getRoles() {
+        return roles;
+    }
+
+    public SamlServiceProvider getServiceProvider() {
+        return serviceProvider;
+    }
+
+    public Set<AuthenticationMethod> getAuthenticationMethods() {
+        return authenticationMethods;
+    }
+
+    public Set<NetworkControl> getNetworkControls() {
+        return networkControls;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{" +
+            "principal='" + principal + '\'' +
+            ", name='" + name + '\'' +
+            ", email='" + email + '\'' +
+            ", roles=" + roles +
+            ", serviceProvider=" + serviceProvider +
+            ", authenticationMethods=" + authenticationMethods +
+            ", networkControls=" + networkControls +
+            '}';
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdPMetadataBuilder.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdPMetadataBuilder.java
new file mode 100644
index 0000000000000..ac60f6f906e05
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdPMetadataBuilder.java
@@ -0,0 +1,264 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.elasticsearch.common.Strings;
+import org.opensaml.saml.common.xml.SAMLConstants;
+import org.opensaml.saml.saml2.metadata.ContactPerson;
+import org.opensaml.saml.saml2.metadata.EmailAddress;
+import org.opensaml.saml.saml2.metadata.EntityDescriptor;
+import org.opensaml.saml.saml2.metadata.GivenName;
+import org.opensaml.saml.saml2.metadata.IDPSSODescriptor;
+import org.opensaml.saml.saml2.metadata.KeyDescriptor;
+import org.opensaml.saml.saml2.metadata.NameIDFormat;
+import org.opensaml.saml.saml2.metadata.Organization;
+import org.opensaml.saml.saml2.metadata.OrganizationDisplayName;
+import org.opensaml.saml.saml2.metadata.OrganizationName;
+import org.opensaml.saml.saml2.metadata.OrganizationURL;
+import org.opensaml.saml.saml2.metadata.SingleLogoutService;
+import org.opensaml.saml.saml2.metadata.SingleSignOnService;
+import org.opensaml.saml.saml2.metadata.SurName;
+import org.opensaml.saml.saml2.metadata.impl.ContactPersonBuilder;
+import org.opensaml.saml.saml2.metadata.impl.EmailAddressBuilder;
+import org.opensaml.saml.saml2.metadata.impl.EntityDescriptorBuilder;
+import org.opensaml.saml.saml2.metadata.impl.GivenNameBuilder;
+import org.opensaml.saml.saml2.metadata.impl.IDPSSODescriptorBuilder;
+import org.opensaml.saml.saml2.metadata.impl.KeyDescriptorBuilder;
+import org.opensaml.saml.saml2.metadata.impl.NameIDFormatBuilder;
+import org.opensaml.saml.saml2.metadata.impl.OrganizationBuilder;
+import org.opensaml.saml.saml2.metadata.impl.OrganizationDisplayNameBuilder;
+import org.opensaml.saml.saml2.metadata.impl.OrganizationNameBuilder;
+import org.opensaml.saml.saml2.metadata.impl.OrganizationURLBuilder;
+import org.opensaml.saml.saml2.metadata.impl.SingleLogoutServiceBuilder;
+import org.opensaml.saml.saml2.metadata.impl.SingleSignOnServiceBuilder;
+import org.opensaml.saml.saml2.metadata.impl.SurNameBuilder;
+import org.opensaml.security.credential.UsageType;
+import org.opensaml.xmlsec.keyinfo.KeyInfoSupport;
+import org.opensaml.xmlsec.signature.KeyInfo;
+import org.opensaml.xmlsec.signature.impl.KeyInfoBuilder;
+
+import java.net.URL;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+public class SamlIdPMetadataBuilder {
+
+    private Locale locale;
+    private final String entityId;
+    private Set<String> nameIdFormats;
+    private boolean wantAuthnRequestsSigned;
+    private Map<String, URL> singleSignOnServiceUrls = new HashMap<>();
+    private Map<String, URL> singleLogoutServiceUrls = new HashMap<>();
+    private List<X509Certificate> signingCertificates;
+    private SamlIdentityProvider.OrganizationInfo organization;
+    private final List<SamlIdentityProvider.ContactInfo> contacts;
+
+
+    public SamlIdPMetadataBuilder(String entityId) {
+        this.entityId = entityId;
+        this.locale = Locale.getDefault();
+        this.contacts = new ArrayList<>();
+        this.nameIdFormats = new HashSet<>();
+        this.signingCertificates = new ArrayList<>();
+        this.wantAuthnRequestsSigned = false;
+    }
+
+    public SamlIdPMetadataBuilder withLocale(Locale locale) {
+        this.locale = locale;
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withNameIdFormat(String nameIdFormat) {
+        if (Strings.isNullOrEmpty(nameIdFormat) == false) {
+            this.nameIdFormats.add(nameIdFormat);
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder wantAuthnRequestsSigned(boolean wants) {
+        this.wantAuthnRequestsSigned = wants;
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withSingleSignOnServiceUrl(String binding, URL url) {
+        if ( null != url) {
+            this.singleSignOnServiceUrls.put(binding, url);
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withSingleLogoutServiceUrl(String binding, URL url) {
+        if (null != url) {
+            this.singleLogoutServiceUrls.put(binding, url);
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withSigningCertificates(List<X509Certificate> signingCertificates) {
+        if (null != signingCertificates) {
+            this.signingCertificates.addAll(signingCertificates);
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withSigningCertificate(X509Certificate signingCertificate) {
+        if ( null != signingCertificate ) {
+            return withSigningCertificates(Collections.singletonList(signingCertificate));
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder organization(SamlIdentityProvider.OrganizationInfo organization) {
+        if (null != organization) {
+            this.organization = organization;
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder organization(String orgName, String displayName, String url) {
+        return organization(new SamlIdentityProvider.OrganizationInfo(orgName, displayName, url));
+    }
+
+    public SamlIdPMetadataBuilder withContact(SamlIdentityProvider.ContactInfo contact) {
+        if (null != contact) {
+            this.contacts.add(contact);
+        }
+        return this;
+    }
+
+    public SamlIdPMetadataBuilder withContact(String type, String givenName, String surName, String email) {
+        return withContact(new SamlIdentityProvider.ContactInfo(SamlIdentityProvider.ContactInfo.getType(type), givenName, surName, email));
+    }
+
+    public EntityDescriptor build() throws CertificateEncodingException {
+        final IDPSSODescriptor idpSsoDescriptor = new IDPSSODescriptorBuilder().buildObject();
+        idpSsoDescriptor.removeAllSupportedProtocols();
+        idpSsoDescriptor.addSupportedProtocol(SAMLConstants.SAML20P_NS);
+        idpSsoDescriptor.setWantAuthnRequestsSigned(this.wantAuthnRequestsSigned);
+        if (nameIdFormats.isEmpty() == false) {
+            idpSsoDescriptor.getNameIDFormats().addAll(buildNameIDFormats());
+        }
+        if (singleSignOnServiceUrls.isEmpty() == false) {
+            idpSsoDescriptor.getSingleSignOnServices().addAll(buildSingleSignOnServices());
+        }
+        if (singleLogoutServiceUrls.isEmpty() == false) {
+            idpSsoDescriptor.getSingleLogoutServices().addAll(buildSingleLogoutServices());
+        }
+        idpSsoDescriptor.getKeyDescriptors().addAll(buildKeyDescriptors());
+
+        final EntityDescriptor descriptor = new EntityDescriptorBuilder().buildObject();
+        descriptor.setEntityID(this.entityId);
+        descriptor.getRoleDescriptors().add(idpSsoDescriptor);
+        if (organization != null) {
+            descriptor.setOrganization(buildOrganization());
+        }
+        for (SamlIdentityProvider.ContactInfo contact : contacts) {
+            descriptor.getContactPersons().add(buildContact(contact));
+        }
+
+        return descriptor;
+    }
+
+    private List<SingleSignOnService> buildSingleSignOnServices() {
+        List<SingleSignOnService> ssoServices = new ArrayList<>();
+        if (singleSignOnServiceUrls.isEmpty()) {
+            throw new IllegalStateException("At least one SingleSignOnService URL should be specified");
+        }
+        for (Map.Entry<String, URL> entry : singleSignOnServiceUrls.entrySet()) {
+            final SingleSignOnService sso = new SingleSignOnServiceBuilder().buildObject();
+            sso.setBinding(entry.getKey());
+            sso.setLocation(entry.getValue().toString());
+            ssoServices.add(sso);
+        }
+        return ssoServices;
+    }
+
+    private List<SingleLogoutService> buildSingleLogoutServices() {
+        List<SingleLogoutService> sloServices = new ArrayList<>();
+        for (Map.Entry<String, URL> entry : singleLogoutServiceUrls.entrySet()) {
+            final SingleLogoutService slo = new SingleLogoutServiceBuilder().buildObject();
+            slo.setBinding(entry.getKey());
+            slo.setLocation(entry.getValue().toString());
+            sloServices.add(slo);
+        }
+        return sloServices;
+    }
+
+    private List<NameIDFormat> buildNameIDFormats() {
+        List<NameIDFormat> formats = new ArrayList();
+        if (nameIdFormats.isEmpty()) {
+            throw new IllegalStateException("NameID format has not been specified");
+        }
+        for (String nameIdFormat : nameIdFormats) {
+            final NameIDFormat format = new NameIDFormatBuilder().buildObject();
+            format.setFormat(nameIdFormat);
+            formats.add(format);
+
+        }
+        return formats;
+    }
+
+    private List<? extends KeyDescriptor> buildKeyDescriptors() throws CertificateEncodingException {
+        if (signingCertificates.isEmpty()) {
+            return Collections.emptyList();
+        }
+        List<KeyDescriptor> keys = new ArrayList<>();
+        for (X509Certificate signingCertificate : signingCertificates) {
+            if (signingCertificate != null) {
+                final KeyDescriptor descriptor = new KeyDescriptorBuilder().buildObject();
+                descriptor.setUse(UsageType.SIGNING);
+                final KeyInfo keyInfo = new KeyInfoBuilder().buildObject();
+                KeyInfoSupport.addCertificate(keyInfo, signingCertificate);
+                descriptor.setKeyInfo(keyInfo);
+                keys.add(descriptor);
+            }
+        }
+        return keys;
+    }
+
+    private Organization buildOrganization() {
+        final String lang = locale.toLanguageTag();
+        final OrganizationName name = new OrganizationNameBuilder().buildObject();
+        name.setValue(this.organization.organizationName);
+        name.setXMLLang(lang);
+        final OrganizationDisplayName displayName = new OrganizationDisplayNameBuilder().buildObject();
+        displayName.setValue(this.organization.displayName);
+        displayName.setXMLLang(lang);
+        final OrganizationURL url = new OrganizationURLBuilder().buildObject();
+        url.setValue(this.organization.url);
+        url.setXMLLang(lang);
+
+        final Organization org = new OrganizationBuilder().buildObject();
+        org.getOrganizationNames().add(name);
+        org.getDisplayNames().add(displayName);
+        org.getURLs().add(url);
+        return org;
+    }
+
+    private ContactPerson buildContact(SamlIdentityProvider.ContactInfo contact) {
+        final GivenName givenName = new GivenNameBuilder().buildObject();
+        givenName.setName(contact.givenName);
+        final SurName surName = new SurNameBuilder().buildObject();
+        surName.setName(contact.surName);
+        final EmailAddress email = new EmailAddressBuilder().buildObject();
+        email.setAddress(contact.email);
+
+        final ContactPerson person = new ContactPersonBuilder().buildObject();
+        person.setType(contact.type);
+        person.setGivenName(givenName);
+        person.setSurName(surName);
+        person.getEmailAddresses().add(email);
+        return person;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProvider.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProvider.java
new file mode 100644
index 0000000000000..037894cd3b685
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProvider.java
@@ -0,0 +1,208 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.idp;
+
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.opensaml.saml.saml2.metadata.ContactPersonTypeEnumeration;
+import org.opensaml.security.x509.X509Credential;
+
+import java.net.URL;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * SAML 2.0 configuration information about this IdP
+ */
+public class SamlIdentityProvider {
+
+    private final Logger logger = LogManager.getLogger();
+
+    private final String entityId;
+    private final Map<String, URL> ssoEndpoints;
+    private final Map<String, URL> sloEndpoints;
+    private final Set<String> allowedNameIdFormats;
+    private final ServiceProviderDefaults serviceProviderDefaults;
+    private final X509Credential signingCredential;
+    private final SamlServiceProviderResolver serviceProviderResolver;
+    private final X509Credential metadataSigningCredential;
+    private ContactInfo technicalContact;
+    private OrganizationInfo organization;
+
+    // Package access - use Builder instead
+    SamlIdentityProvider(String entityId, Map<String, URL> ssoEndpoints, Map<String, URL> sloEndpoints, Set<String> allowedNameIdFormats,
+                         X509Credential signingCredential, X509Credential metadataSigningCredential,
+                         ContactInfo technicalContact, OrganizationInfo organization,
+                         ServiceProviderDefaults serviceProviderDefaults, SamlServiceProviderResolver serviceProviderResolver) {
+        this.entityId = entityId;
+        this.ssoEndpoints = ssoEndpoints;
+        this.sloEndpoints = sloEndpoints;
+        this.allowedNameIdFormats = allowedNameIdFormats;
+        this.signingCredential = signingCredential;
+        this.serviceProviderDefaults = serviceProviderDefaults;
+        this.metadataSigningCredential = metadataSigningCredential;
+        this.technicalContact = technicalContact;
+        this.organization = organization;
+        this.serviceProviderResolver = serviceProviderResolver;
+    }
+
+    public static SamlIdentityProviderBuilder builder(SamlServiceProviderResolver resolver) {
+        return new SamlIdentityProviderBuilder(resolver);
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    public URL getSingleSignOnEndpoint(String binding) {
+        return ssoEndpoints.get(binding);
+    }
+
+    public URL getSingleLogoutEndpoint(String binding) {
+        return sloEndpoints.get(binding);
+    }
+
+    public Set<String> getAllowedNameIdFormats() {
+        return allowedNameIdFormats;
+    }
+
+    public X509Credential getSigningCredential() {
+        return signingCredential;
+    }
+
+    public X509Credential getMetadataSigningCredential() {
+        return metadataSigningCredential;
+    }
+
+    public OrganizationInfo getOrganization() {
+        return organization;
+    }
+
+    public ContactInfo getTechnicalContact() {
+        return technicalContact;
+    }
+
+    public ServiceProviderDefaults getServiceProviderDefaults() {
+        return serviceProviderDefaults;
+    }
+
+    /**
+     * Asynchronously lookup the specified {@link SamlServiceProvider} by entity-id.
+     * @param allowDisabled whether to return service providers that are not {@link SamlServiceProvider#isEnabled() enabled}.
+     *                      For security reasons, callers should typically avoid working with disabled service providers.
+     * @param listener Responds with the requested Service Provider object, or {@code null} if no such SP exists.
+     *                 {@link ActionListener#onFailure} is only used for fatal errors (e.g. being unable to access
+     *                 the backing store (elasticsearch index) that hold the SP data).
+     */
+    public void getRegisteredServiceProvider(String spEntityId, boolean allowDisabled, ActionListener<SamlServiceProvider> listener) {
+        serviceProviderResolver.resolve(spEntityId, ActionListener.wrap(
+            sp -> {
+                if (sp == null) {
+                    logger.info("No service provider exists for entityId [{}]", spEntityId);
+                    listener.onResponse(null);
+                } else if (allowDisabled == false && sp.isEnabled() == false) {
+                    logger.info("Service provider [{}][{}] is not enabled", sp.getEntityId(), sp.getName());
+                    listener.onResponse(null);
+                } else {
+                    logger.debug("Service provider for [{}] is [{}]", sp.getEntityId(), sp);
+                    listener.onResponse(sp);
+                }
+            },
+            listener::onFailure
+        ));
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final SamlIdentityProvider that = (SamlIdentityProvider) o;
+        return Objects.equals(entityId, that.entityId) &&
+            Objects.equals(ssoEndpoints, that.ssoEndpoints) &&
+            Objects.equals(sloEndpoints, that.sloEndpoints) &&
+            Objects.equals(allowedNameIdFormats, that.allowedNameIdFormats) &&
+            Objects.equals(signingCredential, that.signingCredential) &&
+            Objects.equals(metadataSigningCredential, that.metadataSigningCredential) &&
+            Objects.equals(technicalContact, that.technicalContact) &&
+            Objects.equals(organization, that.organization);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(entityId);
+    }
+
+    public static class ContactInfo {
+        static final Map<String, ContactPersonTypeEnumeration> TYPES = Collections.unmodifiableMap(
+            MapBuilder.newMapBuilder(new LinkedHashMap<String, ContactPersonTypeEnumeration>())
+                .put(ContactPersonTypeEnumeration.ADMINISTRATIVE.toString(), ContactPersonTypeEnumeration.ADMINISTRATIVE)
+                .put(ContactPersonTypeEnumeration.BILLING.toString(), ContactPersonTypeEnumeration.BILLING)
+                .put(ContactPersonTypeEnumeration.SUPPORT.toString(), ContactPersonTypeEnumeration.SUPPORT)
+                .put(ContactPersonTypeEnumeration.TECHNICAL.toString(), ContactPersonTypeEnumeration.TECHNICAL)
+                .put(ContactPersonTypeEnumeration.OTHER.toString(), ContactPersonTypeEnumeration.OTHER)
+                .map());
+
+        public final ContactPersonTypeEnumeration type;
+        public final String givenName;
+        public final String surName;
+        public final String email;
+
+        public ContactInfo(ContactPersonTypeEnumeration type, String givenName, String surName, String email) {
+            this.type = Objects.requireNonNull(type, "Contact Person Type is required");
+            this.givenName = givenName;
+            this.surName = surName;
+            this.email = Objects.requireNonNull(email, "Contact Person email is required");
+        }
+
+        public static ContactPersonTypeEnumeration getType(String name) {
+            final ContactPersonTypeEnumeration type = TYPES.get(name.toLowerCase(Locale.ROOT));
+            if (type == null) {
+                throw new IllegalArgumentException("Invalid contact type " + name + " allowed values are "
+                    + Strings.collectionToCommaDelimitedString(TYPES.keySet()));
+            }
+            return type;
+        }
+    }
+
+    public static class OrganizationInfo {
+        public final String organizationName;
+        public final String displayName;
+        public final String url;
+
+        public OrganizationInfo(String organizationName, String displayName, String url) {
+            this.organizationName = organizationName;
+            this.displayName = displayName;
+            this.url = url;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            OrganizationInfo that = (OrganizationInfo) o;
+            return Objects.equals(organizationName, that.organizationName) &&
+                Objects.equals(displayName, that.displayName) &&
+                Objects.equals(url, that.url);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(organizationName, displayName, url);
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java
new file mode 100644
index 0000000000000..6ddbf533b9904
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java
@@ -0,0 +1,356 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.X509KeyPairSettings;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.opensaml.saml.saml2.metadata.ContactPersonTypeEnumeration;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.security.x509.impl.X509KeyManagerX509CredentialAdapter;
+
+import javax.net.ssl.X509KeyManager;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.security.PrivateKey;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+/**
+ * Builds a {@link SamlIdentityProvider} instance, from either direct properties, or defined node {@link Settings}.
+ */
+public class SamlIdentityProviderBuilder {
+
+    private static final List<String> ALLOWED_NAMEID_FORMATS = List.of(TRANSIENT);
+    public static final Setting<String> IDP_ENTITY_ID = Setting.simpleString("xpack.idp.entity_id", Setting.Property.NodeScope);
+
+    public static final Setting<URL> IDP_SSO_REDIRECT_ENDPOINT = new Setting<>("xpack.idp.sso_endpoint.redirect", "https:",
+        value -> parseUrl("xpack.idp.sso_endpoint.redirect", value), Setting.Property.NodeScope);
+    public static final Setting<URL> IDP_SSO_POST_ENDPOINT = new Setting<>("xpack.idp.sso_endpoint.post", "https:",
+        value -> parseUrl("xpack.idp.sso_endpoint.post", value), Setting.Property.NodeScope);
+    public static final Setting<URL> IDP_SLO_REDIRECT_ENDPOINT = new Setting<>("xpack.idp.slo_endpoint.redirect", "https:",
+        value -> parseUrl("xpack.idp.slo_endpoint.redirect", value), Setting.Property.NodeScope);
+    public static final Setting<URL> IDP_SLO_POST_ENDPOINT = new Setting<>("xpack.idp.slo_endpoint.post", "https:",
+        value -> parseUrl("xpack.idp.slo_endpoint.post", value), Setting.Property.NodeScope);
+    public static final Setting<List<String>> IDP_ALLOWED_NAMEID_FORMATS = Setting.listSetting("xpack.idp.allowed_nameid_formats",
+        List.of(TRANSIENT), Function.identity(), SamlIdentityProviderBuilder::validateNameIDs, Setting.Property.NodeScope);
+
+    public static final Setting<String> IDP_SIGNING_KEY_ALIAS = Setting.simpleString("xpack.idp.signing.keystore.alias",
+        Setting.Property.NodeScope);
+    public static final Setting<String> IDP_METADATA_SIGNING_KEY_ALIAS = Setting.simpleString("xpack.idp.metadata.signing.keystore.alias",
+        Setting.Property.NodeScope);
+
+    public static final Setting<String> IDP_ORGANIZATION_NAME = Setting.simpleString("xpack.idp.organization.name",
+        Setting.Property.NodeScope);
+    public static final Setting<String> IDP_ORGANIZATION_DISPLAY_NAME = Setting.simpleString("xpack.idp.organization.display_name",
+        IDP_ORGANIZATION_NAME, Setting.Property.NodeScope);
+    public static final Setting<URL> IDP_ORGANIZATION_URL = new Setting<>("xpack.idp.organization.url", "http:",
+        value -> parseUrl("xpack.idp.organization.url", value), Setting.Property.NodeScope);
+
+    public static final Setting<String> IDP_CONTACT_GIVEN_NAME = Setting.simpleString("xpack.idp.contact.given_name",
+        Setting.Property.NodeScope);
+    public static final Setting<String> IDP_CONTACT_SURNAME = Setting.simpleString("xpack.idp.contact.surname",
+        Setting.Property.NodeScope);
+    public static final Setting<String> IDP_CONTACT_EMAIL = Setting.simpleString("xpack.idp.contact.email", Setting.Property.NodeScope);
+
+    private final SamlServiceProviderResolver serviceProviderResolver;
+
+    private String entityId;
+    private Map<String, URL> ssoEndpoints;
+    private Map<String, URL> sloEndpoints;
+    private Set<String> allowedNameIdFormats;
+    private X509Credential signingCredential;
+    private X509Credential metadataSigningCredential;
+    private SamlIdentityProvider.ContactInfo technicalContact;
+    private SamlIdentityProvider.OrganizationInfo organization;
+    private ServiceProviderDefaults serviceProviderDefaults;
+
+    SamlIdentityProviderBuilder(SamlServiceProviderResolver serviceProviderResolver) {
+        this.serviceProviderResolver = serviceProviderResolver;
+        this.ssoEndpoints = new HashMap<>();
+        this.sloEndpoints = new HashMap<>();
+    }
+
+    public SamlIdentityProvider build() throws ValidationException {
+        ValidationException ex = new ValidationException();
+
+        if (Strings.isNullOrEmpty(entityId)) {
+            ex.addValidationError("IDP Entity ID must be set (was [" + entityId + "])");
+        }
+
+        if (ssoEndpoints == null || ssoEndpoints.containsKey(SAML2_REDIRECT_BINDING_URI) == false) {
+            ex.addValidationError("The redirect ([ " + SAML2_REDIRECT_BINDING_URI + "]) SSO binding is required");
+        }
+
+        if (signingCredential == null) {
+            ex.addValidationError("Signing credential must be specified");
+        } else {
+            try {
+                validateSigningKey(signingCredential.getPrivateKey());
+            } catch (ElasticsearchSecurityException e) {
+                ex.addValidationError("Signing credential is invalid - " + e.getMessage());
+            }
+        }
+
+        if (metadataSigningCredential != null) {
+            try {
+                validateSigningKey(metadataSigningCredential.getPrivateKey());
+            } catch (ElasticsearchSecurityException e) {
+                ex.addValidationError("Metadata signing credential is invalid - " + e.getMessage());
+            }
+        }
+
+        if (serviceProviderDefaults == null) {
+            ex.addValidationError("Service provider defaults must be specified");
+        }
+
+        if (ex.validationErrors().isEmpty() == false) {
+            throw ex;
+        }
+
+        return new SamlIdentityProvider(
+            entityId,
+            Map.copyOf(ssoEndpoints),
+            sloEndpoints == null ? Map.of() : Map.copyOf(sloEndpoints),
+            Set.copyOf(allowedNameIdFormats),
+            signingCredential, metadataSigningCredential,
+            technicalContact, organization,
+            serviceProviderDefaults,
+            serviceProviderResolver);
+    }
+
+    public SamlIdentityProviderBuilder fromSettings(Environment env) {
+        final Settings settings = env.settings();
+        this.entityId = require(settings, IDP_ENTITY_ID);
+        this.ssoEndpoints = new HashMap<>();
+        this.sloEndpoints = new HashMap<>();
+        this.ssoEndpoints.put(SAML2_REDIRECT_BINDING_URI, requiredUrl(settings, IDP_SSO_REDIRECT_ENDPOINT));
+        if (IDP_SSO_POST_ENDPOINT.exists(settings)) {
+            this.ssoEndpoints.put(SAML2_POST_BINDING_URI, IDP_SSO_POST_ENDPOINT.get(settings));
+        }
+        if (IDP_SLO_POST_ENDPOINT.exists(settings)) {
+            this.sloEndpoints.put(SAML2_POST_BINDING_URI, IDP_SLO_POST_ENDPOINT.get(settings));
+        }
+        if (IDP_SLO_REDIRECT_ENDPOINT.exists(settings)) {
+            this.sloEndpoints.put(SAML2_REDIRECT_BINDING_URI, IDP_SLO_REDIRECT_ENDPOINT.get(settings));
+        }
+        this.allowedNameIdFormats = new HashSet<>(IDP_ALLOWED_NAMEID_FORMATS.get(settings));
+        this.signingCredential = buildSigningCredential(env, settings, "xpack.idp.signing.");
+        this.metadataSigningCredential = buildSigningCredential(env, settings, "xpack.idp.metadata_signing.");
+        this.technicalContact = buildContactInfo(settings);
+        this.organization = buildOrganization(settings);
+        return this;
+    }
+
+    public static List<? extends Setting<?>> getSettings() {
+        return List.of(
+            IDP_ENTITY_ID,
+            IDP_SLO_REDIRECT_ENDPOINT,
+            IDP_SLO_POST_ENDPOINT,
+            IDP_SSO_REDIRECT_ENDPOINT,
+            IDP_SSO_POST_ENDPOINT,
+            IDP_ALLOWED_NAMEID_FORMATS,
+            IDP_SIGNING_KEY_ALIAS,
+            IDP_METADATA_SIGNING_KEY_ALIAS,
+            IDP_ORGANIZATION_NAME,
+            IDP_ORGANIZATION_DISPLAY_NAME,
+            IDP_ORGANIZATION_URL,
+            IDP_CONTACT_GIVEN_NAME,
+            IDP_CONTACT_SURNAME,
+            IDP_CONTACT_EMAIL);
+    }
+
+    public SamlIdentityProviderBuilder serviceProviderDefaults(ServiceProviderDefaults serviceProviderDefaults) {
+        this.serviceProviderDefaults = serviceProviderDefaults;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder entityId(String entityId) {
+        this.entityId = entityId;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder singleSignOnEndpoints(Map<String, URL> ssoEndpoints) {
+        this.ssoEndpoints = ssoEndpoints;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder singleLogoutEndpoints(Map<String, URL> sloEndpoints) {
+        this.sloEndpoints = sloEndpoints;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder singleSignOnEndpoint(String binding, URL endpoint) {
+        this.ssoEndpoints.put(binding, endpoint);
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder singleLogoutEndpoint(String binding, URL endpoint) {
+        this.sloEndpoints.put(binding, endpoint);
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder allowedNameIdFormat(String nameIdFormat) {
+        this.allowedNameIdFormats.add(nameIdFormat);
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder signingCredential(X509Credential signingCredential) {
+        this.signingCredential = signingCredential;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder metadataSigningCredential(X509Credential metadataSigningCredential) {
+        this.metadataSigningCredential = metadataSigningCredential;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder technicalContact(SamlIdentityProvider.ContactInfo technicalContact) {
+        this.technicalContact = technicalContact;
+        return this;
+    }
+
+    public SamlIdentityProviderBuilder organization(SamlIdentityProvider.OrganizationInfo organization) {
+        this.organization = organization;
+        return this;
+    }
+
+    private static URL parseUrl(String key, String value) {
+        try {
+            return new URL(value);
+        } catch (MalformedURLException e) {
+            throw new IllegalArgumentException("Invalid value [" + value + "] for [" + key + "]. Not a valid URL", e);
+        }
+    }
+
+    private static void validateNameIDs(List<String> values) {
+        final Set<String> invalidFormats =
+            values.stream().distinct().filter(e -> ALLOWED_NAMEID_FORMATS.contains(e) == false).collect(Collectors.toSet());
+        if (invalidFormats.size() > 0) {
+            throw new IllegalArgumentException(
+                invalidFormats + " are not valid NameID formats. Allowed values are " + ALLOWED_NAMEID_FORMATS);
+        }
+    }
+
+    static String require(Settings settings, Setting<String> setting) {
+        if (settings.hasValue(setting.getKey())) {
+            return setting.get(settings);
+        } else {
+            throw new IllegalArgumentException("The configuration setting [" + setting.getKey() + "] is required");
+        }
+    }
+
+    static URL requiredUrl(Settings settings, Setting<URL> setting) {
+        if (settings.hasValue(setting.getKey())) {
+            return setting.get(settings);
+        } else {
+            throw new IllegalArgumentException("The configuration setting [" + setting.getKey() + "] is required");
+        }
+    }
+
+    // Package protected for testing
+    static X509Credential buildSigningCredential(Environment environment, Settings settings, String prefix) {
+        List<X509Credential> credentials = buildCredentials(environment, settings, prefix, false);
+        if (credentials.isEmpty()) {
+            return null;
+        }
+        return credentials.get(0);
+    }
+
+    static List<X509Credential> buildCredentials(Environment env, Settings settings, String prefix, boolean allowMultiple) {
+        final X509KeyPairSettings keyPairSettings = X509KeyPairSettings.withPrefix(prefix, false);
+        final X509KeyManager keyManager = CertParsingUtils.getKeyManager(keyPairSettings, settings, null, env);
+        if (keyManager == null) {
+            return Collections.emptyList();
+        }
+
+        final List<X509Credential> credentials = new ArrayList<>();
+        final Set<String> selectedAliases = new HashSet<>();
+        final String configAlias = settings.get(prefix + "keystore.alias");
+        if (Strings.isNullOrEmpty(configAlias)) {
+            final String[] rsaAliases = keyManager.getServerAliases("RSA", null);
+            if (null != rsaAliases) {
+                selectedAliases.addAll(Arrays.asList(rsaAliases));
+            }
+            final String[] ecAliases = keyManager.getServerAliases("EC", null);
+            if (null != ecAliases) {
+                selectedAliases.addAll(Arrays.asList(ecAliases));
+            }
+            if (selectedAliases.isEmpty()) {
+                throw new IllegalArgumentException(
+                    "The configured keystore for [" + prefix + "keystore] does not contain any RSA or EC key pairs.");
+            }
+            if (selectedAliases.size() > 1 && allowMultiple == false) {
+                throw new IllegalArgumentException(
+                    "The configured keystore for [" + prefix + "keystore] contains multiple private key entries, when one was expected.");
+            }
+        } else {
+            selectedAliases.add(configAlias);
+        }
+        for (String alias : selectedAliases) {
+            try {
+                validateSigningKey(keyManager.getPrivateKey(alias));
+            } catch (ElasticsearchSecurityException e) {
+                throw new IllegalArgumentException("The configured credential [" + prefix + "keystore] with alias [" + alias
+                    + "] is not a valid signing key - " + e.getMessage());
+            }
+            credentials.add(new X509KeyManagerX509CredentialAdapter(keyManager, alias));
+        }
+        return credentials;
+    }
+
+    private static void validateSigningKey(PrivateKey privateKey) {
+        if (privateKey == null) {
+            throw new ElasticsearchSecurityException("There is no private key available for this credential");
+        }
+        final String keyType = privateKey.getAlgorithm();
+        if (keyType.equals("RSA") == false && keyType.equals("EC") == false) {
+            throw new ElasticsearchSecurityException("The private key uses unsupported key algorithm type [" + keyType
+                + "], only RSA and EC are supported");
+        }
+    }
+
+    private static SamlIdentityProvider.OrganizationInfo buildOrganization(Settings settings) {
+        final String name = settings.hasValue(IDP_ORGANIZATION_NAME.getKey()) ? IDP_ORGANIZATION_NAME.get(settings) : null;
+        final String displayName = settings.hasValue(IDP_ORGANIZATION_DISPLAY_NAME.getKey()) ?
+            IDP_ORGANIZATION_DISPLAY_NAME.get(settings) : null;
+        final String url = settings.hasValue(IDP_ORGANIZATION_URL.getKey()) ? IDP_ORGANIZATION_URL.get(settings).toString() : null;
+        if (Stream.of(name, displayName, url).allMatch(Objects::isNull) == false) {
+            return new SamlIdentityProvider.OrganizationInfo(name, displayName, url);
+        }
+        return null;
+    }
+
+    private static SamlIdentityProvider.ContactInfo buildContactInfo(Settings settings) {
+        if (settings.hasValue(IDP_CONTACT_EMAIL.getKey())) {
+            return new SamlIdentityProvider.ContactInfo(ContactPersonTypeEnumeration.TECHNICAL,
+                IDP_CONTACT_GIVEN_NAME.get(settings), IDP_CONTACT_SURNAME.get(settings), IDP_CONTACT_EMAIL.get(settings));
+        }
+        return null;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGenerator.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGenerator.java
new file mode 100644
index 0000000000000..a6425e341e366
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGenerator.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.xpack.idp.action.SamlMetadataResponse;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.opensaml.core.xml.io.MarshallingException;
+import org.opensaml.saml.saml2.metadata.EntityDescriptor;
+import org.opensaml.saml.saml2.metadata.impl.EntityDescriptorMarshaller;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.xmlsec.signature.Signature;
+import org.opensaml.xmlsec.signature.support.SignatureException;
+import org.opensaml.xmlsec.signature.support.Signer;
+import org.w3c.dom.Element;
+
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+import static org.opensaml.xmlsec.signature.Signature.DEFAULT_ELEMENT_NAME;
+import static org.opensaml.xmlsec.signature.support.SignatureConstants.ALGO_ID_C14N_EXCL_OMIT_COMMENTS;
+import static org.opensaml.xmlsec.signature.support.SignatureConstants.ALGO_ID_SIGNATURE_RSA_SHA256;
+
+public class SamlMetadataGenerator {
+
+    private final SamlFactory samlFactory;
+    private final SamlIdentityProvider idp;
+    private final Logger logger = LogManager.getLogger(SamlMetadataGenerator.class);
+
+    public SamlMetadataGenerator(SamlFactory samlFactory, SamlIdentityProvider idp) {
+        this.samlFactory = samlFactory;
+        this.idp = idp;
+        SamlInit.initialize();
+    }
+
+    public void generateMetadata(String spEntityId, ActionListener<SamlMetadataResponse> listener) {
+        idp.getRegisteredServiceProvider(spEntityId, true, ActionListener.wrap(
+            sp -> {
+                try {
+                    if (null == sp) {
+                        listener.onFailure(new IllegalArgumentException("Service provider with Entity ID [" + spEntityId
+                            + "] is not registered with this Identity Provider"));
+                        return;
+                    }
+                    EntityDescriptor metadata = buildEntityDescriptor(sp);
+                    final X509Credential signingCredential = idp.getMetadataSigningCredential();
+                    Element metadataElement = possiblySignDescriptor(metadata, signingCredential);
+                    listener.onResponse(new SamlMetadataResponse(samlFactory.toString(metadataElement, false)));
+                } catch (Exception e) {
+                    logger.debug("Error generating IDP metadata to share with [" + spEntityId + "]", e);
+                    listener.onFailure(e);
+                }
+            },
+            listener::onFailure
+        ));
+    }
+
+    EntityDescriptor buildEntityDescriptor(SamlServiceProvider sp) throws Exception {
+        final SamlIdPMetadataBuilder builder = new SamlIdPMetadataBuilder(idp.getEntityId())
+            .wantAuthnRequestsSigned(sp.shouldSignAuthnRequests())
+            .withSingleSignOnServiceUrl(SAML2_REDIRECT_BINDING_URI,
+                idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI))
+            .withSingleSignOnServiceUrl(SAML2_POST_BINDING_URI,
+                idp.getSingleSignOnEndpoint(SAML2_POST_BINDING_URI))
+            .withSingleLogoutServiceUrl(SAML2_REDIRECT_BINDING_URI,
+                idp.getSingleLogoutEndpoint(SAML2_REDIRECT_BINDING_URI))
+            .withSingleLogoutServiceUrl(SAML2_POST_BINDING_URI,
+                idp.getSingleLogoutEndpoint(SAML2_POST_BINDING_URI))
+            .withNameIdFormat(PERSISTENT)
+            .withNameIdFormat(TRANSIENT)
+            .organization(idp.getOrganization())
+            .withContact(idp.getTechnicalContact());
+        final X509Credential signingCredential = idp.getSigningCredential();
+        if (null != signingCredential) {
+            builder.withSigningCertificate(signingCredential.getEntityCertificate());
+        }
+        return builder.build();
+    }
+
+    Element possiblySignDescriptor(EntityDescriptor descriptor, X509Credential signingCredential) throws MarshallingException,
+        SignatureException {
+        EntityDescriptorMarshaller marshaller = new EntityDescriptorMarshaller();
+        if (null == signingCredential) {
+            return marshaller.marshall(descriptor);
+        } else {
+            Signature signature = samlFactory.buildObject(Signature.class, DEFAULT_ELEMENT_NAME);
+            signature.setSigningCredential(signingCredential);
+            signature.setSignatureAlgorithm(ALGO_ID_SIGNATURE_RSA_SHA256);
+            signature.setCanonicalizationAlgorithm(ALGO_ID_C14N_EXCL_OMIT_COMMENTS);
+            descriptor.setSignature(signature);
+            Element element = new EntityDescriptorMarshaller().marshall(descriptor);
+            Signer.signObject(signature);
+            return element;
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandler.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandler.java
new file mode 100644
index 0000000000000..0ef8c4b61ac84
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandler.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+
+import java.io.IOException;
+
+public abstract class IdpBaseRestHandler extends BaseRestHandler {
+
+    private static License.OperationMode MINIMUM_ALLOWED_LICENSE = License.OperationMode.ENTERPRISE;
+
+    protected final XPackLicenseState licenseState;
+
+    protected IdpBaseRestHandler(XPackLicenseState licenseState) {
+        this.licenseState = licenseState;
+    }
+
+    protected final RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        RestChannelConsumer consumer = innerPrepareRequest(request, client);
+        if (isIdpFeatureAllowed()) {
+            return consumer;
+        } else {
+            return channel -> channel.sendResponse(new BytesRestResponse(channel,
+                LicenseUtils.newComplianceException("Identity Provider")));
+        }
+    }
+
+    protected boolean isIdpFeatureAllowed() {
+        return licenseState.isAllowedByLicense(MINIMUM_ALLOWED_LICENSE);
+    }
+
+    /**
+     * Implementers should implement this method as they normally would for
+     * {@link BaseRestHandler#prepareRequest(RestRequest, NodeClient)} and ensure that all request
+     * parameters are consumed prior to returning a value.
+     */
+    protected abstract RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException;
+}
+
+
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestDeleteSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestDeleteSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..a270f9409004b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestDeleteSamlServiceProviderAction.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestRequest.Method;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.idp.action.DeleteSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.DeleteSamlServiceProviderRequest;
+import org.elasticsearch.xpack.idp.action.DeleteSamlServiceProviderResponse;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Rest endpoint to remove a service provider (by Entity ID) from the IdP.
+ */
+public class RestDeleteSamlServiceProviderAction extends IdpBaseRestHandler {
+
+    public RestDeleteSamlServiceProviderAction(XPackLicenseState licenseState) {
+        super(licenseState);
+    }
+
+    @Override
+    public String getName() {
+        return "idp_delete_saml_sp_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(Method.DELETE, "/_idp/saml/sp/{sp_entity_id}"));
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        final String entityId = restRequest.param("sp_entity_id");
+        final WriteRequest.RefreshPolicy refresh = restRequest.hasParam("refresh")
+            ? WriteRequest.RefreshPolicy.parse(restRequest.param("refresh")) : WriteRequest.RefreshPolicy.NONE;
+        final DeleteSamlServiceProviderRequest request = new DeleteSamlServiceProviderRequest(entityId, refresh);
+        return channel -> client.execute(DeleteSamlServiceProviderAction.INSTANCE, request,
+            new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(DeleteSamlServiceProviderResponse response, XContentBuilder builder) throws Exception {
+                    response.toXContent(builder, restRequest);
+                    return new BytesRestResponse(response.found() ? RestStatus.OK : RestStatus.NOT_FOUND, builder);
+                }
+            });
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestPutSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestPutSamlServiceProviderAction.java
new file mode 100644
index 0000000000000..26f9e2c7d81d5
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestPutSamlServiceProviderAction.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestRequest.Method;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.idp.action.PutSamlServiceProviderAction;
+import org.elasticsearch.xpack.idp.action.PutSamlServiceProviderRequest;
+import org.elasticsearch.xpack.idp.action.PutSamlServiceProviderResponse;
+
+import java.io.IOException;
+import java.util.List;
+
+public class RestPutSamlServiceProviderAction extends IdpBaseRestHandler {
+
+    public RestPutSamlServiceProviderAction(XPackLicenseState licenseState) {
+        super(licenseState);
+    }
+
+    @Override
+    public String getName() {
+        return "idp_put_saml_sp_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(
+            new Route(Method.PUT, "/_idp/saml/sp/{sp_entity_id}"),
+            new Route(Method.POST, "/_idp/saml/sp/{sp_entity_id}")
+        );
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        final String entityId = restRequest.param("sp_entity_id");
+        final WriteRequest.RefreshPolicy refreshPolicy = restRequest.hasParam("refresh")
+            ? WriteRequest.RefreshPolicy.parse(restRequest.param("refresh")) : PutSamlServiceProviderRequest.DEFAULT_REFRESH_POLICY;
+        try (XContentParser parser = restRequest.contentParser()) {
+            final PutSamlServiceProviderRequest request = PutSamlServiceProviderRequest.fromXContent(entityId, refreshPolicy, parser);
+            return channel -> client.execute(PutSamlServiceProviderAction.INSTANCE, request,
+                new RestBuilderListener<>(channel) {
+                    @Override
+                    public RestResponse buildResponse(PutSamlServiceProviderResponse response, XContentBuilder builder) throws Exception {
+                        response.toXContent(builder, restRequest);
+                        return new BytesRestResponse(RestStatus.OK, builder);
+                    }
+                });
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlInitiateSingleSignOnAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlInitiateSingleSignOnAction.java
new file mode 100644
index 0000000000000..1903f4bca7b44
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlInitiateSingleSignOnAction.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.idp.action.SamlInitiateSingleSignOnAction;
+import org.elasticsearch.xpack.idp.action.SamlInitiateSingleSignOnRequest;
+import org.elasticsearch.xpack.idp.action.SamlInitiateSingleSignOnResponse;
+import org.elasticsearch.xpack.idp.saml.support.SamlAuthenticationState;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestSamlInitiateSingleSignOnAction extends IdpBaseRestHandler {
+    static final ObjectParser<SamlInitiateSingleSignOnRequest, Void> PARSER = new ObjectParser<>("idp_init_sso",
+        SamlInitiateSingleSignOnRequest::new);
+
+    static {
+        PARSER.declareString(SamlInitiateSingleSignOnRequest::setSpEntityId, new ParseField("entity_id"));
+        PARSER.declareObject(SamlInitiateSingleSignOnRequest::setSamlAuthenticationState, (p, c) -> SamlAuthenticationState.fromXContent(p),
+            new ParseField("authn_state"));
+    }
+
+    public RestSamlInitiateSingleSignOnAction(XPackLicenseState licenseState) {
+        super(licenseState);
+    }
+
+    @Override
+    public List<Route> routes() {
+        return Collections.singletonList(
+            new Route(POST, "/_idp/saml/init")
+        );
+    }
+
+    @Override
+    public String getName() {
+        return "saml_idp_init_sso_action";
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        try (XContentParser parser = request.contentParser()) {
+            final SamlInitiateSingleSignOnRequest initRequest = PARSER.parse(parser, null);
+            return channel -> client.execute(SamlInitiateSingleSignOnAction.INSTANCE, initRequest,
+                new RestBuilderListener<SamlInitiateSingleSignOnResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(SamlInitiateSingleSignOnResponse response, XContentBuilder builder) throws Exception {
+                        builder.startObject();
+                        builder.field("post_url", response.getPostUrl());
+                        builder.field("saml_response", response.getSamlResponse());
+                        builder.startObject("service_provider");
+                        builder.field("entity_id", response.getEntityId());
+                        builder.endObject();
+                        builder.endObject();
+                        return new BytesRestResponse(RestStatus.OK, builder);
+                    }
+                });
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlMetadataAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlMetadataAction.java
new file mode 100644
index 0000000000000..9212d956d1609
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlMetadataAction.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.idp.action.SamlMetadataAction;
+import org.elasticsearch.xpack.idp.action.SamlMetadataRequest;
+import org.elasticsearch.xpack.idp.action.SamlMetadataResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
+public class RestSamlMetadataAction extends IdpBaseRestHandler {
+
+    public RestSamlMetadataAction(XPackLicenseState licenseState) {
+        super(licenseState);
+    }
+
+    @Override
+    public String getName() {
+        return "saml_metadata_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return Collections.singletonList(new Route(GET, "/_idp/saml/metadata/{sp_entity_id}"));
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        final String spEntityId = request.param("sp_entity_id");
+        final SamlMetadataRequest metadataRequest = new SamlMetadataRequest(spEntityId);
+        return channel -> client.execute(SamlMetadataAction.INSTANCE, metadataRequest,
+            new RestBuilderListener<SamlMetadataResponse>(channel) {
+                @Override
+                public RestResponse buildResponse(SamlMetadataResponse response, XContentBuilder builder) throws Exception {
+                    builder.startObject();
+                    builder.field("metadata", response.getXmlString());
+                    builder.endObject();
+                    return new BytesRestResponse(RestStatus.OK, builder);
+                }
+            });
+
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlValidateAuthenticationRequestAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlValidateAuthenticationRequestAction.java
new file mode 100644
index 0000000000000..14fb48ad2fc16
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/rest/action/RestSamlValidateAuthenticationRequestAction.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestAction;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestRequest;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestSamlValidateAuthenticationRequestAction extends IdpBaseRestHandler {
+
+    static final ObjectParser<SamlValidateAuthnRequestRequest, Void> PARSER =
+        new ObjectParser<>("idp_validate_authn_request", SamlValidateAuthnRequestRequest::new);
+
+    static {
+        PARSER.declareString(SamlValidateAuthnRequestRequest::setQueryString, new ParseField("authn_request_query"));
+    }
+
+    public RestSamlValidateAuthenticationRequestAction(XPackLicenseState licenseState) {
+        super(licenseState);
+    }
+
+    @Override
+    public String getName() {
+        return "saml_idp_validate_authn_request_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return Collections.singletonList(new Route(POST, "/_idp/saml/validate"));
+    }
+
+    @Override
+    protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        try (XContentParser parser = request.contentParser()) {
+            final SamlValidateAuthnRequestRequest validateRequest = PARSER.parse(parser, null);
+            return channel -> client.execute(SamlValidateAuthnRequestAction.INSTANCE, validateRequest,
+                new RestBuilderListener<SamlValidateAuthnRequestResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(SamlValidateAuthnRequestResponse response, XContentBuilder builder) throws Exception {
+                        builder.startObject();
+                        builder.startObject("service_provider");
+                        builder.field("entity_id", response.getSpEntityId());
+                        builder.endObject();
+                        builder.field("force_authn", response.isForceAuthn());
+                        builder.field("authn_state", response.getAuthnState());
+                        builder.endObject();
+                        return new BytesRestResponse(RestStatus.OK, builder);
+                    }
+                });
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/CloudServiceProvider.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/CloudServiceProvider.java
new file mode 100644
index 0000000000000..4e99ac5004dc4
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/CloudServiceProvider.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.idp.privileges.ServiceProviderPrivileges;
+import org.joda.time.ReadableDuration;
+import org.opensaml.security.x509.X509Credential;
+
+import java.net.URL;
+import java.util.Set;
+
+
+public class CloudServiceProvider implements SamlServiceProvider {
+
+    private final String entityId;
+    private final String name;
+    private final boolean enabled;
+    private final URL assertionConsumerService;
+    private final String allowedNameIdFormat;
+    private final ReadableDuration authnExpiry;
+    private final ServiceProviderPrivileges privileges;
+    private final AttributeNames attributeNames;
+    private final Set<X509Credential> spSigningCredentials;
+    private final boolean signAuthnRequests;
+    private final boolean signLogoutRequests;
+
+    public CloudServiceProvider(String entityId, String name, boolean enabled, URL assertionConsumerService, String allowedNameIdFormat,
+                                ReadableDuration authnExpiry, ServiceProviderPrivileges privileges, AttributeNames attributeNames,
+                                Set<X509Credential> spSigningCredentials, boolean signAuthnRequests, boolean signLogoutRequests) {
+        if (Strings.isNullOrEmpty(entityId)) {
+            throw new IllegalArgumentException("Service Provider Entity ID cannot be null or empty");
+        }
+        this.entityId = entityId;
+        this.name = name;
+        this.enabled = enabled;
+        this.assertionConsumerService = assertionConsumerService;
+        this.allowedNameIdFormat = allowedNameIdFormat;
+        this.authnExpiry = authnExpiry;
+        this.privileges = privileges;
+        this.attributeNames = attributeNames;
+        this.spSigningCredentials = spSigningCredentials == null ? Set.of() : Set.copyOf(spSigningCredentials);
+        this.signLogoutRequests = signLogoutRequests;
+        this.signAuthnRequests = signAuthnRequests;
+    }
+
+    @Override
+    public String getEntityId() {
+        return entityId;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    @Override
+    public String getAllowedNameIdFormat() {
+        return allowedNameIdFormat;
+    }
+
+    @Override
+    public URL getAssertionConsumerService() {
+        return assertionConsumerService;
+    }
+
+    @Override
+    public ReadableDuration getAuthnExpiry() {
+        return authnExpiry;
+    }
+
+    @Override
+    public AttributeNames getAttributeNames() {
+        return attributeNames;
+    }
+
+    @Override
+    public Set<X509Credential> getSpSigningCredentials() {
+        return spSigningCredentials;
+    }
+
+    @Override
+    public boolean shouldSignAuthnRequests() {
+        return signAuthnRequests;
+    }
+
+    @Override
+    public boolean shouldSignLogoutRequests() {
+        return signLogoutRequests;
+    }
+
+    @Override
+    public ServiceProviderPrivileges getPrivileges() {
+        return privileges;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName()
+            + "{"
+            + "entityId=[" + entityId + ']'
+            + " name=[" + name + ']'
+            + " enabled=" + enabled
+            + " acs=[" + assertionConsumerService + "]"
+            + "}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProvider.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProvider.java
new file mode 100644
index 0000000000000..bb22857cacf45
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProvider.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.xpack.idp.privileges.ServiceProviderPrivileges;
+import org.joda.time.ReadableDuration;
+import org.opensaml.security.x509.X509Credential;
+
+import java.net.URL;
+import java.util.Set;
+
+/**
+ * SAML 2.0 configuration information about a specific service provider
+ */
+public interface SamlServiceProvider {
+
+    String getName();
+
+    boolean isEnabled();
+
+    String getEntityId();
+
+    String getAllowedNameIdFormat();
+
+    URL getAssertionConsumerService();
+
+    ReadableDuration getAuthnExpiry();
+
+    class AttributeNames {
+        public final String principal;
+        public final String name;
+        public final String email;
+        public final String roles;
+
+        public AttributeNames(String principal, String name, String email, String roles) {
+            this.principal = principal;
+            this.name = name;
+            this.email = email;
+            this.roles = roles;
+        }
+    }
+
+    AttributeNames getAttributeNames();
+
+    ServiceProviderPrivileges getPrivileges();
+
+    Set<X509Credential> getSpSigningCredentials();
+
+    boolean shouldSignAuthnRequests();
+
+    boolean shouldSignLogoutRequests();
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocument.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocument.java
new file mode 100644
index 0000000000000..cde39f13cdebd
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocument.java
@@ -0,0 +1,539 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.joda.time.Duration;
+import org.joda.time.ReadableDuration;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.Instant;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+
+/**
+ * This class models the storage of a {@link SamlServiceProvider} as an Elasticsearch document.
+ */
+public class SamlServiceProviderDocument implements ToXContentObject, Writeable {
+
+    public static final String SIGN_AUTHN = "authn";
+    public static final String SIGN_LOGOUT = "logout";
+    private static final Set<String> ALLOWED_SIGN_MESSAGES = Set.of(SIGN_AUTHN, SIGN_LOGOUT);
+
+    public static class Privileges {
+        public String resource;
+        public Map<String, String> roleActions = Map.of();
+
+        public void setResource(String resource) {
+            this.resource = resource;
+        }
+
+        public void setRoleActions(Map<String, String> roleActions) {
+            this.roleActions = roleActions;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            final Privileges that = (Privileges) o;
+            return Objects.equals(resource, that.resource) &&
+                Objects.equals(roleActions, that.roleActions);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(resource, roleActions);
+        }
+    }
+
+    public static class AttributeNames {
+        public String principal;
+        @Nullable
+        public String email;
+        @Nullable
+        public String name;
+        @Nullable
+        public String roles;
+
+        public void setPrincipal(String principal) {
+            this.principal = principal;
+        }
+
+        public void setEmail(String email) {
+            this.email = email;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public void setRoles(String roles) {
+            this.roles = roles;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            final AttributeNames that = (AttributeNames) o;
+            return Objects.equals(principal, that.principal) &&
+                Objects.equals(email, that.email) &&
+                Objects.equals(name, that.name) &&
+                Objects.equals(roles, that.roles);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(principal, email, name, roles);
+        }
+    }
+
+    public static class Certificates {
+        public List<String> serviceProviderSigning = List.of();
+        public List<String> identityProviderSigning = List.of();
+        public List<String> identityProviderMetadataSigning = List.of();
+
+        public void setServiceProviderSigning(Collection<String> serviceProviderSigning) {
+            this.serviceProviderSigning = serviceProviderSigning == null ? List.of() : List.copyOf(serviceProviderSigning);
+        }
+
+        public void setIdentityProviderSigning(Collection<String> identityProviderSigning) {
+            this.identityProviderSigning = identityProviderSigning == null ? List.of() : List.copyOf(identityProviderSigning);
+        }
+
+        public void setIdentityProviderMetadataSigning(Collection<String> identityProviderMetadataSigning) {
+            this.identityProviderMetadataSigning
+                = identityProviderMetadataSigning == null ? List.of() : List.copyOf(identityProviderMetadataSigning);
+        }
+
+        public void setServiceProviderX509SigningCertificates(Collection<X509Certificate> certificates) {
+            this.serviceProviderSigning = encodeCertificates(certificates);
+        }
+
+        public List<X509Certificate> getServiceProviderX509SigningCertificates() {
+            return decodeCertificates(this.serviceProviderSigning);
+        }
+
+        public void setIdentityProviderX509SigningCertificates(Collection<X509Certificate> certificates) {
+            this.identityProviderSigning = encodeCertificates(certificates);
+        }
+
+        public List<X509Certificate> getIdentityProviderX509SigningCertificates() {
+            return decodeCertificates(this.identityProviderSigning);
+        }
+
+        public void setIdentityProviderX509MetadataSigningCertificates(Collection<X509Certificate> certificates) {
+            this.identityProviderMetadataSigning = encodeCertificates(certificates);
+        }
+
+        public List<X509Certificate> getIdentityProviderX509MetadataSigningCertificates() {
+            return decodeCertificates(this.identityProviderMetadataSigning);
+        }
+
+        private List<String> encodeCertificates(Collection<X509Certificate> certificates) {
+            return certificates == null ? List.of() : certificates.stream()
+                .map(cert -> {
+                    try {
+                        return cert.getEncoded();
+                    } catch (CertificateEncodingException e) {
+                        throw new ElasticsearchException("Cannot read certificate", e);
+                    }
+                })
+                .map(Base64.getEncoder()::encodeToString)
+                .collect(Collectors.toUnmodifiableList());
+        }
+
+        private List<X509Certificate> decodeCertificates(List<String> encodedCertificates) {
+            if (encodedCertificates == null || encodedCertificates.isEmpty()) {
+                return List.of();
+            }
+            return encodedCertificates.stream().map(this::decodeCertificate).collect(Collectors.toUnmodifiableList());
+        }
+
+        private X509Certificate decodeCertificate(String base64Cert) {
+            final byte[] bytes = base64Cert.getBytes(StandardCharsets.UTF_8);
+            try (InputStream stream = new ByteArrayInputStream(bytes)) {
+                final List<Certificate> certificates = CertParsingUtils.readCertificates(Base64.getDecoder().wrap(stream));
+                if (certificates.size() == 1) {
+                    final Certificate certificate = certificates.get(0);
+                    if (certificate instanceof X509Certificate) {
+                        return (X509Certificate) certificate;
+                    } else {
+                        throw new ElasticsearchException("Certificate ({}) is not a X.509 certificate", certificate.getClass());
+                    }
+                } else {
+                    throw new ElasticsearchException("Expected a single certificate, but found {}", certificates.size());
+                }
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            } catch (CertificateException e) {
+                throw new ElasticsearchException("Cannot parse certificate(s)", e);
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            final Certificates that = (Certificates) o;
+            return Objects.equals(serviceProviderSigning, that.serviceProviderSigning) &&
+                Objects.equals(identityProviderSigning, that.identityProviderSigning) &&
+                Objects.equals(identityProviderMetadataSigning, that.identityProviderMetadataSigning);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(serviceProviderSigning, identityProviderSigning, identityProviderMetadataSigning);
+        }
+    }
+
+    @Nullable
+    public String docId;
+
+    public String name;
+
+    public String entityId;
+
+    public String acs;
+    
+    @Nullable
+    public String nameIdFormat;
+
+    public boolean enabled = true;
+    public Instant created;
+    public Instant lastModified;
+
+    public Set<String> signMessages = Set.of();
+
+    @Nullable
+    public Long authenticationExpiryMillis;
+
+    public final Privileges privileges = new Privileges();
+    public final AttributeNames attributeNames = new AttributeNames();
+    public final Certificates certificates = new Certificates();
+
+    public SamlServiceProviderDocument() {
+    }
+
+    public SamlServiceProviderDocument(StreamInput in) throws IOException {
+        docId = in.readOptionalString();
+        name = in.readString();
+        entityId = in.readString();
+        acs = in.readString();
+        enabled = in.readBoolean();
+        created = in.readInstant();
+        lastModified = in.readInstant();
+        nameIdFormat = in.readOptionalString();
+        authenticationExpiryMillis = in.readOptionalVLong();
+
+        privileges.resource = in.readString();
+        privileges.roleActions = in.readMap(StreamInput::readString, StreamInput::readString);
+
+        attributeNames.principal = in.readString();
+        attributeNames.email = in.readOptionalString();
+        attributeNames.name = in.readOptionalString();
+        attributeNames.roles = in.readOptionalString();
+
+        certificates.serviceProviderSigning = in.readStringList();
+        certificates.identityProviderSigning = in.readStringList();
+        certificates.identityProviderMetadataSigning = in.readStringList();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(docId);
+        out.writeString(name);
+        out.writeString(entityId);
+        out.writeString(acs);
+        out.writeBoolean(enabled);
+        out.writeInstant(created);
+        out.writeInstant(lastModified);
+        out.writeOptionalString(nameIdFormat);
+        out.writeOptionalVLong(authenticationExpiryMillis);
+
+        out.writeString(privileges.resource);
+        out.writeMap(privileges.roleActions == null ? Map.of() : privileges.roleActions,
+            StreamOutput::writeString, StreamOutput::writeString);
+
+        out.writeString(attributeNames.principal);
+        out.writeOptionalString(attributeNames.email);
+        out.writeOptionalString(attributeNames.name);
+        out.writeOptionalString(attributeNames.roles);
+
+        out.writeStringCollection(certificates.serviceProviderSigning);
+        out.writeStringCollection(certificates.identityProviderSigning);
+        out.writeStringCollection(certificates.identityProviderMetadataSigning);
+    }
+
+    public String getDocId() {
+        return docId;
+    }
+
+    public void setDocId(String docId) {
+        this.docId = docId;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public void setEntityId(String entityId) {
+        this.entityId = entityId;
+    }
+
+    public void setAcs(String acs) {
+        this.acs = acs;
+    }
+
+    public void setEnabled(boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    public void setCreated(Instant created) {
+        this.created = created;
+    }
+
+    public void setCreatedMillis(Long millis) {
+        this.created = Instant.ofEpochMilli(millis);
+    }
+
+    public void setLastModifiedMillis(Long millis) {
+        this.lastModified = Instant.ofEpochMilli(millis);
+    }
+
+    public void setNameIdFormat(String nameIdFormat) {
+        this.nameIdFormat = nameIdFormat;
+    }
+
+    public void setSignMessages(Collection<String> signMessages) {
+        this.signMessages = signMessages == null ? Set.of() : Set.copyOf(signMessages);
+    }
+
+    public void setAuthenticationExpiryMillis(Long authenticationExpiryMillis) {
+        this.authenticationExpiryMillis = authenticationExpiryMillis;
+    }
+
+    public void setAuthenticationExpiry(ReadableDuration authnExpiry) {
+        this.authenticationExpiryMillis = authnExpiry == null ? null : authnExpiry.getMillis();
+    }
+
+    public ReadableDuration getAuthenticationExpiry() {
+        return authenticationExpiryMillis == null ? null : Duration.millis(this.authenticationExpiryMillis);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final SamlServiceProviderDocument that = (SamlServiceProviderDocument) o;
+        return Objects.equals(docId, that.docId) &&
+            Objects.equals(name, that.name) &&
+            Objects.equals(entityId, that.entityId) &&
+            Objects.equals(acs, that.acs) &&
+            Objects.equals(enabled, that.enabled) &&
+            Objects.equals(created, that.created) &&
+            Objects.equals(lastModified, that.lastModified) &&
+            Objects.equals(nameIdFormat, that.nameIdFormat) &&
+            Objects.equals(authenticationExpiryMillis, that.authenticationExpiryMillis) &&
+            Objects.equals(certificates, that.certificates) &&
+            Objects.equals(privileges, that.privileges) &&
+            Objects.equals(attributeNames, that.attributeNames);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(docId, name, entityId, acs, enabled, created, lastModified, nameIdFormat, authenticationExpiryMillis,
+            certificates, privileges, attributeNames);
+    }
+
+    private static final ObjectParser<SamlServiceProviderDocument, SamlServiceProviderDocument> DOC_PARSER
+        = new ObjectParser<>("service_provider_doc", true, SamlServiceProviderDocument::new);
+    private static final ObjectParser<Privileges, Void> PRIVILEGES_PARSER = new ObjectParser<>("service_provider_priv", true, null);
+    private static final ObjectParser<AttributeNames, Void> ATTRIBUTES_PARSER = new ObjectParser<>("service_provider_attr", true, null);
+    private static final ObjectParser<Certificates, Void> CERTIFICATES_PARSER = new ObjectParser<>("service_provider_cert", true, null);
+
+    private static final BiConsumer<SamlServiceProviderDocument, Object> NULL_CONSUMER = (doc, obj) -> {
+    };
+
+    static {
+        DOC_PARSER.declareString(SamlServiceProviderDocument::setName, Fields.NAME);
+        DOC_PARSER.declareString(SamlServiceProviderDocument::setEntityId, Fields.ENTITY_ID);
+        DOC_PARSER.declareString(SamlServiceProviderDocument::setAcs, Fields.ACS);
+        DOC_PARSER.declareBoolean(SamlServiceProviderDocument::setEnabled, Fields.ENABLED);
+        DOC_PARSER.declareLong(SamlServiceProviderDocument::setCreatedMillis, Fields.CREATED_DATE);
+        DOC_PARSER.declareLong(SamlServiceProviderDocument::setLastModifiedMillis, Fields.LAST_MODIFIED);
+        DOC_PARSER.declareStringOrNull(SamlServiceProviderDocument::setNameIdFormat, Fields.NAME_ID);
+        DOC_PARSER.declareStringArray(SamlServiceProviderDocument::setSignMessages, Fields.SIGN_MSGS);
+        DOC_PARSER.declareField(SamlServiceProviderDocument::setAuthenticationExpiryMillis,
+            parser -> parser.currentToken() == XContentParser.Token.VALUE_NULL ? null : parser.longValue(),
+            Fields.AUTHN_EXPIRY, ObjectParser.ValueType.LONG_OR_NULL);
+
+        DOC_PARSER.declareObject(NULL_CONSUMER, (parser, doc) -> PRIVILEGES_PARSER.parse(parser, doc.privileges, null), Fields.PRIVILEGES);
+        PRIVILEGES_PARSER.declareString(Privileges::setResource, Fields.Privileges.RESOURCE);
+        PRIVILEGES_PARSER.declareField(Privileges::setRoleActions,
+            (parser, ignore) -> parser.currentToken() == XContentParser.Token.VALUE_NULL ? null : parser.mapStrings(),
+            Fields.Privileges.ROLES, ObjectParser.ValueType.OBJECT_OR_NULL);
+
+        DOC_PARSER.declareObject(NULL_CONSUMER, (p, doc) -> ATTRIBUTES_PARSER.parse(p, doc.attributeNames, null), Fields.ATTRIBUTES);
+        ATTRIBUTES_PARSER.declareString(AttributeNames::setPrincipal, Fields.Attributes.PRINCIPAL);
+        ATTRIBUTES_PARSER.declareStringOrNull(AttributeNames::setEmail, Fields.Attributes.EMAIL);
+        ATTRIBUTES_PARSER.declareStringOrNull(AttributeNames::setName, Fields.Attributes.NAME);
+        ATTRIBUTES_PARSER.declareStringOrNull(AttributeNames::setRoles, Fields.Attributes.ROLES);
+
+        DOC_PARSER.declareObject(NULL_CONSUMER, (p, doc) -> CERTIFICATES_PARSER.parse(p, doc.certificates, null), Fields.CERTIFICATES);
+        CERTIFICATES_PARSER.declareStringArray(Certificates::setServiceProviderSigning, Fields.Certificates.SP_SIGNING);
+        CERTIFICATES_PARSER.declareStringArray(Certificates::setIdentityProviderSigning, Fields.Certificates.IDP_SIGNING);
+        CERTIFICATES_PARSER.declareStringArray(Certificates::setIdentityProviderMetadataSigning, Fields.Certificates.IDP_METADATA);
+    }
+
+    public static SamlServiceProviderDocument fromXContent(String docId, XContentParser parser) throws IOException {
+        SamlServiceProviderDocument doc = new SamlServiceProviderDocument();
+        doc.setDocId(docId);
+        return DOC_PARSER.parse(parser, doc, doc);
+    }
+
+    public ValidationException validate() {
+        final ValidationException validation = new ValidationException();
+        if (Strings.isNullOrEmpty(name)) {
+            validation.addValidationError("field [" + Fields.NAME + "] is required, but was [" + name + "]");
+        }
+        if (Strings.isNullOrEmpty(entityId)) {
+            validation.addValidationError("field [" + Fields.ENTITY_ID + "] is required, but was [" + entityId + "]");
+        }
+        if (Strings.isNullOrEmpty(acs)) {
+            validation.addValidationError("field [" + Fields.ACS + "] is required, but was [" + acs + "]");
+        }
+        if (created == null) {
+            validation.addValidationError("field [" + Fields.CREATED_DATE + "] is required, but was [" + created + "]");
+        }
+        if (lastModified == null) {
+            validation.addValidationError("field [" + Fields.LAST_MODIFIED + "] is required, but was [" + lastModified + "]");
+        }
+
+        final Set<String> invalidSignOptions = Sets.difference(signMessages, ALLOWED_SIGN_MESSAGES);
+        if (invalidSignOptions.isEmpty() == false) {
+            validation.addValidationError("the values [" + invalidSignOptions + "] are not permitted for [" + Fields.SIGN_MSGS
+                + "] - permitted values are [" + ALLOWED_SIGN_MESSAGES + "]");
+        }
+
+        if (Strings.isNullOrEmpty(privileges.resource)) {
+            validation.addValidationError("field [" + Fields.PRIVILEGES + "." + Fields.Privileges.RESOURCE
+                + "] is required, but was [" + privileges.resource + "]");
+        }
+        if (Strings.isNullOrEmpty(attributeNames.principal)) {
+            validation.addValidationError("field [" + Fields.ATTRIBUTES + "." + Fields.Attributes.PRINCIPAL
+                + "] is required, but was [" + attributeNames.principal + "]");
+        }
+        if (validation.validationErrors().isEmpty()) {
+            return null;
+        } else {
+            return validation;
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Fields.NAME.getPreferredName(), name);
+        builder.field(Fields.ENTITY_ID.getPreferredName(), entityId);
+        builder.field(Fields.ACS.getPreferredName(), acs);
+        builder.field(Fields.ENABLED.getPreferredName(), enabled);
+        builder.field(Fields.CREATED_DATE.getPreferredName(), created == null ? null : created.toEpochMilli());
+        builder.field(Fields.LAST_MODIFIED.getPreferredName(), lastModified == null ? null : lastModified.toEpochMilli());
+        builder.field(Fields.NAME_ID.getPreferredName(), nameIdFormat);
+        builder.field(Fields.SIGN_MSGS.getPreferredName(), signMessages == null ? List.of() : signMessages);
+        builder.field(Fields.AUTHN_EXPIRY.getPreferredName(), authenticationExpiryMillis);
+
+        builder.startObject(Fields.PRIVILEGES.getPreferredName());
+        builder.field(Fields.Privileges.RESOURCE.getPreferredName(), privileges.resource);
+        builder.field(Fields.Privileges.ROLES.getPreferredName(), privileges.roleActions);
+        builder.endObject();
+
+        builder.startObject(Fields.ATTRIBUTES.getPreferredName());
+        builder.field(Fields.Attributes.PRINCIPAL.getPreferredName(), attributeNames.principal);
+        builder.field(Fields.Attributes.EMAIL.getPreferredName(), attributeNames.email);
+        builder.field(Fields.Attributes.NAME.getPreferredName(), attributeNames.name);
+        builder.field(Fields.Attributes.ROLES.getPreferredName(), attributeNames.roles);
+        builder.endObject();
+
+        builder.startObject(Fields.CERTIFICATES.getPreferredName());
+        builder.field(Fields.Certificates.SP_SIGNING.getPreferredName(), certificates.serviceProviderSigning);
+        builder.field(Fields.Certificates.IDP_SIGNING.getPreferredName(), certificates.identityProviderSigning);
+        builder.field(Fields.Certificates.IDP_METADATA.getPreferredName(), certificates.identityProviderMetadataSigning);
+        builder.endObject();
+
+        return builder.endObject();
+    }
+
+    public interface Fields {
+        ParseField NAME = new ParseField("name");
+        ParseField ENTITY_ID = new ParseField("entity_id");
+        ParseField ACS = new ParseField("acs");
+        ParseField ENABLED = new ParseField("enabled");
+        ParseField NAME_ID = new ParseField("name_id_format");
+        ParseField SIGN_MSGS = new ParseField("sign_messages");
+        ParseField AUTHN_EXPIRY = new ParseField("authn_expiry_ms");
+
+        ParseField CREATED_DATE = new ParseField("created");
+        ParseField LAST_MODIFIED = new ParseField("last_modified");
+
+        ParseField PRIVILEGES = new ParseField("privileges");
+        ParseField ATTRIBUTES = new ParseField("attributes");
+        ParseField CERTIFICATES = new ParseField("certificates");
+
+        interface Privileges {
+            ParseField RESOURCE = new ParseField("resource");
+            ParseField ROLES = new ParseField("roles");
+        }
+
+        interface Attributes {
+            ParseField PRINCIPAL = new ParseField("principal");
+            ParseField EMAIL = new ParseField("email");
+            ParseField NAME = new ParseField("name");
+            ParseField ROLES = new ParseField("roles");
+        }
+
+        interface Certificates {
+            ParseField SP_SIGNING = new ParseField("sp_signing");
+            ParseField IDP_SIGNING = new ParseField("idp_signing");
+            ParseField IDP_METADATA = new ParseField("idp_metadata");
+        }
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{docId=" + docId + ", name=" + name + ", entityId=" + entityId + "}@" + hashCode();
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java
new file mode 100644
index 0000000000000..e509654520d22
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndex.java
@@ -0,0 +1,337 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.OriginSettingClient;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.util.CachedSupplier;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * This class provides utility methods to read/write {@link SamlServiceProviderDocument} to an Elasticsearch index.
+ */
+public class SamlServiceProviderIndex implements Closeable {
+
+    private final Logger logger = LogManager.getLogger();
+
+    private final Client client;
+    private final ClusterService clusterService;
+    private final ClusterStateListener clusterStateListener;
+    private volatile boolean aliasExists;
+    private volatile boolean templateInstalled;
+
+    public static final String ALIAS_NAME = "saml-service-provider";
+    public static final String INDEX_NAME = "saml-service-provider-v1";
+    static final String TEMPLATE_NAME = ALIAS_NAME;
+
+    private static final String TEMPLATE_RESOURCE = "/org/elasticsearch/xpack/idp/saml-service-provider-template.json";
+    private static final String TEMPLATE_META_VERSION_KEY = "idp-version";
+    private static final String TEMPLATE_VERSION_SUBSTITUTE = "idp.template.version";
+
+    public static final class DocumentVersion {
+        public final String id;
+        public final long primaryTerm;
+        public final long seqNo;
+
+        public DocumentVersion(String id, long primaryTerm, long seqNo) {
+            this.id = id;
+            this.primaryTerm = primaryTerm;
+            this.seqNo = seqNo;
+        }
+
+        public DocumentVersion(GetResponse get) {
+            this(get.getId(), get.getPrimaryTerm(), get.getSeqNo());
+        }
+
+        public DocumentVersion(GetResult get) {
+            this(get.getId(), get.getPrimaryTerm(), get.getSeqNo());
+        }
+
+        public DocumentVersion(SearchHit hit) {
+            this(hit.getId(), hit.getPrimaryTerm(), hit.getSeqNo());
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            final DocumentVersion that = (DocumentVersion) o;
+            return Objects.equals(this.id, that.id) && primaryTerm == that.primaryTerm &&
+                seqNo == that.seqNo;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(id, primaryTerm, seqNo);
+        }
+    }
+
+    public static final class DocumentSupplier {
+        public final DocumentVersion version;
+        public final Supplier<SamlServiceProviderDocument> document;
+
+        public DocumentSupplier(DocumentVersion version, Supplier<SamlServiceProviderDocument> document) {
+            this.version = version;
+            this.document = new CachedSupplier<>(document);
+        }
+
+        public SamlServiceProviderDocument getDocument() {
+            return document.get();
+        }
+    }
+
+    public SamlServiceProviderIndex(Client client, ClusterService clusterService) {
+        this.client = new OriginSettingClient(client, ClientHelper.IDP_ORIGIN);
+        this.clusterService = clusterService;
+        this.clusterStateListener = this::clusterChanged;
+        clusterService.addListener(clusterStateListener);
+    }
+
+    private void clusterChanged(ClusterChangedEvent clusterChangedEvent) {
+        final ClusterState state = clusterChangedEvent.state();
+        installTemplateIfRequired(state);
+        checkForAliasStateChange(state);
+    }
+
+    private void installTemplateIfRequired(ClusterState state) {
+        if (templateInstalled) {
+            return;
+        }
+        if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            return;
+        }
+        if (isTemplateUpToDate(state)) {
+            templateInstalled = true;
+            return;
+        }
+        if (state.nodes().isLocalNodeElectedMaster() == false) {
+            return;
+        }
+        installIndexTemplate(ActionListener.wrap(
+            installed -> {
+                templateInstalled = true;
+                if (installed) {
+                    logger.debug("Template [{}] has been updated", TEMPLATE_NAME);
+                } else {
+                    logger.debug("Template [{}] appears to be up to date", TEMPLATE_NAME);
+                }
+            }, e -> logger.warn(new ParameterizedMessage("Failed to install template [{}]", TEMPLATE_NAME), e)
+        ));
+    }
+
+    private void checkForAliasStateChange(ClusterState state) {
+        final AliasOrIndex aliasInfo = state.getMetaData().getAliasAndIndexLookup().get(ALIAS_NAME);
+        final boolean previousState = aliasExists;
+        this.aliasExists = aliasInfo != null;
+        if (aliasExists != previousState) {
+            logChangedAliasState(aliasInfo);
+        }
+    }
+
+    @Override
+    public void close() {
+        logger.debug("Closing ... removing cluster state listener");
+        clusterService.removeListener(clusterStateListener);
+    }
+
+    private void logChangedAliasState(AliasOrIndex aliasInfo) {
+        if (aliasInfo == null) {
+            logger.warn("service provider index/alias [{}] no longer exists", ALIAS_NAME);
+        } else if (aliasInfo.isAlias() == false) {
+            logger.warn("service provider index [{}] exists as a concrete index, but it should be an alias", ALIAS_NAME);
+        } else if (aliasInfo.getIndices().size() != 1) {
+            logger.warn("service provider alias [{}] refers to multiple indices [{}] - this is unexpected and is likely to cause problems",
+                ALIAS_NAME, Strings.collectionToCommaDelimitedString(aliasInfo.getIndices()));
+        } else {
+            logger.info("service provider alias [{}] refers to [{}]", ALIAS_NAME, aliasInfo.getIndices().get(0).getIndex());
+        }
+    }
+
+    public void installIndexTemplate(ActionListener<Boolean> listener) {
+        final ClusterState state = clusterService.state();
+        if (isTemplateUpToDate(state)) {
+            listener.onResponse(false);
+        }
+        final String template = TemplateUtils.loadTemplate(TEMPLATE_RESOURCE, Version.CURRENT.toString(), TEMPLATE_VERSION_SUBSTITUTE);
+        final PutIndexTemplateRequest request = new PutIndexTemplateRequest(TEMPLATE_NAME).source(template, XContentType.JSON);
+        client.admin().indices().putTemplate(request, ActionListener.wrap(response -> {
+            logger.info("Installed template [{}]", TEMPLATE_NAME);
+            listener.onResponse(true);
+        }, listener::onFailure));
+    }
+
+    private boolean isTemplateUpToDate(ClusterState state) {
+        return TemplateUtils.checkTemplateExistsAndIsUpToDate(TEMPLATE_NAME, TEMPLATE_META_VERSION_KEY, state, logger);
+    }
+
+    public void deleteDocument(DocumentVersion version, WriteRequest.RefreshPolicy refreshPolicy, ActionListener<DeleteResponse> listener) {
+        final DeleteRequest request = new DeleteRequest(aliasExists ? ALIAS_NAME : INDEX_NAME)
+            .id(version.id)
+            .setIfSeqNo(version.seqNo)
+            .setIfPrimaryTerm(version.primaryTerm)
+            .setRefreshPolicy(refreshPolicy);
+        client.delete(request, ActionListener.wrap(response -> {
+            logger.debug("Deleted service provider document [{}] ({})", version.id, response.getResult());
+            listener.onResponse(response);
+        }, listener::onFailure));
+    }
+
+    public void writeDocument(SamlServiceProviderDocument document, DocWriteRequest.OpType opType,
+                              WriteRequest.RefreshPolicy refreshPolicy, ActionListener<DocWriteResponse> listener) {
+        final ValidationException exception = document.validate();
+        if (exception != null) {
+            listener.onFailure(exception);
+            return;
+        }
+
+        if (templateInstalled) {
+            _writeDocument(document, opType, refreshPolicy, listener);
+        } else {
+            installIndexTemplate(ActionListener.wrap(installed ->
+                _writeDocument(document, opType, refreshPolicy, listener), listener::onFailure));
+        }
+    }
+
+    private void _writeDocument(SamlServiceProviderDocument document, DocWriteRequest.OpType opType,
+                                WriteRequest.RefreshPolicy refreshPolicy, ActionListener<DocWriteResponse> listener) {
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream();
+             XContentBuilder xContentBuilder = new XContentBuilder(XContentType.JSON.xContent(), out)) {
+            document.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+            // Due to the lack of "alias templates" (at the current time), we cannot write to the alias if it doesn't exist yet
+            // - that would cause the alias to be created as a concrete index, which is not what we want.
+            // So, until we know that the alias exists we have to write to the expected index name instead.
+            final IndexRequest request = new IndexRequest(aliasExists ? ALIAS_NAME : INDEX_NAME)
+                .opType(opType)
+                .source(xContentBuilder)
+                .id(document.docId)
+                .setRefreshPolicy(refreshPolicy);
+            client.index(request, ActionListener.wrap(response -> {
+                logger.debug("Wrote service provider [{}][{}] as document [{}] ({})",
+                    document.name, document.entityId, response.getId(), response.getResult());
+                listener.onResponse(response);
+            }, listener::onFailure));
+        } catch (IOException e) {
+            listener.onFailure(e);
+        }
+    }
+
+    public void readDocument(String documentId, ActionListener<DocumentSupplier> listener) {
+        final GetRequest request = new GetRequest(ALIAS_NAME, documentId);
+        client.get(request, ActionListener.wrap(response -> {
+            if (response.isExists()) {
+                listener.onResponse(
+                    new DocumentSupplier(new DocumentVersion(response), () -> toDocument(documentId, response.getSourceAsBytesRef()))
+                );
+            } else {
+                listener.onResponse(null);
+            }
+        }, listener::onFailure));
+    }
+
+    public void findByEntityId(String entityId, ActionListener<Set<DocumentSupplier>> listener) {
+        final QueryBuilder query = QueryBuilders.termQuery(SamlServiceProviderDocument.Fields.ENTITY_ID.getPreferredName(), entityId);
+        findDocuments(query, listener);
+    }
+
+    public void findAll(ActionListener<Set<DocumentSupplier>> listener) {
+        final QueryBuilder query = QueryBuilders.matchAllQuery();
+        findDocuments(query, listener);
+    }
+
+    public void refresh(ActionListener<Void> listener) {
+        client.admin().indices().refresh(new RefreshRequest(ALIAS_NAME), ActionListener.wrap(
+            response -> listener.onResponse(null), listener::onFailure));
+    }
+
+    private void findDocuments(QueryBuilder query, ActionListener<Set<DocumentSupplier>> listener) {
+        logger.trace("Searching [{}] for [{}]", ALIAS_NAME, query);
+        final SearchRequest request = client.prepareSearch(ALIAS_NAME)
+            .setQuery(query)
+            .setSize(1000)
+            .setFetchSource(true)
+            .request();
+        client.search(request, ActionListener.wrap(response -> {
+            if (logger.isTraceEnabled()) {
+                logger.trace("Search hits: [{}] [{}]", response.getHits().getTotalHits(), Arrays.toString(response.getHits().getHits()));
+            }
+            final Set<DocumentSupplier> docs = Stream.of(response.getHits().getHits())
+                .map(hit -> new DocumentSupplier(new DocumentVersion(hit), () -> toDocument(hit.getId(), hit.getSourceRef())))
+                .collect(Collectors.toUnmodifiableSet());
+            listener.onResponse(docs);
+        }, ex -> {
+            if (ex instanceof IndexNotFoundException) {
+                listener.onResponse(Set.of());
+            } else {
+                listener.onFailure(ex);
+            }
+        }));
+    }
+
+    private SamlServiceProviderDocument toDocument(String documentId, BytesReference source) {
+        try (StreamInput in = source.streamInput();
+             XContentParser parser = XContentType.JSON.xContent().createParser(
+                 NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, in)) {
+            return SamlServiceProviderDocument.fromXContent(documentId, parser);
+        } catch (IOException e) {
+            throw new UncheckedIOException("failed to parse document [" + documentId + "]", e);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{alias=" + ALIAS_NAME + " [" + (aliasExists ? "exists" : "not-found") + "]}";
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolver.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolver.java
new file mode 100644
index 0000000000000..c4d1b89472101
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolver.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.xpack.idp.privileges.ServiceProviderPrivileges;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex.DocumentSupplier;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex.DocumentVersion;
+import org.joda.time.ReadableDuration;
+import org.opensaml.security.x509.BasicX509Credential;
+import org.opensaml.security.x509.X509Credential;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class SamlServiceProviderResolver {
+
+    private static final int CACHE_SIZE_DEFAULT = 1000;
+    private static final TimeValue CACHE_TTL_DEFAULT = TimeValue.timeValueMinutes(60);
+
+    public static final Setting<Integer> CACHE_SIZE
+        = Setting.intSetting("xpack.idp.sp.cache.size", CACHE_SIZE_DEFAULT, Setting.Property.NodeScope);
+    public static final Setting<TimeValue> CACHE_TTL
+        = Setting.timeSetting("xpack.idp.sp.cache.ttl", CACHE_TTL_DEFAULT, Setting.Property.NodeScope);
+
+    private final Cache<String, CachedServiceProvider> cache;
+    private final SamlServiceProviderIndex index;
+    private final ServiceProviderDefaults defaults;
+
+    public SamlServiceProviderResolver(Settings settings, SamlServiceProviderIndex index, ServiceProviderDefaults defaults) {
+        this.cache = CacheBuilder.<String, CachedServiceProvider>builder()
+            .setMaximumWeight(CACHE_SIZE.get(settings))
+            .setExpireAfterAccess(CACHE_TTL.get(settings))
+            .build();
+        this.index = index;
+        this.defaults = defaults;
+    }
+
+    /**
+     * Find a {@link SamlServiceProvider} by entity-id.
+     *
+     * @param listener Callback for the service provider object. Calls {@link ActionListener#onResponse} with a {@code null} value if the
+     *                 service provider does not exist.
+     */
+    public void resolve(String entityId, ActionListener<SamlServiceProvider> listener) {
+        index.findByEntityId(entityId, ActionListener.wrap(
+            documentSuppliers -> {
+                if (documentSuppliers.isEmpty()) {
+                    listener.onResponse(null);
+                    return;
+                }
+                if (documentSuppliers.size() > 1) {
+                    listener.onFailure(new IllegalStateException(
+                        "Found multiple service providers with entity ID [" + entityId
+                            + "] - document ids ["
+                            + documentSuppliers.stream().map(s -> s.version.id).collect(Collectors.joining(","))
+                            + "] in index [" + index + "]"));
+                    return;
+                }
+                final DocumentSupplier doc = Iterables.get(documentSuppliers, 0);
+                final CachedServiceProvider cached = cache.get(entityId);
+                if (cached != null && cached.documentVersion.equals(doc.version)) {
+                    listener.onResponse(cached.serviceProvider);
+                    return;
+                } else {
+                    populateCacheAndReturn(entityId, doc, listener);
+                }
+            },
+            listener::onFailure
+        ));
+
+    }
+
+    private void populateCacheAndReturn(String entityId, DocumentSupplier doc, ActionListener<SamlServiceProvider> listener) {
+        final SamlServiceProvider serviceProvider = buildServiceProvider(doc.document.get());
+        final CachedServiceProvider cacheEntry = new CachedServiceProvider(entityId, doc.version, serviceProvider);
+        cache.put(entityId, cacheEntry);
+        listener.onResponse(serviceProvider);
+    }
+
+    private SamlServiceProvider buildServiceProvider(SamlServiceProviderDocument document) {
+        final ServiceProviderPrivileges privileges = buildPrivileges(document.privileges);
+        final SamlServiceProvider.AttributeNames attributes = new SamlServiceProvider.AttributeNames(
+            document.attributeNames.principal, document.attributeNames.name, document.attributeNames.email, document.attributeNames.roles
+        );
+        final Set<X509Credential> credentials = document.certificates.getServiceProviderX509SigningCertificates()
+            .stream()
+            .map(BasicX509Credential::new)
+            .collect(Collectors.toUnmodifiableSet());
+
+        final URL acs = parseUrl(document);
+        String nameIdFormat = document.nameIdFormat;
+        if (nameIdFormat == null) {
+            nameIdFormat = defaults.nameIdFormat;
+        }
+
+        final ReadableDuration authnExpiry = Optional.ofNullable(document.getAuthenticationExpiry())
+            .orElse(defaults.authenticationExpiry);
+
+        final boolean signAuthnRequests = document.signMessages.contains(SamlServiceProviderDocument.SIGN_AUTHN);
+        final boolean signLogoutRequests = document.signMessages.contains(SamlServiceProviderDocument.SIGN_LOGOUT);
+
+        return new CloudServiceProvider(document.entityId, document.name, document.enabled, acs, nameIdFormat, authnExpiry,
+            privileges, attributes, credentials, signAuthnRequests, signLogoutRequests);
+    }
+
+    private ServiceProviderPrivileges buildPrivileges(SamlServiceProviderDocument.Privileges configuredPrivileges) {
+        final String resource = configuredPrivileges.resource;
+        final Map<String, String> roles = Optional.ofNullable(configuredPrivileges.roleActions).orElse(Map.of());
+        return new ServiceProviderPrivileges(defaults.applicationName, resource, roles);
+    }
+
+    private URL parseUrl(SamlServiceProviderDocument document) {
+        final URL acs;
+        try {
+            acs = new URL(document.acs);
+        } catch (MalformedURLException e) {
+            final ServiceProviderException exception = new ServiceProviderException(
+                "Service provider [{}] (doc {}) has an invalid ACS [{}]", e, document.entityId, document.docId, document.acs);
+            exception.setEntityId(document.entityId);
+            throw exception;
+        }
+        return acs;
+    }
+
+    private class CachedServiceProvider {
+        private final String entityId;
+        private final DocumentVersion documentVersion;
+        private final SamlServiceProvider serviceProvider;
+
+        private CachedServiceProvider(String entityId, DocumentVersion documentVersion, SamlServiceProvider serviceProvider) {
+            this.entityId = entityId;
+            this.documentVersion = documentVersion;
+            this.serviceProvider = serviceProvider;
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderDefaults.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderDefaults.java
new file mode 100644
index 0000000000000..a4e5523cc0e9f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderDefaults.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.joda.time.Duration;
+import org.joda.time.ReadableDuration;
+import org.opensaml.saml.saml2.core.NameID;
+
+import java.util.List;
+
+/**
+ * Defines default values for a service provider if they are not configured in {@link SamlServiceProviderDocument}
+ */
+public final class ServiceProviderDefaults {
+
+    public static final Setting<String> APPLICATION_NAME_SETTING
+        = Setting.simpleString("xpack.idp.privileges.application", Setting.Property.NodeScope);
+    public static final Setting<String> NAMEID_FORMAT_SETTING
+        = Setting.simpleString("xpack.idp.defaults.nameid_format", NameID.TRANSIENT, Setting.Property.NodeScope);
+    public static final Setting<TimeValue> AUTHN_EXPIRY_SETTING
+        = Setting.timeSetting("xpack.idp.defaults.authn_expiry", TimeValue.timeValueMinutes(5), Setting.Property.NodeScope);
+
+    public final String applicationName;
+    public final String nameIdFormat;
+    public final ReadableDuration authenticationExpiry;
+
+    public ServiceProviderDefaults(String applicationName,
+                                   String nameIdFormat,
+                                   ReadableDuration authenticationExpiry) {
+        this.applicationName = applicationName;
+        this.nameIdFormat = nameIdFormat;
+        this.authenticationExpiry = authenticationExpiry;
+    }
+
+    public static ServiceProviderDefaults forSettings(Settings settings) {
+        final String appplication = require(settings, APPLICATION_NAME_SETTING);
+        final String nameId = NAMEID_FORMAT_SETTING.get(settings);
+        final TimeValue expiry = AUTHN_EXPIRY_SETTING.get(settings);
+        return new ServiceProviderDefaults(appplication, nameId, Duration.millis(expiry.millis()));
+    }
+
+    private static <T> T require(Settings settings, Setting<T> setting) {
+        if (setting.exists(settings)) {
+            return setting.get(settings);
+        }
+        throw new IllegalStateException("Setting [" + setting.getKey() + "] must be configured");
+    }
+
+    public static List<Setting<?>> getSettings() {
+        return List.of(APPLICATION_NAME_SETTING, NAMEID_FORMAT_SETTING, AUTHN_EXPIRY_SETTING);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderException.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderException.java
new file mode 100644
index 0000000000000..9a6b03c1c73fe
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderException.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.ElasticsearchException;
+
+/**
+ * Indicates a configuration or execution problem specific to a SAML ServiceProvider
+ */
+public class ServiceProviderException extends ElasticsearchException {
+    public static final String ENTITY_ID = "es.idp.sp.entity_id";
+
+    public ServiceProviderException(String msg, Throwable cause, Object... args) {
+        super(msg, cause, args);
+    }
+
+    public void setEntityId(String entityId) {
+        super.addMetadata(ENTITY_ID, entityId);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationState.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationState.java
new file mode 100644
index 0000000000000..779ffa89a61b2
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationState.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Represents a state object that can be used during an SP initiated flow in order to pass necessary parameters from the response of the
+ * /validate API to the /init API. State properties are not integrity protected, might end up in a cookie on the user's side and as such
+ * are treated as tainted parameters for informational purposes only. No authentication/authorization decisions should be made based on
+ * these.
+ */
+public class SamlAuthenticationState implements Writeable, ToXContentObject {
+    private String entityId;
+    private String requestedAcsUrl;
+    @Nullable
+    private String requestedNameidFormat;
+    @Nullable
+    private String authnRequestId;
+
+    public SamlAuthenticationState() {
+
+    }
+
+    public SamlAuthenticationState(StreamInput in) throws IOException {
+        entityId = in.readString();
+        requestedAcsUrl = in.readString();
+        requestedNameidFormat = in.readOptionalString();
+        authnRequestId = in.readOptionalString();
+    }
+
+    public String getRequestedNameidFormat() {
+        return requestedNameidFormat;
+    }
+
+    public void setRequestedNameidFormat(String requestedNameidFormat) {
+        this.requestedNameidFormat = requestedNameidFormat;
+    }
+
+    public String getAuthnRequestId() {
+        return authnRequestId;
+    }
+
+    public void setAuthnRequestId(String authnRequestId) {
+        this.authnRequestId = authnRequestId;
+    }
+
+    public String getEntityId() {
+        return entityId;
+    }
+
+    public void setEntityId(String entityId) {
+        this.entityId = entityId;
+    }
+
+    public String getRequestedAcsUrl() {
+        return requestedAcsUrl;
+    }
+
+    public void setRequestedAcsUrl(String requestedAcsUrl) {
+        this.requestedAcsUrl = requestedAcsUrl;
+    }
+
+    public ValidationException validate() {
+        final ValidationException validation = new ValidationException();
+        if (Strings.isNullOrEmpty(entityId)) {
+            validation.addValidationError("field [" + Fields.ENTITY_ID + "] is required, but was [" + entityId + "]");
+        }
+        if (Strings.isNullOrEmpty(requestedAcsUrl)) {
+            validation.addValidationError("field [" + Fields.ACS_URL + "] is required, but was [" + requestedAcsUrl + "]");
+        }
+        return validation;
+    }
+
+    public static final ObjectParser<SamlAuthenticationState, SamlAuthenticationState> PARSER
+        = new ObjectParser<>("saml_authn_state", true, SamlAuthenticationState::new);
+
+    static {
+        PARSER.declareString(SamlAuthenticationState::setEntityId, Fields.ENTITY_ID);
+        PARSER.declareString(SamlAuthenticationState::setRequestedAcsUrl, Fields.ACS_URL);
+        PARSER.declareStringOrNull(SamlAuthenticationState::setRequestedNameidFormat, Fields.NAMEID_FORMAT);
+        PARSER.declareStringOrNull(SamlAuthenticationState::setAuthnRequestId, Fields.AUTHN_REQUEST_ID);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Fields.NAMEID_FORMAT.getPreferredName(), requestedNameidFormat);
+        builder.field(Fields.AUTHN_REQUEST_ID.getPreferredName(), authnRequestId);
+        builder.field(Fields.ENTITY_ID.getPreferredName(), entityId);
+        builder.field(Fields.ACS_URL.getPreferredName(), requestedAcsUrl);
+        return builder.endObject();
+    }
+
+    public static SamlAuthenticationState fromXContent(XContentParser parser) throws IOException {
+        SamlAuthenticationState authnState = new SamlAuthenticationState();
+        return PARSER.parse(parser, authnState, null);
+    }
+
+    public interface Fields {
+        ParseField NAMEID_FORMAT = new ParseField("nameid_format");
+        ParseField AUTHN_REQUEST_ID = new ParseField("authn_request_id");
+        ParseField ENTITY_ID = new ParseField("entity_id");
+        ParseField ACS_URL = new ParseField("acs_url");
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(entityId);
+        out.writeString(requestedAcsUrl);
+        out.writeOptionalString(requestedNameidFormat);
+        out.writeOptionalString(authnRequestId);
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + Strings.toString(this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SamlAuthenticationState that = (SamlAuthenticationState) o;
+        return entityId.equals(that.entityId) &&
+            requestedAcsUrl.equals(that.requestedAcsUrl) &&
+            Objects.equals(requestedNameidFormat, that.requestedNameidFormat) &&
+            Objects.equals(authnRequestId, that.authnRequestId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(entityId, requestedAcsUrl, requestedNameidFormat, authnRequestId);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlFactory.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlFactory.java
new file mode 100644
index 0000000000000..31dd6671cc078
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlFactory.java
@@ -0,0 +1,359 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.opensaml.core.xml.XMLObject;
+import org.opensaml.core.xml.XMLObjectBuilderFactory;
+import org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport;
+import org.opensaml.core.xml.io.MarshallingException;
+import org.opensaml.core.xml.io.Unmarshaller;
+import org.opensaml.core.xml.io.UnmarshallerFactory;
+import org.opensaml.core.xml.io.UnmarshallingException;
+import org.opensaml.core.xml.util.XMLObjectSupport;
+import org.opensaml.saml.common.SAMLObject;
+import org.opensaml.security.credential.Credential;
+import org.opensaml.security.x509.X509Credential;
+import org.w3c.dom.Element;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import javax.xml.XMLConstants;
+import javax.xml.namespace.QName;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.net.URISyntaxException;
+import java.security.SecureRandom;
+import java.security.cert.CertificateEncodingException;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport.getUnmarshallerFactory;
+
+/**
+ * Utility object for constructing new objects and values in a SAML 2.0 / OpenSAML context
+ */
+public class SamlFactory {
+
+    private final XMLObjectBuilderFactory builderFactory;
+    private final SecureRandom random;
+    private static final Logger LOGGER = LogManager.getLogger(SamlFactory.class);
+
+    public SamlFactory() {
+        SamlInit.initialize();
+        builderFactory = XMLObjectProviderRegistrySupport.getBuilderFactory();
+        random = new SecureRandom();
+    }
+
+    public <T extends XMLObject> T object(Class<T> type, QName elementName) {
+        final XMLObject obj = builderFactory.getBuilder(elementName).buildObject(elementName);
+        return cast(type, elementName, obj);
+    }
+
+    public <T extends XMLObject> T object(Class<T> type, QName elementName, QName schemaType) {
+        final XMLObject obj = builderFactory.getBuilder(schemaType).buildObject(elementName, schemaType);
+        return cast(type, elementName, obj);
+    }
+
+    private <T extends XMLObject> T cast(Class<T> type, QName elementName, XMLObject obj) {
+        if (type.isInstance(obj)) {
+            return type.cast(obj);
+        } else {
+            throw new IllegalArgumentException("Object for element " + elementName.getLocalPart() + " is of type " + obj.getClass()
+                + " not " + type);
+        }
+    }
+
+    public String secureIdentifier() {
+        return randomNCName(20);
+    }
+
+    private String randomNCName(int numberBytes) {
+        final byte[] randomBytes = new byte[numberBytes];
+        random.nextBytes(randomBytes);
+        // NCNames (https://www.w3.org/TR/xmlschema-2/#NCName) can't start with a number, so start them all with "_" to be safe
+        return "_".concat(MessageDigests.toHexString(randomBytes));
+    }
+
+    public <T extends XMLObject> T buildObject(Class<T> type, QName elementName) {
+        final XMLObject obj = builderFactory.getBuilder(elementName).buildObject(elementName);
+        if (type.isInstance(obj)) {
+            return type.cast(obj);
+        } else {
+            throw new IllegalArgumentException("Object for element " + elementName.getLocalPart() + " is of type " + obj.getClass()
+                + " not " + type);
+        }
+    }
+
+    public String toString(Element element, boolean pretty) {
+        try {
+            StringWriter writer = new StringWriter();
+            print(element, writer, pretty);
+            return writer.toString();
+        } catch (TransformerException e) {
+            return "[" + element.getNamespaceURI() + "]" + element.getLocalName();
+        }
+    }
+
+    public <T extends XMLObject> T buildXmlObject(Element element, Class<T> type) {
+        try {
+            UnmarshallerFactory unmarshallerFactory = getUnmarshallerFactory();
+            Unmarshaller unmarshaller = unmarshallerFactory.getUnmarshaller(element);
+            if (unmarshaller == null) {
+                throw new ElasticsearchSecurityException("XML element [{}] cannot be unmarshalled to SAML type [{}] (no unmarshaller)",
+                    element.getTagName(), type);
+            }
+            final XMLObject object = unmarshaller.unmarshall(element);
+            if (type.isInstance(object)) {
+                return type.cast(object);
+            }
+            Object[] args = new Object[]{element.getTagName(), type.getName(), object.getClass().getName()};
+            throw new ElasticsearchSecurityException("SAML object [{}] is incorrect type. Expected [{}] but was [{}]", args);
+        } catch (UnmarshallingException e) {
+            throw new ElasticsearchSecurityException("Failed to unmarshall SAML content [{}]", e, element.getTagName());
+        }
+    }
+
+    void print(Element element, Writer writer, boolean pretty) throws TransformerException {
+        final Transformer serializer = getHardenedXMLTransformer();
+        if (pretty) {
+            serializer.setOutputProperty(OutputKeys.INDENT, "yes");
+        }
+        serializer.transform(new DOMSource(element), new StreamResult(writer));
+    }
+
+    public String getXmlContent(SAMLObject object){
+        return getXmlContent(object, false);
+    }
+
+    public String getXmlContent(SAMLObject object, boolean prettyPrint) {
+        try {
+            return toString(XMLObjectSupport.marshall(object), prettyPrint);
+        } catch (MarshallingException e) {
+            LOGGER.info("Error marshalling SAMLObject ", e);
+            return "_unserializable_";
+        }
+    }
+
+    public boolean elementNameMatches(Element element, String namespace, String localName) {
+        return localName.equals(element.getLocalName()) && namespace.equals(element.getNamespaceURI());
+    }
+
+    public String text(Element dom, int length) {
+        return text(dom, length, 0);
+    }
+
+    public String text(XMLObject xml, int prefixLength, int suffixLength) {
+        final Element dom = xml.getDOM();
+        if (dom == null) {
+            return null;
+        }
+        return text(dom, prefixLength, suffixLength);
+    }
+
+    public String text(XMLObject xml, int length) {
+        return text(xml, length, 0);
+    }
+
+    protected static String text(Element dom, int prefixLength, int suffixLength) {
+
+        final String text = dom.getTextContent().trim();
+        final int totalLength = prefixLength + suffixLength;
+        if (text.length() > totalLength) {
+            final String prefix = Strings.cleanTruncate(text, prefixLength) + "...";
+            if (suffixLength == 0) {
+                return prefix;
+            }
+            int suffixIndex = text.length() - suffixLength;
+            if (Character.isHighSurrogate(text.charAt(suffixIndex))) {
+                suffixIndex++;
+            }
+            return prefix + text.substring(suffixIndex);
+        } else {
+            return text;
+        }
+    }
+
+    public String describeCredentials(Collection<? extends Credential> credentials) {
+        return credentials.stream()
+            .map(c -> {
+                if (c == null) {
+                    return "<null>";
+                }
+                byte[] encoded;
+                if (c instanceof X509Credential) {
+                    X509Credential x = (X509Credential) c;
+                    try {
+                        encoded = x.getEntityCertificate().getEncoded();
+                    } catch (CertificateEncodingException e) {
+                        encoded = c.getPublicKey().getEncoded();
+                    }
+                } else {
+                    encoded = c.getPublicKey().getEncoded();
+                }
+                return Base64.getEncoder().encodeToString(encoded).substring(0, 64) + "...";
+            })
+            .collect(Collectors.joining(","));
+    }
+
+    public Element toDomElement(XMLObject object) {
+        try {
+            return XMLObjectSupport.marshall(object);
+        } catch (MarshallingException e) {
+            throw new ElasticsearchSecurityException("failed to marshall SAML object to DOM element", e);
+        }
+    }
+
+
+    @SuppressForbidden(reason = "This is the only allowed way to construct a Transformer")
+    public Transformer getHardenedXMLTransformer() throws TransformerConfigurationException {
+        final TransformerFactory tfactory = TransformerFactory.newInstance();
+        tfactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        tfactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+        tfactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+        tfactory.setAttribute("indent-number", 2);
+        Transformer transformer = tfactory.newTransformer();
+        transformer.setErrorListener(new SamlFactory.TransformerErrorListener());
+        return transformer;
+    }
+
+    /**
+     * Constructs a DocumentBuilder with all the necessary features for it to be secure
+     *
+     * @throws ParserConfigurationException if one of the features can't be set on the DocumentBuilderFactory
+     */
+    @SuppressForbidden(reason = "This is the only allowed way to construct a DocumentBuilder")
+    public static DocumentBuilder getHardenedBuilder(String[] schemaFiles) throws ParserConfigurationException {
+        final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        dbf.setNamespaceAware(true);
+        // Ensure that Schema Validation is enabled for the factory
+        dbf.setValidating(true);
+        // Disallow internal and external entity expansion
+        dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+        dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
+        dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        dbf.setFeature("http://xml.org/sax/features/validation", true);
+        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false);
+        dbf.setIgnoringComments(true);
+        // This is required, otherwise schema validation causes signature invalidation
+        dbf.setFeature("http://apache.org/xml/features/validation/schema/normalized-value", false);
+        // Make sure that URL schema namespaces are not resolved/downloaded from URLs we do not control
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "file,jar");
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "file,jar");
+        dbf.setFeature("http://apache.org/xml/features/honour-all-schemaLocations", true);
+        // Ensure we do not resolve XIncludes. Defaults to false, but set it explicitly to be future-proof
+        dbf.setXIncludeAware(false);
+        // Ensure we do not expand entity reference nodes
+        dbf.setExpandEntityReferences(false);
+        // Further limit danger from denial of service attacks
+        dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        dbf.setAttribute("http://apache.org/xml/features/validation/schema", true);
+        dbf.setAttribute("http://apache.org/xml/features/validation/schema-full-checking", true);
+        dbf.setAttribute("http://java.sun.com/xml/jaxp/properties/schemaLanguage",
+            XMLConstants.W3C_XML_SCHEMA_NS_URI);
+        // We ship our own xsd files for schema validation since we do not trust anyone else.
+        dbf.setAttribute("http://java.sun.com/xml/jaxp/properties/schemaSource", resolveSchemaFilePaths(schemaFiles));
+        DocumentBuilder documentBuilder = dbf.newDocumentBuilder();
+        documentBuilder.setErrorHandler(new SamlFactory.DocumentBuilderErrorHandler());
+        return documentBuilder;
+    }
+
+    public String getJavaAlorithmNameFromUri(String sigAlg) {
+        switch (sigAlg) {
+            case "http://www.w3.org/2000/09/xmldsig#dsa-sha1":
+                return "SHA1withDSA";
+            case "http://www.w3.org/2000/09/xmldsig#dsa-sha256":
+                return "SHA256withDSA";
+            case "http://www.w3.org/2000/09/xmldsig#rsa-sha1":
+                return "SHA1withRSA";
+            case "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256":
+                return "SHA256withRSA";
+            case "http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha256":
+                return "SHA256withECDSA";
+            default:
+                throw new IllegalArgumentException("Unsupported signing algorithm identifier: " + sigAlg);
+        }
+    }
+
+
+    private static String[] resolveSchemaFilePaths(String[] relativePaths) {
+
+        return Arrays.stream(relativePaths).
+            map(file -> {
+                try {
+                    return SamlFactory.class.getResource(file).toURI().toString();
+                } catch (URISyntaxException e) {
+                    LOGGER.warn("Error resolving schema file path", e);
+                    return null;
+                }
+            }).filter(Objects::nonNull).toArray(String[]::new);
+    }
+
+    private static class DocumentBuilderErrorHandler implements org.xml.sax.ErrorHandler {
+        /**
+         * Enabling schema validation with `setValidating(true)` in our
+         * DocumentBuilderFactory requires that we provide our own
+         * ErrorHandler implementation
+         *
+         * @throws SAXException If the document we attempt to parse is not valid according to the specified schema.
+         */
+        @Override
+        public void warning(SAXParseException e) throws SAXException {
+            LOGGER.debug("XML Parser error ", e);
+            throw e;
+        }
+
+        @Override
+        public void error(SAXParseException e) throws SAXException {
+            warning(e);
+        }
+
+        @Override
+        public void fatalError(SAXParseException e) throws SAXException {
+            warning(e);
+        }
+    }
+
+    private static class TransformerErrorListener implements javax.xml.transform.ErrorListener {
+
+        @Override
+        public void warning(TransformerException e) throws TransformerException {
+            LOGGER.debug("XML transformation error", e);
+            throw e;
+        }
+
+        @Override
+        public void error(TransformerException e) throws TransformerException {
+            warning(e);
+        }
+
+        @Override
+        public void fatalError(TransformerException e) throws TransformerException {
+            warning(e);
+        }
+    }
+
+}
+
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlInit.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlInit.java
new file mode 100644
index 0000000000000..e6bfe1db8b219
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlInit.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.xpack.core.security.support.RestorableContextClassLoader;
+import org.opensaml.core.config.InitializationService;
+import org.slf4j.LoggerFactory;
+
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public final class SamlInit {
+
+    private static final AtomicBoolean INITIALISED = new AtomicBoolean(false);
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    private SamlInit() { }
+
+    /**
+     * This is needed in order to initialize the underlying OpenSAML library.
+     * It must be called before doing anything that potentially interacts with OpenSAML (whether in server code, or in tests).
+     * The initialization happens within do privileged block as the underlying Apache XML security library has a permission check.
+     * The initialization happens with a specific context classloader as OpenSAML loads resources from its jar file.
+     */
+    public static void initialize() {
+        if (INITIALISED.compareAndSet(false, true)) {
+            // We want to force these classes to be loaded _before_ we fiddle with the context classloader
+            LoggerFactory.getLogger(InitializationService.class);
+            SpecialPermission.check();
+            try {
+                AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                    LOGGER.debug("Initializing OpenSAML");
+                    try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(InitializationService.class)) {
+                        InitializationService.initialize();
+                    }
+                    LOGGER.debug("Initialized OpenSAML");
+                    return null;
+                });
+            } catch (PrivilegedActionException e) {
+                throw new ElasticsearchSecurityException("failed to set context classloader for SAML IdP", e);
+            }
+        }
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSigner.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSigner.java
new file mode 100644
index 0000000000000..d401e6bfc1b9d
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSigner.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.elasticsearch.xpack.core.security.support.RestorableContextClassLoader;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.opensaml.xmlsec.signature.SignableXMLObject;
+import org.opensaml.xmlsec.signature.Signature;
+import org.opensaml.xmlsec.signature.support.SignatureConstants;
+import org.opensaml.xmlsec.signature.support.SignatureException;
+import org.opensaml.xmlsec.signature.support.Signer;
+import org.opensaml.xmlsec.signature.support.SignerProvider;
+import org.w3c.dom.Element;
+
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * Signs OpenSAML {@link SignableXMLObject} instances using {@link SamlIdentityProvider#getSigningCredential()}.
+ */
+public class SamlObjectSigner {
+
+    private final SamlIdentityProvider idp;
+    private final SamlFactory samlFactory;
+
+    public SamlObjectSigner(SamlFactory samlFactory, SamlIdentityProvider idp) {
+        SamlInit.initialize();
+        this.idp = idp;
+        this.samlFactory = samlFactory;
+    }
+
+    public Element sign(SignableXMLObject object) {
+        final Signature signature = samlFactory.buildObject(Signature.class, Signature.DEFAULT_ELEMENT_NAME);
+        signature.setSigningCredential(idp.getSigningCredential());
+        signature.setSignatureAlgorithm(SignatureConstants.ALGO_ID_SIGNATURE_RSA_SHA256);
+        signature.setCanonicalizationAlgorithm(SignatureConstants.ALGO_ID_C14N_EXCL_OMIT_COMMENTS);
+        object.setSignature(signature);
+        Element element = samlFactory.toDomElement(object);
+        try {
+            AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(SignerProvider.class)) {
+                    Signer.signObject(signature);
+                } catch (SignatureException e) {
+                    throw new SecurityException("failed to sign SAML object " + object, e);
+                }
+                return null;
+            });
+        } catch (PrivilegedActionException e) {
+            throw new SecurityException("failed to sign SAML object " + object, e);
+        }
+        return element;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/XmlValidator.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/XmlValidator.java
new file mode 100644
index 0000000000000..023c893dfb666
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/support/XmlValidator.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.w3c.dom.bootstrap.DOMImplementationRegistry;
+import org.w3c.dom.ls.DOMImplementationLS;
+import org.w3c.dom.ls.LSInput;
+import org.w3c.dom.ls.LSResourceResolver;
+
+import javax.xml.XMLConstants;
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+import javax.xml.validation.Validator;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Validates an XML stream against a specified schema.
+ */
+public class XmlValidator {
+
+    private final SchemaFactory schemaFactory;
+    private final String xsdName;
+
+    public XmlValidator(String xsdName) {
+        this.schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+        this.xsdName = xsdName;
+    }
+
+    public void validate(String xml) throws Exception {
+        try(InputStream stream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) {
+            validate(stream);
+        }
+    }
+
+    public void validate(InputStream xml) throws Exception {
+        try (InputStream xsdStream = loadSchema(xsdName);
+             ResourceResolver resolver = new ResourceResolver()) {
+            schemaFactory.setResourceResolver(resolver);
+            Schema schema = schemaFactory.newSchema(new StreamSource(xsdStream));
+            Validator validator = schema.newValidator();
+            validator.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+            validator.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+            validator.validate(new StreamSource(xml));
+        }
+    }
+
+    private InputStream loadSchema(String name) {
+        if (name.endsWith(".xsd") && name.indexOf('/') == -1 && name.indexOf('\\') == -1) {
+            return getClass().getResourceAsStream(name);
+        } else {
+            return null;
+        }
+    }
+
+    private class ResourceResolver implements LSResourceResolver, AutoCloseable {
+        private final DOMImplementationLS domLS;
+        private final List<InputStream> streams;
+
+        private ResourceResolver() throws InstantiationException, IllegalAccessException, ClassNotFoundException {
+            // Seriously, who thought this was a good idea for an API ???
+            DOMImplementationRegistry registry = DOMImplementationRegistry.newInstance();
+            domLS = (DOMImplementationLS) registry.getDOMImplementation("LS");
+            streams = new ArrayList<>();
+        }
+
+        @Override
+        public LSInput resolveResource(String type, String namespaceURI, String publicId, String systemId, String baseURI) {
+            InputStream stream = loadSchema(systemId);
+            if (stream == null) {
+                return null;
+            }
+            streams.add(stream);
+            final LSInput input = domLS.createLSInput();
+            input.setByteStream(stream);
+            return input;
+        }
+
+        @Override
+        public void close() throws IOException {
+            IOUtils.close(streams);
+        }
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 0000000000000..65334a8db5760
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
@@ -0,0 +1,29 @@
+grant {
+  permission java.lang.RuntimePermission "setFactory";
+
+  // ApacheXMLSecurityInitializer
+  permission java.util.PropertyPermission "org.apache.xml.security.ignoreLineBreaks", "read,write";
+
+  // needed because of SAML (cf. o.e.x.c.s.s.RestorableContextClassLoader)
+  permission java.lang.RuntimePermission "getClassLoader";
+  permission java.lang.RuntimePermission "setContextClassLoader";
+
+  // needed during initialization of OpenSAML library where xml security algorithms are registered
+  // see https://github.com/apache/santuario-java/blob/e79f1fe4192de73a975bc7246aee58ed0703343d/src/main/java/org/apache/xml/security/utils/JavaUtils.java#L205-L220
+  // and https://git.shibboleth.net/view/?p=java-opensaml.git;a=blob;f=opensaml-xmlsec-impl/src/main/java/org/opensaml/xmlsec/signature/impl/SignatureMarshaller.java;hb=db0eaa64210f0e32d359cd6c57bedd57902bf811#l52
+  // which uses it in the opensaml-xmlsec-impl
+  permission java.security.SecurityPermission "org.apache.xml.security.register";
+
+  // needed for multiple server implementations used in tests
+  permission java.net.SocketPermission "*", "accept,connect";
+};
+
+grant codeBase "${codebase.elasticsearch-rest-client}" {
+  // rest client uses system properties which gets the default proxy
+  permission java.net.NetPermission "getProxySelector";
+};
+
+grant codeBase "${codebase.httpasyncclient}" {
+  // rest client uses system properties which gets the default proxy
+  permission java.net.NetPermission "getProxySelector";
+};
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-assertion-2.0.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-assertion-2.0.xsd
new file mode 100644
index 0000000000000..759baf8993b3e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-assertion-2.0.xsd
@@ -0,0 +1,283 @@
+<?xml version="1.0" encoding="US-ASCII"?>
+<schema
+    targetNamespace="urn:oasis:names:tc:SAML:2.0:assertion"
+    xmlns="http://www.w3.org/2001/XMLSchema"
+    xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion"
+    xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+    xmlns:xenc="http://www.w3.org/2001/04/xmlenc#"
+    elementFormDefault="unqualified"
+    attributeFormDefault="unqualified"
+    blockDefault="substitution"
+    version="2.0">
+    <import namespace="http://www.w3.org/2000/09/xmldsig#"
+        schemaLocation="xmldsig-core-schema.xsd"/>
+    <import namespace="http://www.w3.org/2001/04/xmlenc#"
+        schemaLocation="xenc-schema.xsd"/>
+    <annotation>
+        <documentation>
+            Document identifier: saml-schema-assertion-2.0
+            Location: http://docs.oasis-open.org/security/saml/v2.0/
+            Revision history:
+            V1.0 (November, 2002):
+              Initial Standard Schema.
+            V1.1 (September, 2003):
+              Updates within the same V1.0 namespace.
+            V2.0 (March, 2005):
+              New assertion schema for SAML V2.0 namespace.
+        </documentation>
+    </annotation>
+    <attributeGroup name="IDNameQualifiers">
+        <attribute name="NameQualifier" type="string" use="optional"/>
+        <attribute name="SPNameQualifier" type="string" use="optional"/>
+    </attributeGroup>
+    <element name="BaseID" type="saml:BaseIDAbstractType"/>
+    <complexType name="BaseIDAbstractType" abstract="true">
+        <attributeGroup ref="saml:IDNameQualifiers"/>
+    </complexType>
+    <element name="NameID" type="saml:NameIDType"/>
+    <complexType name="NameIDType">
+        <simpleContent>
+            <extension base="string">
+                <attributeGroup ref="saml:IDNameQualifiers"/>
+                <attribute name="Format" type="anyURI" use="optional"/>
+                <attribute name="SPProvidedID" type="string" use="optional"/>
+            </extension>
+        </simpleContent>
+    </complexType>
+    <complexType name="EncryptedElementType">
+        <sequence>
+            <element ref="xenc:EncryptedData"/>
+            <element ref="xenc:EncryptedKey" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+    </complexType>
+    <element name="EncryptedID" type="saml:EncryptedElementType"/>
+    <element name="Issuer" type="saml:NameIDType"/>
+    <element name="AssertionIDRef" type="NCName"/>
+    <element name="AssertionURIRef" type="anyURI"/>
+    <element name="Assertion" type="saml:AssertionType"/>
+    <complexType name="AssertionType">
+        <sequence>
+            <element ref="saml:Issuer"/>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="saml:Subject" minOccurs="0"/>
+            <element ref="saml:Conditions" minOccurs="0"/>
+            <element ref="saml:Advice" minOccurs="0"/>
+            <choice minOccurs="0" maxOccurs="unbounded">
+                <element ref="saml:Statement"/>
+                <element ref="saml:AuthnStatement"/>
+                <element ref="saml:AuthzDecisionStatement"/>
+                <element ref="saml:AttributeStatement"/>
+            </choice>
+        </sequence>
+        <attribute name="Version" type="string" use="required"/>
+        <attribute name="ID" type="ID" use="required"/>
+        <attribute name="IssueInstant" type="dateTime" use="required"/>
+    </complexType>
+    <element name="Subject" type="saml:SubjectType"/>
+    <complexType name="SubjectType">
+        <choice>
+            <sequence>
+                <choice>
+                    <element ref="saml:BaseID"/>
+                    <element ref="saml:NameID"/>
+                    <element ref="saml:EncryptedID"/>
+                </choice>
+                <element ref="saml:SubjectConfirmation" minOccurs="0" maxOccurs="unbounded"/>
+            </sequence>
+            <element ref="saml:SubjectConfirmation" maxOccurs="unbounded"/>
+        </choice>
+    </complexType>
+    <element name="SubjectConfirmation" type="saml:SubjectConfirmationType"/>
+    <complexType name="SubjectConfirmationType">
+        <sequence>
+            <choice minOccurs="0">
+                <element ref="saml:BaseID"/>
+                <element ref="saml:NameID"/>
+                <element ref="saml:EncryptedID"/>
+            </choice>
+            <element ref="saml:SubjectConfirmationData" minOccurs="0"/>
+        </sequence>
+        <attribute name="Method" type="anyURI" use="required"/>
+    </complexType>
+    <element name="SubjectConfirmationData" type="saml:SubjectConfirmationDataType"/>
+    <complexType name="SubjectConfirmationDataType" mixed="true">
+        <complexContent>
+            <restriction base="anyType">
+                <sequence>
+                    <any namespace="##any" processContents="lax" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+                <attribute name="NotBefore" type="dateTime" use="optional"/>
+                <attribute name="NotOnOrAfter" type="dateTime" use="optional"/>
+                <attribute name="Recipient" type="anyURI" use="optional"/>
+                <attribute name="InResponseTo" type="NCName" use="optional"/>
+                <attribute name="Address" type="string" use="optional"/>
+                <anyAttribute namespace="##other" processContents="lax"/>
+            </restriction>
+        </complexContent>
+    </complexType>
+    <complexType name="KeyInfoConfirmationDataType" mixed="false">
+        <complexContent>
+            <restriction base="saml:SubjectConfirmationDataType">
+                <sequence>
+                    <element ref="ds:KeyInfo" maxOccurs="unbounded"/>
+                </sequence>
+            </restriction>
+        </complexContent>
+    </complexType>
+    <element name="Conditions" type="saml:ConditionsType"/>
+    <complexType name="ConditionsType">
+        <choice minOccurs="0" maxOccurs="unbounded">
+            <element ref="saml:Condition"/>
+            <element ref="saml:AudienceRestriction"/>
+            <element ref="saml:OneTimeUse"/>
+            <element ref="saml:ProxyRestriction"/>
+        </choice>
+        <attribute name="NotBefore" type="dateTime" use="optional"/>
+        <attribute name="NotOnOrAfter" type="dateTime" use="optional"/>
+    </complexType>
+    <element name="Condition" type="saml:ConditionAbstractType"/>
+    <complexType name="ConditionAbstractType" abstract="true"/>
+    <element name="AudienceRestriction" type="saml:AudienceRestrictionType"/>
+    <complexType name="AudienceRestrictionType">
+        <complexContent>
+            <extension base="saml:ConditionAbstractType">
+                <sequence>
+                    <element ref="saml:Audience" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="Audience" type="anyURI"/>
+    <element name="OneTimeUse" type="saml:OneTimeUseType" />
+    <complexType name="OneTimeUseType">
+        <complexContent>
+            <extension base="saml:ConditionAbstractType"/>
+        </complexContent>
+    </complexType>
+    <element name="ProxyRestriction" type="saml:ProxyRestrictionType"/>
+    <complexType name="ProxyRestrictionType">
+    <complexContent>
+        <extension base="saml:ConditionAbstractType">
+            <sequence>
+                <element ref="saml:Audience" minOccurs="0" maxOccurs="unbounded"/>
+            </sequence>
+            <attribute name="Count" type="nonNegativeInteger" use="optional"/>
+        </extension>
+    </complexContent>
+    </complexType>
+    <element name="Advice" type="saml:AdviceType"/>
+    <complexType name="AdviceType">
+        <choice minOccurs="0" maxOccurs="unbounded">
+            <element ref="saml:AssertionIDRef"/>
+            <element ref="saml:AssertionURIRef"/>
+            <element ref="saml:Assertion"/>
+            <element ref="saml:EncryptedAssertion"/>
+            <any namespace="##other" processContents="lax"/>
+        </choice>
+    </complexType>
+    <element name="EncryptedAssertion" type="saml:EncryptedElementType"/>
+    <element name="Statement" type="saml:StatementAbstractType"/>
+    <complexType name="StatementAbstractType" abstract="true"/>
+    <element name="AuthnStatement" type="saml:AuthnStatementType"/>
+    <complexType name="AuthnStatementType">
+        <complexContent>
+            <extension base="saml:StatementAbstractType">
+                <sequence>
+                    <element ref="saml:SubjectLocality" minOccurs="0"/>
+                    <element ref="saml:AuthnContext"/>
+                </sequence>
+                <attribute name="AuthnInstant" type="dateTime" use="required"/>
+                <attribute name="SessionIndex" type="string" use="optional"/>
+                <attribute name="SessionNotOnOrAfter" type="dateTime" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="SubjectLocality" type="saml:SubjectLocalityType"/>
+    <complexType name="SubjectLocalityType">
+        <attribute name="Address" type="string" use="optional"/>
+        <attribute name="DNSName" type="string" use="optional"/>
+    </complexType>
+    <element name="AuthnContext" type="saml:AuthnContextType"/>
+    <complexType name="AuthnContextType">
+        <sequence>
+            <choice>
+                <sequence>
+                    <element ref="saml:AuthnContextClassRef"/>
+                    <choice minOccurs="0">
+                        <element ref="saml:AuthnContextDecl"/>
+                        <element ref="saml:AuthnContextDeclRef"/>
+                    </choice>
+                </sequence>
+                <choice>
+                    <element ref="saml:AuthnContextDecl"/>
+                    <element ref="saml:AuthnContextDeclRef"/>
+                </choice>
+            </choice>
+            <element ref="saml:AuthenticatingAuthority" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+    </complexType>
+    <element name="AuthnContextClassRef" type="anyURI"/>
+    <element name="AuthnContextDeclRef" type="anyURI"/>
+    <element name="AuthnContextDecl" type="anyType"/>
+    <element name="AuthenticatingAuthority" type="anyURI"/>
+    <element name="AuthzDecisionStatement" type="saml:AuthzDecisionStatementType"/>
+    <complexType name="AuthzDecisionStatementType">
+        <complexContent>
+            <extension base="saml:StatementAbstractType">
+                <sequence>
+                    <element ref="saml:Action" maxOccurs="unbounded"/>
+                    <element ref="saml:Evidence" minOccurs="0"/>
+                </sequence>
+                <attribute name="Resource" type="anyURI" use="required"/>
+                <attribute name="Decision" type="saml:DecisionType" use="required"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <simpleType name="DecisionType">
+        <restriction base="string">
+            <enumeration value="Permit"/>
+            <enumeration value="Deny"/>
+            <enumeration value="Indeterminate"/>
+        </restriction>
+    </simpleType>
+    <element name="Action" type="saml:ActionType"/>
+    <complexType name="ActionType">
+        <simpleContent>
+            <extension base="string">
+                <attribute name="Namespace" type="anyURI" use="required"/>
+            </extension>
+        </simpleContent>
+    </complexType>
+    <element name="Evidence" type="saml:EvidenceType"/>
+    <complexType name="EvidenceType">
+        <choice maxOccurs="unbounded">
+            <element ref="saml:AssertionIDRef"/>
+            <element ref="saml:AssertionURIRef"/>
+            <element ref="saml:Assertion"/>
+            <element ref="saml:EncryptedAssertion"/>
+        </choice>
+    </complexType>
+    <element name="AttributeStatement" type="saml:AttributeStatementType"/>
+    <complexType name="AttributeStatementType">
+        <complexContent>
+            <extension base="saml:StatementAbstractType">
+                <choice maxOccurs="unbounded">
+                    <element ref="saml:Attribute"/>
+                    <element ref="saml:EncryptedAttribute"/>
+                </choice>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="Attribute" type="saml:AttributeType"/>
+    <complexType name="AttributeType">
+        <sequence>
+            <element ref="saml:AttributeValue" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="Name" type="string" use="required"/>
+        <attribute name="NameFormat" type="anyURI" use="optional"/>
+        <attribute name="FriendlyName" type="string" use="optional"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    <element name="AttributeValue" type="anyType" nillable="true"/>
+    <element name="EncryptedAttribute" type="saml:EncryptedElementType"/>
+</schema>
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-metadata-2.0.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-metadata-2.0.xsd
new file mode 100644
index 0000000000000..9d5e4832a1ada
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-metadata-2.0.xsd
@@ -0,0 +1,337 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema
+    targetNamespace="urn:oasis:names:tc:SAML:2.0:metadata"
+    xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata"
+    xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+    xmlns:xenc="http://www.w3.org/2001/04/xmlenc#"
+    xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion"
+    xmlns="http://www.w3.org/2001/XMLSchema"
+    elementFormDefault="unqualified"
+    attributeFormDefault="unqualified"
+    blockDefault="substitution"
+    version="2.0">
+    <import namespace="http://www.w3.org/2000/09/xmldsig#"
+        schemaLocation="xmldsig-core-schema.xsd"/>
+    <import namespace="http://www.w3.org/2001/04/xmlenc#"
+        schemaLocation="xenc-schema.xsd"/>
+    <import namespace="urn:oasis:names:tc:SAML:2.0:assertion"
+        schemaLocation="saml-schema-assertion-2.0.xsd"/>
+    <import namespace="http://www.w3.org/XML/1998/namespace"
+        schemaLocation="xml.xsd" />
+    <annotation>
+        <documentation>
+            Document identifier: saml-schema-metadata-2.0
+            Location: http://docs.oasis-open.org/security/saml/v2.0/
+            Revision history:
+              V2.0 (March, 2005):
+                Schema for SAML metadata, first published in SAML 2.0.
+        </documentation>
+    </annotation>
+
+    <simpleType name="entityIDType">
+        <restriction base="anyURI">
+            <maxLength value="1024"/>
+        </restriction>
+    </simpleType>
+    <complexType name="localizedNameType">
+        <simpleContent>
+            <extension base="string">
+                <attribute ref="xml:lang" use="required"/>
+            </extension>
+        </simpleContent>
+    </complexType>
+    <complexType name="localizedURIType">
+        <simpleContent>
+            <extension base="anyURI">
+                <attribute ref="xml:lang" use="required"/>
+            </extension>
+        </simpleContent>
+    </complexType>
+    
+    <element name="Extensions" type="md:ExtensionsType"/>
+    <complexType final="#all" name="ExtensionsType">
+        <sequence>
+            <any namespace="##other" processContents="lax" maxOccurs="unbounded"/>
+        </sequence>
+    </complexType>
+    
+    <complexType name="EndpointType">
+        <sequence>
+            <any namespace="##other" processContents="lax" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="Binding" type="anyURI" use="required"/>
+        <attribute name="Location" type="anyURI" use="required"/>
+        <attribute name="ResponseLocation" type="anyURI" use="optional"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    
+    <complexType name="IndexedEndpointType">
+        <complexContent>
+            <extension base="md:EndpointType">
+                <attribute name="index" type="unsignedShort" use="required"/>
+                <attribute name="isDefault" type="boolean" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    
+    <element name="EntitiesDescriptor" type="md:EntitiesDescriptorType"/>
+    <complexType name="EntitiesDescriptorType">
+        <sequence>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <choice minOccurs="1" maxOccurs="unbounded">
+                <element ref="md:EntityDescriptor"/>
+                <element ref="md:EntitiesDescriptor"/>
+            </choice>
+        </sequence>
+        <attribute name="validUntil" type="dateTime" use="optional"/>
+        <attribute name="cacheDuration" type="duration" use="optional"/>
+        <attribute name="ID" type="ID" use="optional"/>
+        <attribute name="Name" type="string" use="optional"/>
+    </complexType>
+
+    <element name="EntityDescriptor" type="md:EntityDescriptorType"/>
+    <complexType name="EntityDescriptorType">
+        <sequence>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <choice>
+                <choice maxOccurs="unbounded">
+                    <element ref="md:RoleDescriptor"/>
+                    <element ref="md:IDPSSODescriptor"/>
+                    <element ref="md:SPSSODescriptor"/>
+                    <element ref="md:AuthnAuthorityDescriptor"/>
+                    <element ref="md:AttributeAuthorityDescriptor"/>
+                    <element ref="md:PDPDescriptor"/>
+                </choice>
+                <element ref="md:AffiliationDescriptor"/>
+            </choice>
+            <element ref="md:Organization" minOccurs="0"/>
+            <element ref="md:ContactPerson" minOccurs="0" maxOccurs="unbounded"/>
+            <element ref="md:AdditionalMetadataLocation" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="entityID" type="md:entityIDType" use="required"/>
+        <attribute name="validUntil" type="dateTime" use="optional"/>
+        <attribute name="cacheDuration" type="duration" use="optional"/>
+        <attribute name="ID" type="ID" use="optional"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    
+    <element name="Organization" type="md:OrganizationType"/>
+    <complexType name="OrganizationType">
+        <sequence>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <element ref="md:OrganizationName" maxOccurs="unbounded"/>
+            <element ref="md:OrganizationDisplayName" maxOccurs="unbounded"/>
+            <element ref="md:OrganizationURL" maxOccurs="unbounded"/>
+        </sequence>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    <element name="OrganizationName" type="md:localizedNameType"/>
+    <element name="OrganizationDisplayName" type="md:localizedNameType"/>
+    <element name="OrganizationURL" type="md:localizedURIType"/>
+    <element name="ContactPerson" type="md:ContactType"/>
+    <complexType name="ContactType">
+        <sequence>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <element ref="md:Company" minOccurs="0"/>
+            <element ref="md:GivenName" minOccurs="0"/>
+            <element ref="md:SurName" minOccurs="0"/>
+            <element ref="md:EmailAddress" minOccurs="0" maxOccurs="unbounded"/>
+            <element ref="md:TelephoneNumber" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="contactType" type="md:ContactTypeType" use="required"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    <element name="Company" type="string"/>
+    <element name="GivenName" type="string"/>
+    <element name="SurName" type="string"/>
+    <element name="EmailAddress" type="anyURI"/>
+    <element name="TelephoneNumber" type="string"/>
+    <simpleType name="ContactTypeType">
+        <restriction base="string">
+            <enumeration value="technical"/>
+            <enumeration value="support"/>
+            <enumeration value="administrative"/>
+            <enumeration value="billing"/>
+            <enumeration value="other"/>
+        </restriction>
+    </simpleType>
+
+    <element name="AdditionalMetadataLocation" type="md:AdditionalMetadataLocationType"/>
+    <complexType name="AdditionalMetadataLocationType">
+        <simpleContent>
+            <extension base="anyURI">
+                <attribute name="namespace" type="anyURI" use="required"/>
+            </extension>
+        </simpleContent>
+    </complexType>
+
+    <element name="RoleDescriptor" type="md:RoleDescriptorType"/>
+    <complexType name="RoleDescriptorType" abstract="true">
+        <sequence>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <element ref="md:KeyDescriptor" minOccurs="0" maxOccurs="unbounded"/>
+            <element ref="md:Organization" minOccurs="0"/>
+            <element ref="md:ContactPerson" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="ID" type="ID" use="optional"/>
+        <attribute name="validUntil" type="dateTime" use="optional"/>
+        <attribute name="cacheDuration" type="duration" use="optional"/>
+        <attribute name="protocolSupportEnumeration" type="md:anyURIListType" use="required"/>
+        <attribute name="errorURL" type="anyURI" use="optional"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    <simpleType name="anyURIListType">
+        <list itemType="anyURI"/>
+    </simpleType>
+
+    <element name="KeyDescriptor" type="md:KeyDescriptorType"/>
+    <complexType name="KeyDescriptorType">
+        <sequence>
+            <element ref="ds:KeyInfo"/>
+            <element ref="md:EncryptionMethod" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="use" type="md:KeyTypes" use="optional"/>
+    </complexType>
+    <simpleType name="KeyTypes">
+        <restriction base="string">
+            <enumeration value="encryption"/>
+            <enumeration value="signing"/>
+        </restriction>
+    </simpleType>
+    <element name="EncryptionMethod" type="xenc:EncryptionMethodType"/>
+    
+    <complexType name="SSODescriptorType" abstract="true">
+        <complexContent>
+            <extension base="md:RoleDescriptorType">
+                <sequence>
+                    <element ref="md:ArtifactResolutionService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:SingleLogoutService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:ManageNameIDService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:NameIDFormat" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="ArtifactResolutionService" type="md:IndexedEndpointType"/>
+    <element name="SingleLogoutService" type="md:EndpointType"/>
+    <element name="ManageNameIDService" type="md:EndpointType"/>
+    <element name="NameIDFormat" type="anyURI"/>
+
+    <element name="IDPSSODescriptor" type="md:IDPSSODescriptorType"/>
+    <complexType name="IDPSSODescriptorType">
+        <complexContent>
+            <extension base="md:SSODescriptorType">
+                <sequence>
+                    <element ref="md:SingleSignOnService" maxOccurs="unbounded"/>
+                    <element ref="md:NameIDMappingService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:AssertionIDRequestService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:AttributeProfile" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="saml:Attribute" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+                <attribute name="WantAuthnRequestsSigned" type="boolean" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="SingleSignOnService" type="md:EndpointType"/>
+    <element name="NameIDMappingService" type="md:EndpointType"/>
+    <element name="AssertionIDRequestService" type="md:EndpointType"/>
+    <element name="AttributeProfile" type="anyURI"/>
+    
+    <element name="SPSSODescriptor" type="md:SPSSODescriptorType"/>
+    <complexType name="SPSSODescriptorType">
+        <complexContent>
+            <extension base="md:SSODescriptorType">
+                <sequence>
+                    <element ref="md:AssertionConsumerService" maxOccurs="unbounded"/>
+                    <element ref="md:AttributeConsumingService" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+                <attribute name="AuthnRequestsSigned" type="boolean" use="optional"/>
+                <attribute name="WantAssertionsSigned" type="boolean" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AssertionConsumerService" type="md:IndexedEndpointType"/>
+    <element name="AttributeConsumingService" type="md:AttributeConsumingServiceType"/>
+    <complexType name="AttributeConsumingServiceType">
+        <sequence>
+            <element ref="md:ServiceName" maxOccurs="unbounded"/>
+            <element ref="md:ServiceDescription" minOccurs="0" maxOccurs="unbounded"/>
+            <element ref="md:RequestedAttribute" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="index" type="unsignedShort" use="required"/>
+        <attribute name="isDefault" type="boolean" use="optional"/>
+    </complexType>
+    <element name="ServiceName" type="md:localizedNameType"/>
+    <element name="ServiceDescription" type="md:localizedNameType"/>
+    <element name="RequestedAttribute" type="md:RequestedAttributeType"/>
+    <complexType name="RequestedAttributeType">
+        <complexContent>
+            <extension base="saml:AttributeType">
+                <attribute name="isRequired" type="boolean" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+  
+    <element name="AuthnAuthorityDescriptor" type="md:AuthnAuthorityDescriptorType"/>
+    <complexType name="AuthnAuthorityDescriptorType">
+        <complexContent>
+            <extension base="md:RoleDescriptorType">
+                <sequence>
+                    <element ref="md:AuthnQueryService" maxOccurs="unbounded"/>
+                    <element ref="md:AssertionIDRequestService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:NameIDFormat" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AuthnQueryService" type="md:EndpointType"/>
+
+    <element name="PDPDescriptor" type="md:PDPDescriptorType"/>
+    <complexType name="PDPDescriptorType">
+        <complexContent>
+            <extension base="md:RoleDescriptorType">
+                <sequence>
+                    <element ref="md:AuthzService" maxOccurs="unbounded"/>
+                    <element ref="md:AssertionIDRequestService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:NameIDFormat" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AuthzService" type="md:EndpointType"/>
+
+    <element name="AttributeAuthorityDescriptor" type="md:AttributeAuthorityDescriptorType"/>
+    <complexType name="AttributeAuthorityDescriptorType">
+        <complexContent>
+            <extension base="md:RoleDescriptorType">
+                <sequence>
+                    <element ref="md:AttributeService" maxOccurs="unbounded"/>
+                    <element ref="md:AssertionIDRequestService" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:NameIDFormat" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="md:AttributeProfile" minOccurs="0" maxOccurs="unbounded"/>
+                    <element ref="saml:Attribute" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AttributeService" type="md:EndpointType"/>
+   
+    <element name="AffiliationDescriptor" type="md:AffiliationDescriptorType"/>
+    <complexType name="AffiliationDescriptorType">
+        <sequence>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="md:Extensions" minOccurs="0"/>
+            <element ref="md:AffiliateMember" maxOccurs="unbounded"/>
+            <element ref="md:KeyDescriptor" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="affiliationOwnerID" type="md:entityIDType" use="required"/>
+        <attribute name="validUntil" type="dateTime" use="optional"/>
+        <attribute name="cacheDuration" type="duration" use="optional"/>
+        <attribute name="ID" type="ID" use="optional"/>
+        <anyAttribute namespace="##other" processContents="lax"/>
+    </complexType>
+    <element name="AffiliateMember" type="md:entityIDType"/>
+</schema>
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-protocol-2.0.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-protocol-2.0.xsd
new file mode 100644
index 0000000000000..48ec69cbc967d
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/saml-schema-protocol-2.0.xsd
@@ -0,0 +1,302 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema
+    targetNamespace="urn:oasis:names:tc:SAML:2.0:protocol"
+    xmlns="http://www.w3.org/2001/XMLSchema"
+    xmlns:samlp="urn:oasis:names:tc:SAML:2.0:protocol"
+    xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion"
+    xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+    elementFormDefault="unqualified"
+    attributeFormDefault="unqualified"
+    blockDefault="substitution"
+    version="2.0">
+    <import namespace="urn:oasis:names:tc:SAML:2.0:assertion"
+        schemaLocation="saml-schema-assertion-2.0.xsd"/>
+    <import namespace="http://www.w3.org/2000/09/xmldsig#"
+        schemaLocation="xmldsig-core-schema.xsd"/>
+    <annotation>
+        <documentation>
+            Document identifier: saml-schema-protocol-2.0
+            Location: http://docs.oasis-open.org/security/saml/v2.0/
+            Revision history:
+            V1.0 (November, 2002):
+              Initial Standard Schema.
+            V1.1 (September, 2003):
+              Updates within the same V1.0 namespace.
+            V2.0 (March, 2005):
+              New protocol schema based in a SAML V2.0 namespace.
+     </documentation>
+    </annotation>
+    <complexType name="RequestAbstractType" abstract="true">
+        <sequence>
+            <element ref="saml:Issuer" minOccurs="0"/>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="samlp:Extensions" minOccurs="0"/>
+        </sequence>
+        <attribute name="ID" type="ID" use="required"/>
+        <attribute name="Version" type="string" use="required"/>
+        <attribute name="IssueInstant" type="dateTime" use="required"/>
+        <attribute name="Destination" type="anyURI" use="optional"/>
+        <attribute name="Consent" type="anyURI" use="optional"/>
+    </complexType>
+    <element name="Extensions" type="samlp:ExtensionsType"/>
+    <complexType name="ExtensionsType">
+        <sequence>
+            <any namespace="##other" processContents="lax" maxOccurs="unbounded"/>
+        </sequence>
+    </complexType>
+    <complexType name="StatusResponseType">
+        <sequence>
+            <element ref="saml:Issuer" minOccurs="0"/>
+            <element ref="ds:Signature" minOccurs="0"/>
+            <element ref="samlp:Extensions" minOccurs="0"/>
+            <element ref="samlp:Status"/>
+        </sequence>
+        <attribute name="ID" type="ID" use="required"/>
+        <attribute name="InResponseTo" type="NCName" use="optional"/>
+        <attribute name="Version" type="string" use="required"/>
+        <attribute name="IssueInstant" type="dateTime" use="required"/>
+        <attribute name="Destination" type="anyURI" use="optional"/>
+        <attribute name="Consent" type="anyURI" use="optional"/>
+    </complexType>
+    <element name="Status" type="samlp:StatusType"/>
+    <complexType name="StatusType">
+        <sequence>
+            <element ref="samlp:StatusCode"/>
+            <element ref="samlp:StatusMessage" minOccurs="0"/>
+            <element ref="samlp:StatusDetail" minOccurs="0"/>
+        </sequence>
+    </complexType>
+    <element name="StatusCode" type="samlp:StatusCodeType"/>
+    <complexType name="StatusCodeType">
+        <sequence>
+            <element ref="samlp:StatusCode" minOccurs="0"/>
+        </sequence>
+        <attribute name="Value" type="anyURI" use="required"/>
+    </complexType>
+    <element name="StatusMessage" type="string"/>
+    <element name="StatusDetail" type="samlp:StatusDetailType"/>
+    <complexType name="StatusDetailType">
+        <sequence>
+            <any namespace="##any" processContents="lax" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+    </complexType>
+    <element name="AssertionIDRequest" type="samlp:AssertionIDRequestType"/>
+    <complexType name="AssertionIDRequestType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <element ref="saml:AssertionIDRef" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="SubjectQuery" type="samlp:SubjectQueryAbstractType"/>
+    <complexType name="SubjectQueryAbstractType" abstract="true">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <element ref="saml:Subject"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AuthnQuery" type="samlp:AuthnQueryType"/>
+    <complexType name="AuthnQueryType">
+        <complexContent>
+            <extension base="samlp:SubjectQueryAbstractType">
+                <sequence>
+                    <element ref="samlp:RequestedAuthnContext" minOccurs="0"/>
+                </sequence>
+                <attribute name="SessionIndex" type="string" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="RequestedAuthnContext" type="samlp:RequestedAuthnContextType"/>
+    <complexType name="RequestedAuthnContextType">
+        <choice>
+            <element ref="saml:AuthnContextClassRef" maxOccurs="unbounded"/>
+            <element ref="saml:AuthnContextDeclRef" maxOccurs="unbounded"/>
+        </choice>
+        <attribute name="Comparison" type="samlp:AuthnContextComparisonType" use="optional"/>
+    </complexType>
+    <simpleType name="AuthnContextComparisonType">
+        <restriction base="string">
+            <enumeration value="exact"/>
+            <enumeration value="minimum"/>
+            <enumeration value="maximum"/>
+            <enumeration value="better"/>
+        </restriction>
+    </simpleType>
+    <element name="AttributeQuery" type="samlp:AttributeQueryType"/>
+    <complexType name="AttributeQueryType">
+        <complexContent>
+            <extension base="samlp:SubjectQueryAbstractType">
+                <sequence>
+                    <element ref="saml:Attribute" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AuthzDecisionQuery" type="samlp:AuthzDecisionQueryType"/>
+    <complexType name="AuthzDecisionQueryType">
+        <complexContent>
+            <extension base="samlp:SubjectQueryAbstractType">
+                <sequence>
+                    <element ref="saml:Action" maxOccurs="unbounded"/>
+                    <element ref="saml:Evidence" minOccurs="0"/>
+                </sequence>
+                <attribute name="Resource" type="anyURI" use="required"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="AuthnRequest" type="samlp:AuthnRequestType"/>
+    <complexType name="AuthnRequestType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <element ref="saml:Subject" minOccurs="0"/>
+                    <element ref="samlp:NameIDPolicy" minOccurs="0"/>
+                    <element ref="saml:Conditions" minOccurs="0"/>
+                    <element ref="samlp:RequestedAuthnContext" minOccurs="0"/>
+                    <element ref="samlp:Scoping" minOccurs="0"/>
+                </sequence>
+                <attribute name="ForceAuthn" type="boolean" use="optional"/>
+                <attribute name="IsPassive" type="boolean" use="optional"/>
+                <attribute name="ProtocolBinding" type="anyURI" use="optional"/>
+                <attribute name="AssertionConsumerServiceIndex" type="unsignedShort" use="optional"/>
+                <attribute name="AssertionConsumerServiceURL" type="anyURI" use="optional"/>
+                <attribute name="AttributeConsumingServiceIndex" type="unsignedShort" use="optional"/>
+                <attribute name="ProviderName" type="string" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="NameIDPolicy" type="samlp:NameIDPolicyType"/>
+    <complexType name="NameIDPolicyType">
+        <attribute name="Format" type="anyURI" use="optional"/>
+        <attribute name="SPNameQualifier" type="string" use="optional"/>
+        <attribute name="AllowCreate" type="boolean" use="optional"/>
+    </complexType>
+    <element name="Scoping" type="samlp:ScopingType"/>
+    <complexType name="ScopingType">
+        <sequence>
+            <element ref="samlp:IDPList" minOccurs="0"/>
+            <element ref="samlp:RequesterID" minOccurs="0" maxOccurs="unbounded"/>
+        </sequence>
+        <attribute name="ProxyCount" type="nonNegativeInteger" use="optional"/>
+    </complexType>
+    <element name="RequesterID" type="anyURI"/>
+    <element name="IDPList" type="samlp:IDPListType"/>
+    <complexType name="IDPListType">
+        <sequence>
+            <element ref="samlp:IDPEntry" maxOccurs="unbounded"/>
+            <element ref="samlp:GetComplete" minOccurs="0"/>
+        </sequence>
+    </complexType>
+    <element name="IDPEntry" type="samlp:IDPEntryType"/>
+    <complexType name="IDPEntryType">
+        <attribute name="ProviderID" type="anyURI" use="required"/>
+        <attribute name="Name" type="string" use="optional"/>
+        <attribute name="Loc" type="anyURI" use="optional"/>
+    </complexType>
+    <element name="GetComplete" type="anyURI"/>
+    <element name="Response" type="samlp:ResponseType"/>
+    <complexType name="ResponseType">
+        <complexContent>
+            <extension base="samlp:StatusResponseType">
+                <choice minOccurs="0" maxOccurs="unbounded">
+                    <element ref="saml:Assertion"/>
+                    <element ref="saml:EncryptedAssertion"/>
+                </choice>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="ArtifactResolve" type="samlp:ArtifactResolveType"/>
+    <complexType name="ArtifactResolveType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <element ref="samlp:Artifact"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="Artifact" type="string"/>
+    <element name="ArtifactResponse" type="samlp:ArtifactResponseType"/>
+    <complexType name="ArtifactResponseType">
+        <complexContent>
+            <extension base="samlp:StatusResponseType">
+                <sequence>
+                    <any namespace="##any" processContents="lax" minOccurs="0"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="ManageNameIDRequest" type="samlp:ManageNameIDRequestType"/>
+    <complexType name="ManageNameIDRequestType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <choice>
+                        <element ref="saml:NameID"/>
+                        <element ref="saml:EncryptedID"/>
+                    </choice>
+                    <choice>
+                        <element ref="samlp:NewID"/>
+                        <element ref="samlp:NewEncryptedID"/>
+                        <element ref="samlp:Terminate"/>
+                    </choice>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="NewID" type="string"/>
+    <element name="NewEncryptedID" type="saml:EncryptedElementType"/>
+    <element name="Terminate" type="samlp:TerminateType"/>
+    <complexType name="TerminateType"/>
+    <element name="ManageNameIDResponse" type="samlp:StatusResponseType"/>
+    <element name="LogoutRequest" type="samlp:LogoutRequestType"/>
+    <complexType name="LogoutRequestType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <choice>
+                        <element ref="saml:BaseID"/>
+                        <element ref="saml:NameID"/>
+                        <element ref="saml:EncryptedID"/>
+                    </choice>
+                    <element ref="samlp:SessionIndex" minOccurs="0" maxOccurs="unbounded"/>
+                </sequence>
+                <attribute name="Reason" type="string" use="optional"/>
+                <attribute name="NotOnOrAfter" type="dateTime" use="optional"/>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="SessionIndex" type="string"/>
+    <element name="LogoutResponse" type="samlp:StatusResponseType"/>
+    <element name="NameIDMappingRequest" type="samlp:NameIDMappingRequestType"/>
+    <complexType name="NameIDMappingRequestType">
+        <complexContent>
+            <extension base="samlp:RequestAbstractType">
+                <sequence>
+                    <choice>
+                        <element ref="saml:BaseID"/>
+                        <element ref="saml:NameID"/>
+                        <element ref="saml:EncryptedID"/>
+                    </choice>
+                    <element ref="samlp:NameIDPolicy"/>
+                </sequence>
+            </extension>
+        </complexContent>
+    </complexType>
+    <element name="NameIDMappingResponse" type="samlp:NameIDMappingResponseType"/>
+    <complexType name="NameIDMappingResponseType">
+        <complexContent>
+            <extension base="samlp:StatusResponseType">
+                <choice>
+                    <element ref="saml:NameID"/>
+                    <element ref="saml:EncryptedID"/>
+                </choice>
+            </extension>
+        </complexContent>
+    </complexType>
+</schema>
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xenc-schema.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xenc-schema.xsd
new file mode 100644
index 0000000000000..c902d4fc60772
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xenc-schema.xsd
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="utf-8"?>
+<schema xmlns='http://www.w3.org/2001/XMLSchema' version='1.0'
+        xmlns:xenc='http://www.w3.org/2001/04/xmlenc#'
+        xmlns:ds='http://www.w3.org/2000/09/xmldsig#'
+        targetNamespace='http://www.w3.org/2001/04/xmlenc#'
+        elementFormDefault='qualified'>
+
+  <import namespace='http://www.w3.org/2000/09/xmldsig#'
+          schemaLocation='xmldsig-core-schema.xsd'/>
+
+  <complexType name='EncryptedType' abstract='true'>
+    <sequence>
+      <element name='EncryptionMethod' type='xenc:EncryptionMethodType'
+       minOccurs='0'/>
+      <element ref='ds:KeyInfo' minOccurs='0'/>
+      <element ref='xenc:CipherData'/>
+      <element ref='xenc:EncryptionProperties' minOccurs='0'/>
+    </sequence>
+    <attribute name='Id' type='ID' use='optional'/>
+    <attribute name='Type' type='anyURI' use='optional'/>
+    <attribute name='MimeType' type='string' use='optional'/>
+    <attribute name='Encoding' type='anyURI' use='optional'/>
+  </complexType>
+  
+  <complexType name='EncryptionMethodType' mixed='true'>
+    <sequence>
+      <element name='KeySize' minOccurs='0' type='xenc:KeySizeType'/>
+      <element name='OAEPparams' minOccurs='0' type='base64Binary'/>
+      <any namespace='##other' minOccurs='0' maxOccurs='unbounded'/>
+    </sequence>
+    <attribute name='Algorithm' type='anyURI' use='required'/>
+  </complexType>
+
+    <simpleType name='KeySizeType'>
+      <restriction base="integer"/>
+    </simpleType>
+
+  <element name='CipherData' type='xenc:CipherDataType'/>
+  <complexType name='CipherDataType'>
+     <choice>
+       <element name='CipherValue' type='base64Binary'/>
+       <element ref='xenc:CipherReference'/>
+     </choice>
+    </complexType>
+
+   <element name='CipherReference' type='xenc:CipherReferenceType'/>
+   <complexType name='CipherReferenceType'>
+       <choice>
+         <element name='Transforms' type='xenc:TransformsType' minOccurs='0'/>
+       </choice>
+       <attribute name='URI' type='anyURI' use='required'/>
+   </complexType>
+
+     <complexType name='TransformsType'>
+       <sequence>
+         <element ref='ds:Transform' maxOccurs='unbounded'/>
+       </sequence>
+     </complexType>
+
+
+  <element name='EncryptedData' type='xenc:EncryptedDataType'/>
+  <complexType name='EncryptedDataType'>
+    <complexContent>
+      <extension base='xenc:EncryptedType'>
+       </extension>
+    </complexContent>
+  </complexType>
+
+  <!-- Children of ds:KeyInfo -->
+
+  <element name='EncryptedKey' type='xenc:EncryptedKeyType'/>
+  <complexType name='EncryptedKeyType'>
+    <complexContent>
+      <extension base='xenc:EncryptedType'>
+        <sequence>
+          <element ref='xenc:ReferenceList' minOccurs='0'/>
+          <element name='CarriedKeyName' type='string' minOccurs='0'/>
+        </sequence>
+        <attribute name='Recipient' type='string'
+         use='optional'/>
+      </extension>
+    </complexContent>
+  </complexType>
+
+    <element name="AgreementMethod" type="xenc:AgreementMethodType"/>
+    <complexType name="AgreementMethodType" mixed="true">
+      <sequence>
+        <element name="KA-Nonce" minOccurs="0" type="base64Binary"/>
+        <!-- <element ref="ds:DigestMethod" minOccurs="0"/> -->
+        <any namespace="##other" minOccurs="0" maxOccurs="unbounded"/>
+        <element name="OriginatorKeyInfo" minOccurs="0" type="ds:KeyInfoType"/>
+        <element name="RecipientKeyInfo" minOccurs="0" type="ds:KeyInfoType"/>
+      </sequence>
+      <attribute name="Algorithm" type="anyURI" use="required"/>
+    </complexType>
+
+  <!-- End Children of ds:KeyInfo -->
+
+  <element name='ReferenceList'>
+    <complexType>
+      <choice minOccurs='1' maxOccurs='unbounded'>
+        <element name='DataReference' type='xenc:ReferenceType'/>
+        <element name='KeyReference' type='xenc:ReferenceType'/>
+      </choice>
+    </complexType>
+  </element>
+
+  <complexType name='ReferenceType'>
+    <sequence>
+      <any namespace='##other' minOccurs='0' maxOccurs='unbounded'/>
+    </sequence>
+    <attribute name='URI' type='anyURI' use='required'/>
+  </complexType>
+
+
+  <element name='EncryptionProperties' type='xenc:EncryptionPropertiesType'/>
+  <complexType name='EncryptionPropertiesType'>
+    <sequence>
+      <element ref='xenc:EncryptionProperty' maxOccurs='unbounded'/>
+    </sequence>
+    <attribute name='Id' type='ID' use='optional'/>
+  </complexType>
+
+    <element name='EncryptionProperty' type='xenc:EncryptionPropertyType'/>
+    <complexType name='EncryptionPropertyType' mixed='true'>
+      <choice maxOccurs='unbounded'>
+        <any namespace='##other' processContents='lax'/>
+      </choice>
+      <attribute name='Target' type='anyURI' use='optional'/>
+      <attribute name='Id' type='ID' use='optional'/>
+      <anyAttribute namespace="http://www.w3.org/XML/1998/namespace"/>
+    </complexType>
+
+</schema>
+
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xml.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xml.xsd
new file mode 100644
index 0000000000000..5a282019b6afe
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xml.xsd
@@ -0,0 +1,286 @@
+<?xml version='1.0'?>
+<xs:schema targetNamespace="http://www.w3.org/XML/1998/namespace"
+  xmlns:xs="http://www.w3.org/2001/XMLSchema" 
+  xmlns   ="http://www.w3.org/1999/xhtml"
+  xml:lang="en">
+
+ <xs:annotation>
+  <xs:documentation>
+   <div>
+    <h1>About the XML namespace</h1>
+
+    <div class="bodytext">
+     <p>
+      This schema document describes the XML namespace, in a form
+      suitable for import by other schema documents.
+     </p>
+     <p>
+      See <a href="http://www.w3.org/XML/1998/namespace.html">
+      http://www.w3.org/XML/1998/namespace.html</a> and
+      <a href="http://www.w3.org/TR/REC-xml">
+      http://www.w3.org/TR/REC-xml</a> for information 
+      about this namespace.
+     </p>
+     <p>
+      Note that local names in this namespace are intended to be
+      defined only by the World Wide Web Consortium or its subgroups.
+      The names currently defined in this namespace are listed below.
+      They should not be used with conflicting semantics by any Working
+      Group, specification, or document instance.
+     </p>
+     <p>   
+      See further below in this document for more information about <a
+      href="#usage">how to refer to this schema document from your own
+      XSD schema documents</a> and about <a href="#nsversioning">the
+      namespace-versioning policy governing this schema document</a>.
+     </p>
+    </div>
+   </div>
+  </xs:documentation>
+ </xs:annotation>
+
+ <xs:attribute name="lang">
+  <xs:annotation>
+   <xs:documentation>
+    <div>
+     
+      <h3>lang (as an attribute name)</h3>
+      <p>
+       denotes an attribute whose value
+       is a language code for the natural language of the content of
+       any element; its value is inherited.  This name is reserved
+       by virtue of its definition in the XML specification.</p>
+     
+    </div>
+    <div>
+     <h4>Notes</h4>
+     <p>
+      Attempting to install the relevant ISO 2- and 3-letter
+      codes as the enumerated possible values is probably never
+      going to be a realistic possibility.  
+     </p>
+     <p>
+      See BCP 47 at <a href="http://www.rfc-editor.org/rfc/bcp/bcp47.txt">
+       http://www.rfc-editor.org/rfc/bcp/bcp47.txt</a>
+      and the IANA language subtag registry at
+      <a href="http://www.iana.org/assignments/language-subtag-registry">
+       http://www.iana.org/assignments/language-subtag-registry</a>
+      for further information.
+     </p>
+     <p>
+      The union allows for the 'un-declaration' of xml:lang with
+      the empty string.
+     </p>
+    </div>
+   </xs:documentation>
+  </xs:annotation>
+  <xs:simpleType>
+   <xs:union memberTypes="xs:language">
+    <xs:simpleType>    
+     <xs:restriction base="xs:string">
+      <xs:enumeration value=""/>
+     </xs:restriction>
+    </xs:simpleType>
+   </xs:union>
+  </xs:simpleType>
+ </xs:attribute>
+
+ <xs:attribute name="space">
+  <xs:annotation>
+   <xs:documentation>
+    <div>
+     
+      <h3>space (as an attribute name)</h3>
+      <p>
+       denotes an attribute whose
+       value is a keyword indicating what whitespace processing
+       discipline is intended for the content of the element; its
+       value is inherited.  This name is reserved by virtue of its
+       definition in the XML specification.</p>
+     
+    </div>
+   </xs:documentation>
+  </xs:annotation>
+  <xs:simpleType>
+   <xs:restriction base="xs:NCName">
+    <xs:enumeration value="default"/>
+    <xs:enumeration value="preserve"/>
+   </xs:restriction>
+  </xs:simpleType>
+ </xs:attribute>
+ 
+ <xs:attribute name="base" type="xs:anyURI"> <xs:annotation>
+   <xs:documentation>
+    <div>
+     
+      <h3>base (as an attribute name)</h3>
+      <p>
+       denotes an attribute whose value
+       provides a URI to be used as the base for interpreting any
+       relative URIs in the scope of the element on which it
+       appears; its value is inherited.  This name is reserved
+       by virtue of its definition in the XML Base specification.</p>
+     
+     <p>
+      See <a
+      href="http://www.w3.org/TR/xmlbase/">http://www.w3.org/TR/xmlbase/</a>
+      for information about this attribute.
+     </p>
+    </div>
+   </xs:documentation>
+  </xs:annotation>
+ </xs:attribute>
+ 
+ <xs:attribute name="id" type="xs:ID">
+  <xs:annotation>
+   <xs:documentation>
+    <div>
+     
+      <h3>id (as an attribute name)</h3> 
+      <p>
+       denotes an attribute whose value
+       should be interpreted as if declared to be of type ID.
+       This name is reserved by virtue of its definition in the
+       xml:id specification.</p>
+     
+     <p>
+      See <a
+      href="http://www.w3.org/TR/xml-id/">http://www.w3.org/TR/xml-id/</a>
+      for information about this attribute.
+     </p>
+    </div>
+   </xs:documentation>
+  </xs:annotation>
+ </xs:attribute>
+
+ <xs:attributeGroup name="specialAttrs">
+  <xs:attribute ref="xml:base"/>
+  <xs:attribute ref="xml:lang"/>
+  <xs:attribute ref="xml:space"/>
+  <xs:attribute ref="xml:id"/>
+ </xs:attributeGroup>
+
+ <xs:annotation>
+  <xs:documentation>
+   <div>
+   
+    <h3>Father (in any context at all)</h3> 
+
+    <div class="bodytext">
+     <p>
+      denotes Jon Bosak, the chair of 
+      the original XML Working Group.  This name is reserved by 
+      the following decision of the W3C XML Plenary and 
+      XML Coordination groups:
+     </p>
+     <blockquote>
+       <p>
+        In appreciation for his vision, leadership and
+        dedication the W3C XML Plenary on this 10th day of
+        February, 2000, reserves for Jon Bosak in perpetuity
+        the XML name "xml:Father".
+       </p>
+     </blockquote>
+    </div>
+   </div>
+  </xs:documentation>
+ </xs:annotation>
+
+ <xs:annotation>
+  <xs:documentation>
+   <div xml:id="usage" id="usage">
+    <h2><a name="usage">About this schema document</a></h2>
+
+    <div class="bodytext">
+     <p>
+      This schema defines attributes and an attribute group suitable
+      for use by schemas wishing to allow <code>xml:base</code>,
+      <code>xml:lang</code>, <code>xml:space</code> or
+      <code>xml:id</code> attributes on elements they define.
+     </p>
+     <p>
+      To enable this, such a schema must import this schema for
+      the XML namespace, e.g. as follows:
+     </p>
+     <pre>
+          &lt;schema . . .>
+           . . .
+           &lt;import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2001/xml.xsd"/>
+     </pre>
+     <p>
+      or
+     </p>
+     <pre>
+           &lt;import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2009/01/xml.xsd"/>
+     </pre>
+     <p>
+      Subsequently, qualified reference to any of the attributes or the
+      group defined below will have the desired effect, e.g.
+     </p>
+     <pre>
+          &lt;type . . .>
+           . . .
+           &lt;attributeGroup ref="xml:specialAttrs"/>
+     </pre>
+     <p>
+      will define a type which will schema-validate an instance element
+      with any of those attributes.
+     </p>
+    </div>
+   </div>
+  </xs:documentation>
+ </xs:annotation>
+
+ <xs:annotation>
+  <xs:documentation>
+   <div id="nsversioning" xml:id="nsversioning">
+    <h2><a name="nsversioning">Versioning policy for this schema document</a></h2>
+    <div class="bodytext">
+     <p>
+      In keeping with the XML Schema WG's standard versioning
+      policy, this schema document will persist at
+      <a href="http://www.w3.org/2009/01/xml.xsd">
+       http://www.w3.org/2009/01/xml.xsd</a>.
+     </p>
+     <p>
+      At the date of issue it can also be found at
+      <a href="http://www.w3.org/2001/xml.xsd">
+       http://www.w3.org/2001/xml.xsd</a>.
+     </p>
+     <p>
+      The schema document at that URI may however change in the future,
+      in order to remain compatible with the latest version of XML
+      Schema itself, or with the XML namespace itself.  In other words,
+      if the XML Schema or XML namespaces change, the version of this
+      document at <a href="http://www.w3.org/2001/xml.xsd">
+       http://www.w3.org/2001/xml.xsd 
+      </a> 
+      will change accordingly; the version at 
+      <a href="http://www.w3.org/2009/01/xml.xsd">
+       http://www.w3.org/2009/01/xml.xsd 
+      </a> 
+      will not change.
+     </p>
+     <p>
+      Previous dated (and unchanging) versions of this schema 
+      document are at:
+     </p>
+     <ul>
+      <li><a href="http://www.w3.org/2009/01/xml.xsd">
+        http://www.w3.org/2009/01/xml.xsd</a></li>
+      <li><a href="http://www.w3.org/2007/08/xml.xsd">
+        http://www.w3.org/2007/08/xml.xsd</a></li>
+      <li><a href="http://www.w3.org/2004/10/xml.xsd">
+        http://www.w3.org/2004/10/xml.xsd</a></li>
+      <li><a href="http://www.w3.org/2001/03/xml.xsd">
+        http://www.w3.org/2001/03/xml.xsd</a></li>
+     </ul>
+    </div>
+   </div>
+  </xs:documentation>
+ </xs:annotation>
+
+</xs:schema>
+
diff --git a/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xmldsig-core-schema.xsd b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xmldsig-core-schema.xsd
new file mode 100644
index 0000000000000..8422fdfaaf9d2
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/main/resources/org/elasticsearch/xpack/idp/saml/support/xmldsig-core-schema.xsd
@@ -0,0 +1,308 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Schema for XML Signatures
+    http://www.w3.org/2000/09/xmldsig#
+    $Revision: 1.1 $ on $Date: 2002/02/08 20:32:26 $ by $Author: reagle $
+
+    Copyright 2001 The Internet Society and W3C (Massachusetts Institute
+    of Technology, Institut National de Recherche en Informatique et en
+    Automatique, Keio University). All Rights Reserved.
+    http://www.w3.org/Consortium/Legal/
+
+    This document is governed by the W3C Software License [1] as described
+    in the FAQ [2].
+
+    [1] http://www.w3.org/Consortium/Legal/copyright-software-19980720
+    [2] http://www.w3.org/Consortium/Legal/IPR-FAQ-20000620.html#DTD
+-->
+
+
+<schema xmlns="http://www.w3.org/2001/XMLSchema"
+        xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+        targetNamespace="http://www.w3.org/2000/09/xmldsig#"
+        version="0.1" elementFormDefault="qualified"> 
+
+<!-- Basic Types Defined for Signatures -->
+
+<simpleType name="CryptoBinary">
+  <restriction base="base64Binary">
+  </restriction>
+</simpleType>
+
+<!-- Start Signature -->
+
+<element name="Signature" type="ds:SignatureType"/>
+<complexType name="SignatureType">
+  <sequence> 
+    <element ref="ds:SignedInfo"/> 
+    <element ref="ds:SignatureValue"/> 
+    <element ref="ds:KeyInfo" minOccurs="0"/> 
+    <element ref="ds:Object" minOccurs="0" maxOccurs="unbounded"/> 
+  </sequence>  
+  <attribute name="Id" type="ID" use="optional"/>
+</complexType>
+
+  <element name="SignatureValue" type="ds:SignatureValueType"/> 
+  <complexType name="SignatureValueType">
+    <simpleContent>
+      <extension base="base64Binary">
+        <attribute name="Id" type="ID" use="optional"/>
+      </extension>
+    </simpleContent>
+  </complexType>
+
+<!-- Start SignedInfo -->
+
+<element name="SignedInfo" type="ds:SignedInfoType"/>
+<complexType name="SignedInfoType">
+  <sequence> 
+    <element ref="ds:CanonicalizationMethod"/> 
+    <element ref="ds:SignatureMethod"/> 
+    <element ref="ds:Reference" maxOccurs="unbounded"/> 
+  </sequence>  
+  <attribute name="Id" type="ID" use="optional"/> 
+</complexType>
+
+  <element name="CanonicalizationMethod" type="ds:CanonicalizationMethodType"/> 
+  <complexType name="CanonicalizationMethodType" mixed="true">
+    <sequence>
+      <any namespace="##any" minOccurs="0" maxOccurs="unbounded"/>
+      <!-- (0,unbounded) elements from (1,1) namespace -->
+    </sequence>
+    <attribute name="Algorithm" type="anyURI" use="required"/> 
+  </complexType>
+
+  <element name="SignatureMethod" type="ds:SignatureMethodType"/>
+  <complexType name="SignatureMethodType" mixed="true">
+    <sequence>
+      <element name="HMACOutputLength" minOccurs="0" type="ds:HMACOutputLengthType"/>
+      <any namespace="##other" minOccurs="0" maxOccurs="unbounded"/>
+      <!-- (0,unbounded) elements from (1,1) external namespace -->
+    </sequence>
+    <attribute name="Algorithm" type="anyURI" use="required"/> 
+  </complexType>
+
+<!-- Start Reference -->
+
+<element name="Reference" type="ds:ReferenceType"/>
+<complexType name="ReferenceType">
+  <sequence> 
+    <element ref="ds:Transforms" minOccurs="0"/> 
+    <element ref="ds:DigestMethod"/> 
+    <element ref="ds:DigestValue"/> 
+  </sequence>
+  <attribute name="Id" type="ID" use="optional"/> 
+  <attribute name="URI" type="anyURI" use="optional"/> 
+  <attribute name="Type" type="anyURI" use="optional"/> 
+</complexType>
+
+  <element name="Transforms" type="ds:TransformsType"/>
+  <complexType name="TransformsType">
+    <sequence>
+      <element ref="ds:Transform" maxOccurs="unbounded"/>  
+    </sequence>
+  </complexType>
+
+  <element name="Transform" type="ds:TransformType"/>
+  <complexType name="TransformType" mixed="true">
+    <choice minOccurs="0" maxOccurs="unbounded"> 
+      <any namespace="##other" processContents="lax"/>
+      <!-- (1,1) elements from (0,unbounded) namespaces -->
+      <element name="XPath" type="string"/> 
+    </choice>
+    <attribute name="Algorithm" type="anyURI" use="required"/> 
+  </complexType>
+
+<!-- End Reference -->
+
+<element name="DigestMethod" type="ds:DigestMethodType"/>
+<complexType name="DigestMethodType" mixed="true"> 
+  <sequence>
+    <any namespace="##other" processContents="lax" minOccurs="0" maxOccurs="unbounded"/>
+  </sequence>    
+  <attribute name="Algorithm" type="anyURI" use="required"/> 
+</complexType>
+
+<element name="DigestValue" type="ds:DigestValueType"/>
+<simpleType name="DigestValueType">
+  <restriction base="base64Binary"/>
+</simpleType>
+
+<!-- End SignedInfo -->
+
+<!-- Start KeyInfo -->
+
+<element name="KeyInfo" type="ds:KeyInfoType"/> 
+<complexType name="KeyInfoType" mixed="true">
+  <choice maxOccurs="unbounded">     
+    <element ref="ds:KeyName"/> 
+    <element ref="ds:KeyValue"/> 
+    <element ref="ds:RetrievalMethod"/> 
+    <element ref="ds:X509Data"/> 
+    <element ref="ds:PGPData"/> 
+    <element ref="ds:SPKIData"/>
+    <element ref="ds:MgmtData"/>
+    <any processContents="lax" namespace="##other"/>
+    <!-- (1,1) elements from (0,unbounded) namespaces -->
+  </choice>
+  <attribute name="Id" type="ID" use="optional"/> 
+</complexType>
+
+  <element name="KeyName" type="string"/>
+  <element name="MgmtData" type="string"/>
+
+  <element name="KeyValue" type="ds:KeyValueType"/> 
+  <complexType name="KeyValueType" mixed="true">
+   <choice>
+     <element ref="ds:DSAKeyValue"/>
+     <element ref="ds:RSAKeyValue"/>
+     <any namespace="##other" processContents="lax"/>
+   </choice>
+  </complexType>
+
+  <element name="RetrievalMethod" type="ds:RetrievalMethodType"/> 
+  <complexType name="RetrievalMethodType">
+    <sequence>
+      <element ref="ds:Transforms" minOccurs="0"/> 
+    </sequence>  
+    <attribute name="URI" type="anyURI"/>
+    <attribute name="Type" type="anyURI" use="optional"/>
+  </complexType>
+
+<!-- Start X509Data -->
+
+<element name="X509Data" type="ds:X509DataType"/> 
+<complexType name="X509DataType">
+  <sequence maxOccurs="unbounded">
+    <choice>
+      <element name="X509IssuerSerial" type="ds:X509IssuerSerialType"/>
+      <element name="X509SKI" type="base64Binary"/>
+      <element name="X509SubjectName" type="string"/>
+      <element name="X509Certificate" type="base64Binary"/>
+      <element name="X509CRL" type="base64Binary"/>
+      <any namespace="##other" processContents="lax"/>
+    </choice>
+  </sequence>
+</complexType>
+
+<complexType name="X509IssuerSerialType"> 
+  <sequence> 
+    <element name="X509IssuerName" type="string"/> 
+    <element name="X509SerialNumber" type="integer"/> 
+  </sequence>
+</complexType>
+
+<!-- End X509Data -->
+
+<!-- Begin PGPData -->
+
+<element name="PGPData" type="ds:PGPDataType"/> 
+<complexType name="PGPDataType"> 
+  <choice>
+    <sequence>
+      <element name="PGPKeyID" type="base64Binary"/> 
+      <element name="PGPKeyPacket" type="base64Binary" minOccurs="0"/> 
+      <any namespace="##other" processContents="lax" minOccurs="0"
+       maxOccurs="unbounded"/>
+    </sequence>
+    <sequence>
+      <element name="PGPKeyPacket" type="base64Binary"/> 
+      <any namespace="##other" processContents="lax" minOccurs="0"
+       maxOccurs="unbounded"/>
+    </sequence>
+  </choice>
+</complexType>
+
+<!-- End PGPData -->
+
+<!-- Begin SPKIData -->
+
+<element name="SPKIData" type="ds:SPKIDataType"/> 
+<complexType name="SPKIDataType">
+  <sequence maxOccurs="unbounded">
+    <element name="SPKISexp" type="base64Binary"/>
+    <any namespace="##other" processContents="lax" minOccurs="0"/>
+  </sequence>
+</complexType> 
+
+<!-- End SPKIData -->
+
+<!-- End KeyInfo -->
+
+<!-- Start Object (Manifest, SignatureProperty) -->
+
+<element name="Object" type="ds:ObjectType"/> 
+<complexType name="ObjectType" mixed="true">
+  <sequence minOccurs="0" maxOccurs="unbounded">
+    <any namespace="##any" processContents="lax"/>
+  </sequence>
+  <attribute name="Id" type="ID" use="optional"/> 
+  <attribute name="MimeType" type="string" use="optional"/> <!-- add a grep facet -->
+  <attribute name="Encoding" type="anyURI" use="optional"/> 
+</complexType>
+
+<element name="Manifest" type="ds:ManifestType"/> 
+<complexType name="ManifestType">
+  <sequence>
+    <element ref="ds:Reference" maxOccurs="unbounded"/> 
+  </sequence>
+  <attribute name="Id" type="ID" use="optional"/> 
+</complexType>
+
+<element name="SignatureProperties" type="ds:SignaturePropertiesType"/> 
+<complexType name="SignaturePropertiesType">
+  <sequence>
+    <element ref="ds:SignatureProperty" maxOccurs="unbounded"/> 
+  </sequence>
+  <attribute name="Id" type="ID" use="optional"/> 
+</complexType>
+
+   <element name="SignatureProperty" type="ds:SignaturePropertyType"/> 
+   <complexType name="SignaturePropertyType" mixed="true">
+     <choice maxOccurs="unbounded">
+       <any namespace="##other" processContents="lax"/>
+       <!-- (1,1) elements from (1,unbounded) namespaces -->
+     </choice>
+     <attribute name="Target" type="anyURI" use="required"/> 
+     <attribute name="Id" type="ID" use="optional"/> 
+   </complexType>
+
+<!-- End Object (Manifest, SignatureProperty) -->
+
+<!-- Start Algorithm Parameters -->
+
+<simpleType name="HMACOutputLengthType">
+  <restriction base="integer"/>
+</simpleType>
+
+<!-- Start KeyValue Element-types -->
+
+<element name="DSAKeyValue" type="ds:DSAKeyValueType"/>
+<complexType name="DSAKeyValueType">
+  <sequence>
+    <sequence minOccurs="0">
+      <element name="P" type="ds:CryptoBinary"/>
+      <element name="Q" type="ds:CryptoBinary"/>
+    </sequence>
+    <element name="G" type="ds:CryptoBinary" minOccurs="0"/>
+    <element name="Y" type="ds:CryptoBinary"/>
+    <element name="J" type="ds:CryptoBinary" minOccurs="0"/>
+    <sequence minOccurs="0">
+      <element name="Seed" type="ds:CryptoBinary"/>
+      <element name="PgenCounter" type="ds:CryptoBinary"/>
+    </sequence>
+  </sequence>
+</complexType>
+
+<element name="RSAKeyValue" type="ds:RSAKeyValueType"/>
+<complexType name="RSAKeyValueType">
+  <sequence>
+    <element name="Modulus" type="ds:CryptoBinary"/> 
+    <element name="Exponent" type="ds:CryptoBinary"/> 
+  </sequence>
+</complexType> 
+
+<!-- End KeyValue Element-types -->
+
+<!-- End Signature -->
+
+</schema>
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/LocalStateIdentityProviderPlugin.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/LocalStateIdentityProviderPlugin.java
new file mode 100644
index 0000000000000..2bd810a0a9a93
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/LocalStateIdentityProviderPlugin.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.ssl.SSLService;
+import org.elasticsearch.xpack.security.Security;
+
+import java.nio.file.Path;
+
+public class LocalStateIdentityProviderPlugin extends LocalStateCompositeXPackPlugin {
+
+    public LocalStateIdentityProviderPlugin(Settings settings, Path configPath) throws Exception {
+        super(settings, configPath);
+        LocalStateIdentityProviderPlugin thisVar = this;
+        plugins.add(new Security(settings, configPath) {
+            @Override
+            protected SSLService getSslService() {
+                return thisVar.getSslService();
+            }
+
+            @Override
+            protected XPackLicenseState getLicenseState() {
+                return thisVar.getLicenseState();
+            }
+        });
+
+        plugins.add(new IdentityProviderPlugin() {
+            @Override
+            protected XPackLicenseState getLicenseState() {
+                return thisVar.getLicenseState();
+            }
+        });
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java
new file mode 100644
index 0000000000000..0c1c53ac3e610
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DeleteSamlServiceProviderRequestTests extends IdpSamlTestCase {
+
+    public void testSerialization() throws IOException {
+        final DeleteSamlServiceProviderRequest request = new DeleteSamlServiceProviderRequest(randomAlphaOfLengthBetween(1, 100),
+            randomFrom(WriteRequest.RefreshPolicy.values()));
+        final Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_7_0, Version.CURRENT);
+        final DeleteSamlServiceProviderRequest read = copyWriteable(request, new NamedWriteableRegistry(List.of()),
+            DeleteSamlServiceProviderRequest::new, version);
+        MatcherAssert.assertThat("Serialized request with version [" + version + "] does not match original object",
+            read, equalTo(request));
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponseTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponseTests.java
new file mode 100644
index 0000000000000..785df0a8a58ae
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderResponseTests.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class DeleteSamlServiceProviderResponseTests extends IdpSamlTestCase {
+
+    public void testToXContent() throws Exception {
+        final String docId = randomAlphaOfLength(24);
+        final long seqNo = randomNonNegativeLong();
+        final long primaryTerm = randomNonNegativeLong();
+        final String entityId = randomAlphaOfLengthBetween(4, 128);
+        final DeleteSamlServiceProviderResponse response = new DeleteSamlServiceProviderResponse(docId, seqNo, primaryTerm, entityId);
+        final String xContent = Strings.toString(response);
+        final Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), xContent, randomBoolean());
+        assertThat(map.keySet(), containsInAnyOrder("document", "service_provider"));
+
+        assertThat(map.get("document"), instanceOf(Map.class));
+        assertThat(((Map<?, ?>) map.get("document")).keySet(), containsInAnyOrder("_id", "_seq_no", "_primary_term"));
+
+        assertThat(map.get("service_provider"), instanceOf(Map.class));
+        assertThat(((Map<?, ?>) map.get("service_provider")).keySet(), containsInAnyOrder("entity_id"));
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java
new file mode 100644
index 0000000000000..f81eaf081019c
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestMatchers;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndexTests;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap;
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.startsWith;
+
+public class PutSamlServiceProviderRequestTests extends ESTestCase {
+
+    public void testValidateSuccessfully() {
+        final SamlServiceProviderDocument doc = SamlServiceProviderIndexTests.randomDocument();
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(doc, randomFrom(RefreshPolicy.values()));
+        assertThat(request.validate(), nullValue());
+    }
+
+    public void testValidateAcs() {
+        final SamlServiceProviderDocument doc = SamlServiceProviderIndexTests.randomDocument();
+        doc.acs = "this is not a URL";
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(doc, randomFrom(RefreshPolicy.values()));
+        final ActionRequestValidationException validationException = request.validate();
+        assertThat(validationException, notNullValue());
+        assertThat(validationException.validationErrors(), hasSize(1));
+        assertThat(validationException.validationErrors().get(0), containsString("[acs]"));
+        assertThat(validationException.validationErrors().get(0), containsString("valid URL"));
+    }
+
+    public void testValidateRequiredFields() {
+        final SamlServiceProviderDocument doc = SamlServiceProviderIndexTests.randomDocument();
+        doc.name = null;
+        doc.entityId = null;
+        doc.acs = null;
+        doc.created = null;
+        doc.lastModified = null;
+        doc.attributeNames.principal = null;
+        doc.privileges.resource = null;
+
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(doc, randomFrom(RefreshPolicy.values()));
+        final ActionRequestValidationException validationException = request.validate();
+        assertThat(validationException, notNullValue());
+        assertThat(validationException.validationErrors(), hasSize(7));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[name]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[entity_id]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[acs]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[created]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[last_modified]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[attributes.principal]")));
+        assertThat(validationException.validationErrors(), hasItem(containsString("[privileges.resource]")));
+    }
+
+    public void testSerialization() throws IOException {
+        final SamlServiceProviderDocument doc = SamlServiceProviderIndexTests.randomDocument();
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(doc, RefreshPolicy.NONE);
+        final Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_7_0, Version.CURRENT);
+        final PutSamlServiceProviderRequest read = copyWriteable(request, new NamedWriteableRegistry(List.of()),
+            PutSamlServiceProviderRequest::new, version);
+        MatcherAssert.assertThat("Serialized request with version [" + version + "] does not match original object",
+            read, equalTo(request));
+    }
+
+    public void testParseRequestBodySuccessfully() throws Exception {
+        final Map<String, Object> fields = new HashMap<>();
+        fields.put("name", randomAlphaOfLengthBetween(3, 30));
+        fields.put("acs", "https://www." + randomAlphaOfLengthBetween(3, 30) + ".fake/saml/acs");
+        fields.put("enabled", randomBoolean());
+        fields.put("attributes", Map.of(
+            "principal", "urn:oid:0.1." + randomLongBetween(1, 1000),
+            "email", "urn:oid:0.2." + randomLongBetween(1001, 2000),
+            "name", "urn:oid:0.3." + randomLongBetween(2001, 3000),
+            "roles", "urn:oid:0.4." + randomLongBetween(3001, 4000)
+        ));
+        fields.put("privileges", Map.of(
+            "resource", "ece:deployment:" + randomLongBetween(1_000_000, 999_999_999),
+            "roles", Map.of("role_name", "role:" + randomAlphaOfLengthBetween(4, 8))
+        ));
+        fields.put("certificates", Map.of());
+        final String entityId = "https://www." + randomAlphaOfLengthBetween(5, 12) + ".app/";
+        final PutSamlServiceProviderRequest request = parseRequest(entityId, fields);
+        assertThat(request.getDocument().docId, nullValue());
+        assertThat(request.getDocument().entityId, equalTo(entityId));
+        assertThat(request.getDocument().acs, equalTo(fields.get("acs")));
+        assertThat(request.getDocument().enabled, equalTo(fields.get("enabled")));
+        assertThat(request.getDocument().privileges.resource, notNullValue());
+        assertThat(request.getDocument().privileges.roleActions, aMapWithSize(1));
+        assertThat(request.getDocument().privileges.roleActions.keySet(), contains("role_name"));
+        assertThat(request.getDocument().attributeNames.principal, startsWith("urn:oid:0.1"));
+        assertThat(request.getDocument().attributeNames.email, startsWith("urn:oid:0.2"));
+        assertThat(request.getDocument().attributeNames.name, startsWith("urn:oid:0.3"));
+        assertThat(request.getDocument().attributeNames.roles, startsWith("urn:oid:0.4"));
+        assertThat(request.getDocument().certificates.serviceProviderSigning, emptyIterable());
+        assertThat(request.getDocument().certificates.identityProviderSigning, emptyIterable());
+        assertThat(request.getDocument().certificates.identityProviderMetadataSigning, emptyIterable());
+
+        assertThat(request.validate(), nullValue());
+    }
+
+    public void testParseRequestBodyFailsIfTimestampsAreIncluded() throws Exception {
+        final SamlServiceProviderDocument doc = SamlServiceProviderIndexTests.randomDocument();
+        final Map<String, Object> fields = convertToMap(XContentType.JSON.xContent(), Strings.toString(doc), randomBoolean());
+
+        fields.remove("entity_id");
+        fields.remove("created");
+        fields.remove("last_modified");
+
+        final String field = randomBoolean() ? "created" : "last_modified";
+        fields.put(field, System.currentTimeMillis() + randomLongBetween(-100_000, +100_000));
+
+        ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, () -> parseRequest(doc.entityId, fields));
+        assertThat(exception, TestMatchers.throwableWithMessage("Field [" + field + "] may not be specified in a request"));
+    }
+
+    private PutSamlServiceProviderRequest parseRequest(String entityId, Map<String, Object> fields) throws IOException {
+        try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+            builder.map(fields);
+            try (XContentParser parser = createParser(shuffleXContent(builder))) {
+                return PutSamlServiceProviderRequest.fromXContent(entityId, randomFrom(RefreshPolicy.values()), parser);
+            }
+        }
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java
new file mode 100644
index 0000000000000..925fc04695a38
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java
@@ -0,0 +1,349 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+import org.elasticsearch.xpack.idp.saml.test.IdentityProviderIntegTestCase;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.opensaml.core.xml.util.XMLObjectSupport;
+import org.opensaml.saml.common.SAMLObject;
+import org.opensaml.saml.saml2.core.AuthnRequest;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.NameIDPolicy;
+import org.opensaml.security.SecurityException;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.xmlsec.crypto.XMLSigningUtil;
+import org.opensaml.xmlsec.signature.support.SignatureConstants;
+
+import java.io.ByteArrayOutputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.zip.Deflater;
+import java.util.zip.DeflaterOutputStream;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.joda.time.DateTime.now;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numClientNodes = 0, numDataNodes = 0)
+public class SamlIdentityProviderTests extends IdentityProviderIntegTestCase {
+
+    private final SamlFactory samlFactory = new SamlFactory();
+
+    public void testIdpInitiatedSso() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+
+        // User login a.k.a exchange the user credentials for an API Key
+        final String apiKeyCredentials = getApiKeyFromCredentials(SAMPLE_IDPUSER_NAME,
+            new SecureString(SAMPLE_IDPUSER_PASSWORD.toCharArray()));
+        // Make a request to init an SSO flow with the API Key as secondary authentication
+        Request request = new Request("POST", "/_idp/saml/init");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder()
+            .addHeader("Authorization", basicAuthHeaderValue(CONSOLE_USER_NAME,
+                new SecureString(CONSOLE_USER_PASSWORD.toCharArray())))
+            .addHeader("es-secondary-authorization", "ApiKey " + apiKeyCredentials)
+            .build());
+        request.setJsonEntity("{ \"entity_id\": \"" + entityId + "\"}");
+        Response initResponse = getRestClient().performRequest(request);
+        ObjectPath objectPath = ObjectPath.createFromResponse(initResponse);
+        assertThat(objectPath.evaluate("post_url").toString(), equalTo(acsUrl));
+        final String body = objectPath.evaluate("saml_response").toString();
+        assertThat(body, containsString("Destination=\"" + acsUrl + "\""));
+        assertThat(body, containsString("<saml2:Audience>" + entityId + "</saml2:Audience>"));
+        assertThat(body, containsString("<saml2:NameID Format=\"" + TRANSIENT + "\">"));
+        Map<String, String> serviceProvider = objectPath.evaluate("service_provider");
+        assertThat(serviceProvider, hasKey("entity_id"));
+        assertThat(serviceProvider.get("entity_id"), equalTo(entityId));
+        assertContainsAttributeWithValue(body, "principal", SAMPLE_IDPUSER_NAME);
+    }
+
+    public void testIdPInitiatedSsoFailsForUnknownSP() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        // User login a.k.a exchange the user credentials for an API Key
+        final String apiKeyCredentials = getApiKeyFromCredentials(SAMPLE_IDPUSER_NAME,
+            new SecureString(SAMPLE_IDPUSER_PASSWORD.toCharArray()));
+        // Make a request to init an SSO flow with the API Key as secondary authentication
+        Request request = new Request("POST", "/_idp/saml/init");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder()
+            .addHeader("Authorization", basicAuthHeaderValue(CONSOLE_USER_NAME,
+                new SecureString(CONSOLE_USER_PASSWORD.toCharArray())))
+            .addHeader("es-secondary-authorization", "ApiKey " + apiKeyCredentials)
+            .build());
+        request.setJsonEntity("{ \"entity_id\": \"" + entityId + randomAlphaOfLength(3) + "\"}");
+        ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
+        assertThat(e.getMessage(), containsString("is not registered with this Identity Provider"));
+        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
+    }
+
+    public void testIdPInitiatedSsoFailsWithoutSecondaryAuthentication() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        // Make a request to init an SSO flow with the API Key as secondary authentication
+        Request request = new Request("POST", "/_idp/saml/init");
+        request.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
+        request.setJsonEntity("{ \"entity_id\": \"" + entityId + "\"}");
+        ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
+        assertThat(e.getMessage(), containsString("Request is missing secondary authentication"));
+    }
+
+    public void testSpInitiatedSso() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        // Validate incoming authentication request
+        Request validateRequest = new Request("POST", "/_idp/saml/validate");
+        validateRequest.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
+        final String nameIdFormat = TRANSIENT;
+        final String relayString = randomBoolean() ? randomAlphaOfLength(8) : null;
+        final boolean forceAuthn = true;
+        final AuthnRequest authnRequest = buildAuthnRequest(entityId, new URL(acsUrl),
+            new URL("https://idp.org/sso/redirect"), nameIdFormat, forceAuthn);
+        final String query = getQueryString(authnRequest, relayString, false, null);
+        validateRequest.setJsonEntity("{\"authn_request_query\":\"" + query + "\"}");
+        Response validateResponse = getRestClient().performRequest(validateRequest);
+        ObjectPath validateResponseObject = ObjectPath.createFromResponse(validateResponse);
+        Map<String, String> serviceProvider = validateResponseObject.evaluate("service_provider");
+        assertThat(serviceProvider, hasKey("entity_id"));
+        assertThat(serviceProvider.get("entity_id"), equalTo(entityId));
+        assertThat(validateResponseObject.evaluate("force_authn"), equalTo(forceAuthn));
+        Map<String, String> authnState = validateResponseObject.evaluate("authn_state");
+        assertThat(authnState, hasKey("nameid_format"));
+        assertThat(authnState.get("nameid_format"), equalTo(nameIdFormat));
+        assertThat(authnState, hasKey("entity_id"));
+        assertThat(authnState.get("entity_id"), equalTo(entityId));
+        assertThat(authnState, hasKey("acs_url"));
+        assertThat(authnState.get("acs_url"), equalTo(acsUrl));
+        assertThat(authnState, hasKey("authn_request_id"));
+        final String expectedInResponeTo = authnState.get("authn_request_id");
+
+        // User login a.k.a exchange the user credentials for an API Key
+        final String apiKeyCredentials = getApiKeyFromCredentials(SAMPLE_IDPUSER_NAME,
+            new SecureString(SAMPLE_IDPUSER_PASSWORD.toCharArray()));
+        // Make a request to init an SSO flow with the API Key as secondary authentication
+        Request initRequest = new Request("POST", "/_idp/saml/init");
+        initRequest.setOptions(RequestOptions.DEFAULT.toBuilder()
+            .addHeader("Authorization", basicAuthHeaderValue(CONSOLE_USER_NAME,
+                new SecureString(CONSOLE_USER_PASSWORD.toCharArray())))
+            .addHeader("es-secondary-authorization", "ApiKey " + apiKeyCredentials)
+            .build());
+        XContentBuilder authnStateBuilder = jsonBuilder();
+        authnStateBuilder.map(authnState);
+        initRequest.setJsonEntity("{ \"entity_id\":\"" + entityId + "\", \"authn_state\":" + Strings.toString(authnStateBuilder) + "}");
+        Response initResponse = getRestClient().performRequest(initRequest);
+        ObjectPath initResponseObject = ObjectPath.createFromResponse(initResponse);
+        assertThat(initResponseObject.evaluate("post_url").toString(), equalTo(acsUrl));
+        final String body = initResponseObject.evaluate("saml_response").toString();
+        assertThat(body, containsString("<saml2p:StatusCode Value=\"urn:oasis:names:tc:SAML:2.0:status:Success\"/>"));
+        assertThat(body, containsString("Destination=\"" + acsUrl + "\""));
+        assertThat(body, containsString("<saml2:Audience>" + entityId + "</saml2:Audience>"));
+        assertThat(body, containsString("<saml2:NameID Format=\"" + nameIdFormat + "\">"));
+        assertThat(body, containsString("InResponseTo=\"" + expectedInResponeTo + "\""));
+        Map<String, String> sp = initResponseObject.evaluate("service_provider");
+        assertThat(sp, hasKey("entity_id"));
+        assertThat(sp.get("entity_id"), equalTo(entityId));
+        assertContainsAttributeWithValue(body, "principal", SAMPLE_IDPUSER_NAME);
+    }
+
+    public void testSpInitiatedSsoFailsForUnknownSp() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        // Validate incoming authentication request
+        Request validateRequest = new Request("POST", "/_idp/saml/validate");
+        validateRequest.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
+        final String nameIdFormat = TRANSIENT;
+        final String relayString = null;
+        final boolean forceAuthn = randomBoolean();
+        final AuthnRequest authnRequest = buildAuthnRequest(entityId + randomAlphaOfLength(4), new URL(acsUrl),
+            new URL("https://idp.org/sso/redirect"), nameIdFormat, forceAuthn);
+        final String query = getQueryString(authnRequest, relayString, false, null);
+        validateRequest.setJsonEntity("{\"authn_request_query\":\"" + query + "\"}");
+        ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(validateRequest));
+        assertThat(e.getMessage(), containsString("is not registered with this Identity Provider"));
+        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
+    }
+
+    public void testSpInitiatedSsoFailsForMalformedRequest() throws Exception {
+        String acsUrl = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/saml/acs";
+        String entityId = SP_ENTITY_ID;
+        registerServiceProvider(entityId, acsUrl);
+        ensureGreen(SamlServiceProviderIndex.INDEX_NAME);
+        // Validate incoming authentication request
+        Request validateRequest = new Request("POST", "/_idp/saml/validate");
+        validateRequest.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
+        final String nameIdFormat = TRANSIENT;
+        final String relayString = null;
+        final boolean forceAuthn = randomBoolean();
+        final AuthnRequest authnRequest = buildAuthnRequest(entityId + randomAlphaOfLength(4), new URL(acsUrl),
+            new URL("https://idp.org/sso/redirect"), nameIdFormat, forceAuthn);
+        final String query = getQueryString(authnRequest, relayString, false, null);
+        // Skip http parameter name
+        final String queryWithoutParam = query.substring(12);
+        validateRequest.setJsonEntity("{\"authn_request_query\":\"" + queryWithoutParam + "\"}");
+        ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(validateRequest));
+        assertThat(e.getMessage(), containsString("does not contain a SAMLRequest parameter"));
+        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
+        // arbitrarily trim the request
+        final String malformedRequestQuery = query.substring(0, query.length() - randomIntBetween(10, 15));
+        validateRequest.setJsonEntity("{\"authn_request_query\":\"" + malformedRequestQuery + "\"}");
+        ResponseException e1 = expectThrows(ResponseException.class, () -> getRestClient().performRequest(validateRequest));
+        assertThat(e1.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
+    }
+
+    private void registerServiceProvider(String entityId, String acsUrl) throws Exception {
+        Map<String, Object> spFields = new HashMap<>();
+        spFields.put(SamlServiceProviderDocument.Fields.ACS.getPreferredName(), acsUrl);
+        spFields.put(SamlServiceProviderDocument.Fields.ENTITY_ID.getPreferredName(), entityId);
+        spFields.put(SamlServiceProviderDocument.Fields.NAME_ID.getPreferredName(), TRANSIENT);
+        spFields.put(SamlServiceProviderDocument.Fields.NAME.getPreferredName(), "Dummy SP");
+        spFields.put("attributes", Map.of(
+            "principal", "https://saml.elasticsearch.org/attributes/principal"));
+        spFields.put("privileges", Map.of(
+            "resource", entityId,
+            "roles", Map.of("superuser", "sso:superuser")
+        ));
+        Request request =
+            new Request("PUT", "/_idp/saml/sp/" + urlEncode(entityId) + "?refresh=" + WriteRequest.RefreshPolicy.IMMEDIATE.getValue());
+        request.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
+        final XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.map(spFields);
+        request.setJsonEntity(Strings.toString(builder));
+        Response registerResponse = getRestClient().performRequest(request);
+        assertThat(registerResponse.getStatusLine().getStatusCode(), equalTo(200));
+        ObjectPath registerResponseObject = ObjectPath.createFromResponse(registerResponse);
+        Map<String, Object> document = registerResponseObject.evaluate("document");
+        assertThat(document, hasKey("_created"));
+        assertThat(document.get("_created"), equalTo(true));
+        Map<String, Object> serviceProvider = registerResponseObject.evaluate("service_provider");
+        assertThat(serviceProvider, hasKey("entity_id"));
+        assertThat(serviceProvider.get("entity_id"), equalTo(entityId));
+        assertThat(serviceProvider, hasKey("enabled"));
+        assertThat(serviceProvider.get("enabled"), equalTo(true));
+    }
+
+    private String getApiKeyFromCredentials(String username, SecureString password) {
+        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
+            UsernamePasswordToken.basicAuthHeaderValue(username, password)));
+        final CreateApiKeyResponse response = new CreateApiKeyRequestBuilder(client)
+            .setName("test key")
+            .setExpiration(TimeValue.timeValueHours(TimeUnit.DAYS.toHours(7L)))
+            .get();
+        assertNotNull(response);
+        return Base64.getEncoder().encodeToString(
+            (response.getId() + ":" + response.getKey().toString()).getBytes(StandardCharsets.UTF_8));
+    }
+
+    private AuthnRequest buildAuthnRequest(String entityId, URL acs, URL destination, String nameIdFormat, boolean forceAuthn) {
+        final Issuer issuer = samlFactory.buildObject(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(entityId);
+        final NameIDPolicy nameIDPolicy = samlFactory.buildObject(NameIDPolicy.class, NameIDPolicy.DEFAULT_ELEMENT_NAME);
+        nameIDPolicy.setFormat(nameIdFormat);
+        final AuthnRequest authnRequest = samlFactory.buildObject(AuthnRequest.class, AuthnRequest.DEFAULT_ELEMENT_NAME);
+        authnRequest.setID(samlFactory.secureIdentifier());
+        authnRequest.setIssuer(issuer);
+        authnRequest.setIssueInstant(now());
+        authnRequest.setAssertionConsumerServiceURL(acs.toString());
+        authnRequest.setDestination(destination.toString());
+        authnRequest.setNameIDPolicy(nameIDPolicy);
+        authnRequest.setForceAuthn(forceAuthn);
+        return authnRequest;
+    }
+
+    private String getQueryString(AuthnRequest authnRequest, String relayState, boolean sign, @Nullable X509Credential credential) {
+        try {
+            final String request = deflateAndBase64Encode(authnRequest);
+            String queryParam = "SAMLRequest=" + urlEncode(request);
+            if (relayState != null) {
+                queryParam += "&RelayState=" + urlEncode(relayState);
+            }
+            if (sign) {
+                final String algo = SignatureConstants.ALGO_ID_SIGNATURE_RSA_SHA256;
+                queryParam += "&SigAlg=" + urlEncode(algo);
+                final byte[] sig = sign(queryParam, algo, credential);
+                queryParam += "&Signature=" + urlEncode(base64Encode(sig));
+            }
+            return queryParam;
+        } catch (Exception e) {
+            throw new ElasticsearchException("Cannot construct SAML redirect", e);
+        }
+    }
+
+    private String base64Encode(byte[] bytes) {
+        return Base64.getEncoder().encodeToString(bytes);
+    }
+
+    private static String urlEncode(String param) throws UnsupportedEncodingException {
+        return URLEncoder.encode(param, StandardCharsets.UTF_8.name());
+    }
+
+    private String deflateAndBase64Encode(SAMLObject message) throws Exception {
+        Deflater deflater = new Deflater(Deflater.DEFLATED, true);
+        try (ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+             DeflaterOutputStream deflaterStream = new DeflaterOutputStream(bytesOut, deflater)) {
+            String messageStr = samlFactory.toString(XMLObjectSupport.marshall(message), false);
+            deflaterStream.write(messageStr.getBytes(StandardCharsets.UTF_8));
+            deflaterStream.finish();
+            return base64Encode(bytesOut.toByteArray());
+        }
+    }
+
+    private byte[] sign(String text, String algo, X509Credential credential) throws SecurityException {
+        return sign(text.getBytes(StandardCharsets.UTF_8), algo, credential);
+    }
+
+    private byte[] sign(byte[] content, String algo, X509Credential credential) throws SecurityException {
+        return XMLSigningUtil.signWithURI(credential, algo, content);
+    }
+
+    private void assertContainsAttributeWithValue(String message, String attribute, String value) {
+        assertThat(message, containsString("<saml2:Attribute FriendlyName=\"" + attribute + "\" Name=\"https://saml.elasticsearch" +
+            ".org/attributes/" + attribute + "\" NameFormat=\"urn:oasis:names:tc:SAML:2.0:attrname-format:uri\"><saml2:AttributeValue " +
+            "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"xsd:string\">" + value + "</saml2:AttributeValue></saml2" +
+            ":Attribute>"));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequestTests.java
new file mode 100644
index 0000000000000..4969067eb9055
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlInitiateSingleSignOnRequestTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class SamlInitiateSingleSignOnRequestTests extends ESTestCase {
+
+    public void testSerialization() throws Exception {
+        final SamlInitiateSingleSignOnRequest request = new SamlInitiateSingleSignOnRequest();
+        request.setSpEntityId("https://kibana_url");
+        final BytesStreamOutput out = new BytesStreamOutput();
+        request.writeTo(out);
+
+        final SamlInitiateSingleSignOnRequest request1 = new SamlInitiateSingleSignOnRequest(out.bytes().streamInput());
+        assertThat(request1.getSpEntityId(), equalTo(request.getSpEntityId()));
+        final ActionRequestValidationException validationException = request1.validate();
+        assertNull(validationException);
+    }
+
+    public void testValidation() {
+
+        final SamlInitiateSingleSignOnRequest request1 = new SamlInitiateSingleSignOnRequest();
+        final ActionRequestValidationException validationException = request1.validate();
+        assertNotNull(validationException);
+        assertThat(validationException.validationErrors().size(), equalTo(1));
+        assertThat(validationException.validationErrors().get(0), containsString("entity_id is missing"));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequestTests.java
new file mode 100644
index 0000000000000..7e7522103bee3
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/SamlValidateAuthnRequestRequestTests.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class SamlValidateAuthnRequestRequestTests extends ESTestCase {
+
+    public void testSerialization() throws Exception {
+        final SamlValidateAuthnRequestRequest request = new SamlValidateAuthnRequestRequest();
+        request.setQueryString("?SAMLRequest=x&RelayState=y&SigAlg=z&Signature=sig");
+        final BytesStreamOutput out = new BytesStreamOutput();
+        request.writeTo(out);
+
+        final SamlValidateAuthnRequestRequest request1 = new SamlValidateAuthnRequestRequest(out.bytes().streamInput());
+        assertThat(request.getQueryString(), equalTo(request1.getQueryString()));
+        final ActionRequestValidationException exception = request1.validate();
+        assertNull(exception);
+    }
+
+    public void testValidation() {
+        final SamlValidateAuthnRequestRequest request = new SamlValidateAuthnRequestRequest();
+        final ActionRequestValidationException exception = request.validate();
+        assertNotNull(exception);
+        assertThat(exception.validationErrors().size(), equalTo(1));
+        assertThat(exception.validationErrors().get(0), containsString("Authentication request query string must be provided"));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java
new file mode 100644
index 0000000000000..dccb848e78210
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex.DocumentVersion;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndexTests;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.opensaml.saml.saml2.core.NameIDType.EMAIL;
+import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class TransportPutSamlServiceProviderActionTests extends ESTestCase {
+
+    private SamlServiceProviderIndex index;
+    private TransportPutSamlServiceProviderAction action;
+    private SamlIdentityProvider idp;
+    private Instant now;
+
+    @Before
+    public void setupMocks() {
+        index = mock(SamlServiceProviderIndex.class);
+        idp = mock(SamlIdentityProvider.class);
+        when(idp.getAllowedNameIdFormats()).thenReturn(Set.of(TRANSIENT));
+
+        now = Instant.ofEpochMilli(System.currentTimeMillis() + randomLongBetween(-500_000, 500_000));
+        final Clock clock = Clock.fixed(now, randomZone());
+        action = new TransportPutSamlServiceProviderAction(
+            mock(TransportService.class), mock(ActionFilters.class), index, idp, clock);
+    }
+
+    public void testRegisterNewServiceProvider() throws Exception {
+        final SamlServiceProviderDocument document = SamlServiceProviderIndexTests.randomDocument();
+
+        mockExistingDocuments(document.entityId, Set.of());
+
+        final AtomicReference<DocWriteResponse> writeResponse = mockWriteResponse(document, true);
+
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(document, randomFrom(RefreshPolicy.values()));
+        final PlainActionFuture<PutSamlServiceProviderResponse> future = new PlainActionFuture<>();
+        action.doExecute(null, request, future);
+
+        final PutSamlServiceProviderResponse response = future.actionGet();
+        assertThat(response, notNullValue());
+        assertThat(response.getDocId(), equalTo(writeResponse.get().getId()));
+        assertThat(response.getEntityId(), equalTo(document.entityId));
+        assertThat(response.isCreated(), equalTo(true));
+        assertThat(response.isEnabled(), equalTo(document.enabled));
+        assertThat(response.getPrimaryTerm(), equalTo(writeResponse.get().getPrimaryTerm()));
+        assertThat(response.getSeqNo(), equalTo(writeResponse.get().getSeqNo()));
+
+        assertThat(document.created, equalTo(now));
+        assertThat(document.lastModified, equalTo(now));
+    }
+
+    public void testUpdateExistingServiceProvider() throws Exception {
+        final SamlServiceProviderDocument document = SamlServiceProviderIndexTests.randomDocument();
+
+        final SamlServiceProviderDocument existingDocument = SamlServiceProviderIndexTests.randomDocument();
+        existingDocument.entityId = document.entityId;
+        existingDocument.docId = randomAlphaOfLength(42);
+        mockExistingDocuments(document.entityId, Set.of(existingDocument));
+
+        final AtomicReference<DocWriteResponse> writeResponse = mockWriteResponse(document, false);
+
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(document, randomFrom(RefreshPolicy.values()));
+        final PlainActionFuture<PutSamlServiceProviderResponse> future = new PlainActionFuture<>();
+        action.doExecute(null, request, future);
+
+        final PutSamlServiceProviderResponse response = future.actionGet();
+        assertThat(response, notNullValue());
+        assertThat(response.getDocId(), equalTo(existingDocument.docId));
+        assertThat(response.getDocId(), equalTo(writeResponse.get().getId()));
+        assertThat(response.getEntityId(), equalTo(document.entityId));
+        assertThat(response.isCreated(), equalTo(false));
+        assertThat(response.isEnabled(), equalTo(document.enabled));
+        assertThat(response.getPrimaryTerm(), equalTo(writeResponse.get().getPrimaryTerm()));
+        assertThat(response.getSeqNo(), equalTo(writeResponse.get().getSeqNo()));
+
+        assertThat(document.created, equalTo(existingDocument.created));
+        assertThat(document.lastModified, equalTo(now));
+    }
+
+    public void testUnsupportedNameIDFormat() throws Exception {
+        final SamlServiceProviderDocument document = SamlServiceProviderIndexTests.randomDocument();
+        final String invalidFormat = randomFrom(PERSISTENT, EMAIL, randomAlphaOfLength(12));
+        document.setNameIdFormat(invalidFormat);
+
+        final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(document, randomFrom(RefreshPolicy.values()));
+        final PlainActionFuture<PutSamlServiceProviderResponse> future = new PlainActionFuture<>();
+        action.doExecute(null, request, future);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, future::actionGet);
+        assertThat(e.getMessage(), containsString("NameID format [" + invalidFormat + "] is not supported"));
+    }
+
+    public AtomicReference<DocWriteResponse> mockWriteResponse(SamlServiceProviderDocument document, boolean created) {
+        final AtomicReference<DocWriteResponse> writeResponse = new AtomicReference<>();
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, Matchers.arrayWithSize(4));
+
+            SamlServiceProviderDocument doc = (SamlServiceProviderDocument) args[0];
+            assertThat(doc.getDocId(), notNullValue());
+            assertThat(doc, sameInstance(document));
+
+            final DocWriteResponse docWriteResponse = new IndexResponse(
+                new ShardId(randomAlphaOfLengthBetween(4, 12), randomAlphaOfLength(24), randomIntBetween(1, 10)),
+                doc.docId, randomLong(), randomLong(), randomLong(), created);
+            writeResponse.set(docWriteResponse);
+
+            ActionListener listener = (ActionListener) args[args.length - 1];
+            listener.onResponse(docWriteResponse);
+
+            return null;
+        }).when(index).writeDocument(any(SamlServiceProviderDocument.class), any(DocWriteRequest.OpType.class),
+            any(WriteRequest.RefreshPolicy.class), any(ActionListener.class));
+
+        return writeResponse;
+    }
+
+    public void mockExistingDocuments(String expectedEntityId, Set<SamlServiceProviderDocument> documents) {
+        final Set<SamlServiceProviderIndex.DocumentSupplier> documentSuppliers = documents.stream()
+            .map(doc -> new SamlServiceProviderIndex.DocumentSupplier(
+                new DocumentVersion(randomAlphaOfLength(24), randomLong(), randomLong()),
+                () -> doc))
+            .collect(Collectors.toUnmodifiableSet());
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, Matchers.arrayWithSize(2));
+
+            String entityId = (String) args[0];
+            assertThat(entityId, equalTo(expectedEntityId));
+
+            ActionListener<Set<SamlServiceProviderIndex.DocumentSupplier>> listener = (ActionListener) args[args.length - 1];
+            listener.onResponse(documentSuppliers);
+
+            return null;
+        }).when(index).findByEntityId(anyString(), any(ActionListener.class));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnRequestTests.java
new file mode 100644
index 0000000000000..4b83d765b062d
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnRequestTests.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.support.SecondaryAuthentication;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.idp.privileges.ServiceProviderPrivileges;
+import org.elasticsearch.xpack.idp.privileges.UserPrivilegeResolver;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.CloudServiceProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.joda.time.Duration;
+import org.mockito.Mockito;
+import org.opensaml.security.x509.X509Credential;
+
+import java.net.URL;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Set;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.arrayWithSize;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class TransportSamlInitiateSingleSignOnRequestTests extends IdpSamlTestCase {
+
+    public void testGetResponseForRegisteredSp() throws Exception {
+        final SamlInitiateSingleSignOnRequest request = new SamlInitiateSingleSignOnRequest();
+        request.setSpEntityId("https://sp.some.org");
+
+        final PlainActionFuture<SamlInitiateSingleSignOnResponse> future = new PlainActionFuture<>();
+        final TransportSamlInitiateSingleSignOnAction action = setupTransportAction(true);
+        action.doExecute(mock(Task.class), request, future);
+
+        final SamlInitiateSingleSignOnResponse response = future.get();
+        assertThat(response.getEntityId(), equalTo("https://sp.some.org"));
+        assertThat(response.getPostUrl(), equalTo("https://sp.some.org/api/security/v1/saml"));
+        assertThat(response.getSamlResponse(), containsString(TRANSIENT));
+        assertContainsAttributeWithValue(response.getSamlResponse(), "email", "samlenduser@elastic.co");
+        assertContainsAttributeWithValue(response.getSamlResponse(), "name", "Saml Enduser");
+        assertContainsAttributeWithValue(response.getSamlResponse(), "principal", "saml_enduser");
+        assertThat(response.getSamlResponse(), containsString("https://saml.elasticsearch.org/attributes/roles"));
+    }
+
+    public void testGetResponseWithoutSecondaryAuthentication() throws Exception {
+        final SamlInitiateSingleSignOnRequest request = new SamlInitiateSingleSignOnRequest();
+        request.setSpEntityId("https://sp.some.org");
+
+        final PlainActionFuture<SamlInitiateSingleSignOnResponse> future = new PlainActionFuture<>();
+        final TransportSamlInitiateSingleSignOnAction action = setupTransportAction(false);
+        action.doExecute(mock(Task.class), request, future);
+
+        Exception e = expectThrows(Exception.class, () -> future.get());
+        assertThat(e.getCause().getMessage(), containsString("Request is missing secondary authentication"));
+    }
+
+    public void testGetResponseForNotRegisteredSp() throws Exception {
+        final SamlInitiateSingleSignOnRequest request = new SamlInitiateSingleSignOnRequest();
+        request.setSpEntityId("https://sp2.other.org");
+
+        final PlainActionFuture<SamlInitiateSingleSignOnResponse> future = new PlainActionFuture<>();
+        final TransportSamlInitiateSingleSignOnAction action = setupTransportAction(true);
+        action.doExecute(mock(Task.class), request, future);
+
+        Exception e = expectThrows(Exception.class, () -> future.get());
+        assertThat(e.getCause().getMessage(), containsString("https://sp2.other.org"));
+        assertThat(e.getCause().getMessage(), containsString("is not registered with this Identity Provider"));
+    }
+
+    private TransportSamlInitiateSingleSignOnAction setupTransportAction(boolean withSecondaryAuth) throws Exception {
+        final Settings settings = Settings.builder()
+            .put("path.home", createTempDir())
+            .put("xpack.idp.enabled", true)
+            .put("xpack.idp.entity_id", "https://idp.cloud.elastic.co")
+            .put("xpack.idp.sso_endpoint.redirect", "https://idp.cloud.elastic.co/saml/init")
+            .put("xpack.idp.organization.url", "https://cloud.elastic.co")
+            .put("xpack.idp.organization.name", "Cloud Elastic")
+            .put("xpack.idp.contact.email", "some@other.org")
+            .build();
+        final ThreadContext threadContext = new ThreadContext(settings);
+        final ThreadPool threadPool = mock(ThreadPool.class);
+        final SecurityContext securityContext = new SecurityContext(settings, threadContext);
+        final TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet());
+        final ActionFilters actionFilters = mock(ActionFilters.class);
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
+        new Authentication(new User("saml_service_account", "saml_service_role"),
+            new Authentication.RealmRef("default_native", "native", "node_name"),
+            new Authentication.RealmRef("default_native", "native", "node_name"))
+            .writeToContext(threadContext);
+        if (withSecondaryAuth) {
+            new SecondaryAuthentication(securityContext,
+                new Authentication(
+                    new User("saml_enduser", new String[]{"saml_enduser_role"}, "Saml Enduser", "samlenduser@elastic.co",
+                        new HashMap<>(), true),
+                    new Authentication.RealmRef("_es_api_key", "_es_api_key", "node_name"),
+                    new Authentication.RealmRef("_es_api_key", "_es_api_key", "node_name")))
+                .writeToContext(threadContext);
+        }
+
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        final CloudServiceProvider serviceProvider = new CloudServiceProvider("https://sp.some.org",
+            "test sp",
+            true,
+            new URL("https://sp.some.org/api/security/v1/saml"),
+            TRANSIENT,
+            Duration.standardMinutes(5),
+            null,
+            new SamlServiceProvider.AttributeNames(
+                "https://saml.elasticsearch.org/attributes/principal",
+                "https://saml.elasticsearch.org/attributes/name",
+                "https://saml.elasticsearch.org/attributes/email",
+                "https://saml.elasticsearch.org/attributes/roles"),
+            null, false, false);
+        mockRegisteredServiceProvider(resolver, "https://sp.some.org", serviceProvider);
+        mockRegisteredServiceProvider(resolver, "https://sp2.other.org", null);
+        final ServiceProviderDefaults defaults = new ServiceProviderDefaults(
+            "elastic-cloud", TRANSIENT, Duration.standardMinutes(15));
+        final X509Credential signingCredential = readCredentials("RSA", randomFrom(1024, 2048, 4096));
+        final SamlIdentityProvider idp = SamlIdentityProvider
+            .builder(resolver)
+            .fromSettings(env)
+            .signingCredential(signingCredential)
+            .serviceProviderDefaults(defaults)
+            .build();
+        final SamlFactory factory = new SamlFactory();
+        final UserPrivilegeResolver privilegeResolver = Mockito.mock(UserPrivilegeResolver.class);
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, arrayWithSize(2));
+            ActionListener<UserPrivilegeResolver.UserPrivileges> listener
+                = (ActionListener<UserPrivilegeResolver.UserPrivileges>) args[args.length - 1];
+            final UserPrivilegeResolver.UserPrivileges privileges = new UserPrivilegeResolver.UserPrivileges(
+                "saml_enduser", true, Set.of(generateRandomStringArray(5, 8, false, false))
+            );
+            listener.onResponse(privileges);
+            return null;
+        }).when(privilegeResolver).resolve(any(ServiceProviderPrivileges.class), any(ActionListener.class));
+        return new TransportSamlInitiateSingleSignOnAction(transportService, actionFilters, securityContext,
+            idp, factory, privilegeResolver);
+    }
+
+    private void assertContainsAttributeWithValue(String message, String attribute, String value) {
+        assertThat(message, containsString("<saml2:Attribute FriendlyName=\"" + attribute + "\" Name=\"https://saml.elasticsearch" +
+            ".org/attributes/" + attribute + "\" NameFormat=\"urn:oasis:names:tc:SAML:2.0:attrname-format:uri\"><saml2:AttributeValue " +
+            "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"xsd:string\">" + value + "</saml2:AttributeValue></saml2" +
+            ":Attribute>"));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolverTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolverTests.java
new file mode 100644
index 0000000000000..802db6f0f3b9b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/privileges/UserPrivilegeResolverTests.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.privileges;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.same;
+
+public class UserPrivilegeResolverTests extends ESTestCase {
+
+    private Client client;
+    private SecurityContext securityContext;
+    private UserPrivilegeResolver resolver;
+
+    @Before
+    public void setupTest() {
+        client = Mockito.mock(Client.class);
+        securityContext = new SecurityContext(Settings.EMPTY, new ThreadContext(Settings.EMPTY));
+        resolver = new UserPrivilegeResolver(client, securityContext);
+    }
+
+    public void testResolveZeroAccess() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final String app = randomAlphaOfLengthBetween(3, 8);
+        setupUser(username);
+        setupHasPrivileges(username, app);
+        final PlainActionFuture<UserPrivilegeResolver.UserPrivileges> future = new PlainActionFuture<>();
+        resolver.resolve(service(app, "cluster:" + randomLong(),
+            Map.of("viewer", "role:cluster:view", "admin", "role:cluster:admin")), future);
+        final UserPrivilegeResolver.UserPrivileges privileges = future.get();
+        assertThat(privileges.principal, equalTo(username));
+        assertThat(privileges.hasAccess, equalTo(false));
+        assertThat(privileges.roles, emptyIterable());
+    }
+
+    public void testResolveSsoWithNoRoleAccess() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final String app = randomAlphaOfLengthBetween(3, 8);
+        final String resource = "cluster:" + MessageDigests.toHexString(randomByteArrayOfLength(16));
+        final String viewerAction = "role:cluster:view";
+        final String adminAction = "role:cluster:admin";
+
+        setupUser(username);
+        setupHasPrivileges(username, app, access(resource, viewerAction, false), access(resource, adminAction, false));
+
+        final PlainActionFuture<UserPrivilegeResolver.UserPrivileges> future = new PlainActionFuture<>();
+        resolver.resolve(service(app, resource, Map.of("viewer", viewerAction, "admin", adminAction)), future);
+        final UserPrivilegeResolver.UserPrivileges privileges = future.get();
+        assertThat(privileges.principal, equalTo(username));
+        assertThat(privileges.hasAccess, equalTo(false));
+        assertThat(privileges.roles, emptyIterable());
+    }
+
+    public void testResolveSsoWithSingleRole() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final String app = randomAlphaOfLengthBetween(3, 8);
+        final String resource = "cluster:" + MessageDigests.toHexString(randomByteArrayOfLength(16));
+        final String viewerAction = "role:cluster:view";
+        final String adminAction = "role:cluster:admin";
+
+        setupUser(username);
+        setupHasPrivileges(username, app, access(resource, viewerAction, true), access(resource, adminAction, false));
+
+        final PlainActionFuture<UserPrivilegeResolver.UserPrivileges> future = new PlainActionFuture<>();
+        resolver.resolve(service(app, resource, Map.of("viewer", viewerAction, "admin", adminAction)), future);
+        final UserPrivilegeResolver.UserPrivileges privileges = future.get();
+        assertThat(privileges.principal, equalTo(username));
+        assertThat(privileges.hasAccess, equalTo(true));
+        assertThat(privileges.roles, containsInAnyOrder("viewer"));
+    }
+
+    public void testResolveSsoWithMultipleRoles() throws Exception {
+        final String username = randomAlphaOfLengthBetween(4, 12);
+        final String app = randomAlphaOfLengthBetween(3, 8);
+        final String resource = "cluster:" + MessageDigests.toHexString(randomByteArrayOfLength(16));
+        final String viewerAction = "role:cluster:view";
+        final String adminAction = "role:cluster:admin";
+        final String operatorAction = "role:cluster:operator";
+        final String monitorAction = "role:cluster:monitor";
+
+        setupUser(username);
+        setupHasPrivileges(username, app,
+            access(resource, viewerAction, false),
+            access(resource, adminAction, false),
+            access(resource, operatorAction, true),
+            access(resource, monitorAction, true)
+        );
+
+        final PlainActionFuture<UserPrivilegeResolver.UserPrivileges> future = new PlainActionFuture<>();
+        resolver.resolve(service(app, resource,
+            Map.of("viewer", viewerAction, "admin", adminAction, "operator", operatorAction, "monitor", monitorAction)),
+            future);
+        final UserPrivilegeResolver.UserPrivileges privileges = future.get();
+        assertThat(privileges.principal, equalTo(username));
+        assertThat(privileges.hasAccess, equalTo(true));
+        assertThat(privileges.roles, containsInAnyOrder("operator", "monitor"));
+    }
+
+    private ServiceProviderPrivileges service(String appName, String resource, Map<String, String> roles) {
+        return new ServiceProviderPrivileges(appName, resource, roles);
+    }
+
+    private HasPrivilegesResponse setupHasPrivileges(String username, String appName,
+                                                     Tuple<String, Tuple<String, Boolean>>... resourceActionAccess) {
+        final boolean isCompleteMatch = randomBoolean();
+        final Map<String, Map<String, Boolean>> resourcePrivilegeMap = new HashMap<>(resourceActionAccess.length);
+        for (Tuple<String, Tuple<String, Boolean>> t : resourceActionAccess) {
+            final String resource = t.v1();
+            final String action = t.v2().v1();
+            final Boolean access = t.v2().v2();
+            resourcePrivilegeMap.computeIfAbsent(resource, ignore -> new HashMap<>()).put(action, access);
+        }
+        final Collection<ResourcePrivileges> privileges = resourcePrivilegeMap.entrySet().stream()
+            .map(e -> ResourcePrivileges.builder(e.getKey()).addPrivileges(e.getValue()).build())
+            .collect(Collectors.toList());
+        final Map<String, Collection<ResourcePrivileges>> appPrivs = Map.of(appName, privileges);
+        final HasPrivilegesResponse response = new HasPrivilegesResponse(username, isCompleteMatch, Map.of(), Set.of(), appPrivs);
+
+        Mockito.doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args.length, equalTo(3));
+            ActionListener<HasPrivilegesResponse> listener = (ActionListener<HasPrivilegesResponse>) args[args.length - 1];
+            listener.onResponse(response);
+            return null;
+        }).when(client).execute(same(HasPrivilegesAction.INSTANCE), any(HasPrivilegesRequest.class), any(ActionListener.class));
+        return response;
+    }
+
+    private Tuple<String, Tuple<String, Boolean>> access(String resource, String action, boolean access) {
+        return new Tuple<>(resource, new Tuple<>(action, access));
+    }
+
+    private void setupUser(String principal) {
+        final User user = new User(principal, randomAlphaOfLengthBetween(6, 12));
+        securityContext.setUser(user, Version.CURRENT);
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseBuilderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseBuilderTests.java
new file mode 100644
index 0000000000000..a52eb2fcfd084
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/FailedAuthenticationResponseBuilderTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.support.XmlValidator;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.junit.Before;
+import org.opensaml.saml.saml2.core.Response;
+import org.opensaml.saml.saml2.core.StatusCode;
+
+import java.time.Clock;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class FailedAuthenticationResponseBuilderTests extends IdpSamlTestCase {
+
+    private SamlIdentityProvider idp;
+    private XmlValidator validator;
+    private SamlFactory samlFactory;
+
+    @Before
+    public void setupSaml() throws Exception {
+        SamlInit.initialize();
+        samlFactory = new SamlFactory();
+        validator = new XmlValidator("saml-schema-protocol-2.0.xsd");
+
+        idp = mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn("https://cloud.elastic.co/saml/idp");
+    }
+
+    public void testSimpleErrorResponseIsValid() throws Exception {
+        final Clock clock = Clock.systemUTC();
+        final FailedAuthenticationResponseMessageBuilder builder = new FailedAuthenticationResponseMessageBuilder(samlFactory, clock, idp);
+        final Response response = builder
+            .setAcsUrl("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs")
+            .setPrimaryStatusCode(StatusCode.REQUESTER)
+            .setInResponseTo(randomAlphaOfLength(12))
+            .build();
+        final String xml = super.toString(response);
+        validator.validate(xml);
+    }
+
+    public void testErrorResponseWithCodeIsValid() throws Exception {
+        final Clock clock = Clock.systemUTC();
+        final FailedAuthenticationResponseMessageBuilder builder = new FailedAuthenticationResponseMessageBuilder(samlFactory, clock, idp);
+        final Response response = builder
+            .setAcsUrl("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs")
+            .setPrimaryStatusCode(StatusCode.REQUESTER)
+            .setInResponseTo(randomAlphaOfLength(12))
+            .setSecondaryStatusCode(StatusCode.INVALID_NAMEID_POLICY)
+            .build();
+        final String xml = super.toString(response);
+        validator.validate(xml);
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidatorTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidatorTests.java
new file mode 100644
index 0000000000000..62a42340aca99
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SamlAuthnRequestValidatorTests.java
@@ -0,0 +1,312 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.xpack.idp.action.SamlValidateAuthnRequestResponse;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.junit.Before;
+import org.mockito.Mockito;
+import org.opensaml.core.xml.util.XMLObjectSupport;
+import org.opensaml.saml.common.SAMLObject;
+import org.opensaml.saml.saml2.core.AuthnRequest;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.NameIDPolicy;
+import org.opensaml.security.SecurityException;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.xmlsec.crypto.XMLSigningUtil;
+import org.opensaml.xmlsec.signature.support.SignatureConstants;
+
+import java.io.ByteArrayOutputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Set;
+import java.util.zip.Deflater;
+import java.util.zip.DeflaterOutputStream;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.joda.time.DateTime.now;
+import static org.mockito.Mockito.when;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class SamlAuthnRequestValidatorTests extends IdpSamlTestCase {
+
+    private SamlAuthnRequestValidator validator;
+    private SamlIdentityProvider idp;
+    private SamlFactory samlFactory = new SamlFactory();
+
+    @Before
+    public void setupValidator() throws Exception {
+        SamlInit.initialize();
+        idp = Mockito.mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn("https://cloud.elastic.co/saml/idp");
+        when(idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI)).thenReturn(new URL("https://cloud.elastic.co/saml/init"));
+        final SamlServiceProvider sp1 = Mockito.mock(SamlServiceProvider.class);
+        when(sp1.getEntityId()).thenReturn("https://sp1.kibana.org");
+        when(sp1.getAssertionConsumerService()).thenReturn(new URL("https://sp1.kibana.org/saml/acs"));
+        when(sp1.getAllowedNameIdFormat()).thenReturn(TRANSIENT);
+        when(sp1.shouldSignAuthnRequests()).thenReturn(false);
+        final SamlServiceProvider sp2 = Mockito.mock(SamlServiceProvider.class);
+        when(sp2.getEntityId()).thenReturn("https://sp2.kibana.org");
+        when(sp2.getAssertionConsumerService()).thenReturn(new URL("https://sp2.kibana.org/saml/acs"));
+        when(sp2.getAllowedNameIdFormat()).thenReturn(PERSISTENT);
+        when(sp2.getSpSigningCredentials()).thenReturn(Set.of(readCredentials("RSA", 4096)));
+        when(sp2.shouldSignAuthnRequests()).thenReturn(true);
+        mockRegisteredServiceProvider(idp, "https://sp1.kibana.org", sp1);
+        mockRegisteredServiceProvider(idp, "https://sp2.kibana.org", sp2);
+        validator = new SamlAuthnRequestValidator(samlFactory, idp);
+    }
+
+    public void testValidAuthnRequest() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        SamlValidateAuthnRequestResponse response = future.actionGet();
+        assertThat(response.isForceAuthn(), equalTo(false));
+        assertThat(response.getSpEntityId(), equalTo("https://sp1.kibana.org"));
+        assertThat(response.getAuthnState().size(), equalTo(4));
+        assertThat(response.getAuthnState().get("authn_request_id"), equalTo(authnRequest.getID()));
+        assertThat(response.getAuthnState().get("nameid_format"), equalTo(TRANSIENT));
+    }
+
+    public void testValidSignedAuthnRequest() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp2.kibana.org", new URL("https://sp2.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), PERSISTENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState, true,
+            readCredentials("RSA", 4096)), future);
+        SamlValidateAuthnRequestResponse response = future.actionGet();
+        assertThat(response.isForceAuthn(), equalTo(false));
+        assertThat(response.getSpEntityId(), equalTo("https://sp2.kibana.org"));
+        assertThat(response.getAuthnState().size(), equalTo(4));
+        assertThat(response.getAuthnState().get("authn_request_id"), equalTo(authnRequest.getID()));
+        assertThat(response.getAuthnState().get("nameid_format"), equalTo(PERSISTENT));
+    }
+
+    public void testValidSignedAuthnRequestWithoutRelayState() throws Exception {
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp2.kibana.org", new URL("https://sp2.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), PERSISTENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, null, true,
+            readCredentials("RSA", 4096)), future);
+        SamlValidateAuthnRequestResponse response = future.actionGet();
+        assertThat(response.isForceAuthn(), equalTo(false));
+        assertThat(response.getSpEntityId(), equalTo("https://sp2.kibana.org"));
+        assertThat(response.getAuthnState().size(), equalTo(4));
+        assertThat(response.getAuthnState().get("authn_request_id"), equalTo(authnRequest.getID()));
+        assertThat(response.getAuthnState().get("nameid_format"), equalTo(PERSISTENT));
+    }
+
+    public void testValidSignedAuthnRequestWhenServiceProviderShouldNotSign() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState, true, readCredentials("RSA", 4096)), future);
+        SamlValidateAuthnRequestResponse response = future.actionGet();
+        assertThat(response.isForceAuthn(), equalTo(false));
+        assertThat(response.getSpEntityId(), equalTo("https://sp1.kibana.org"));
+        assertThat(response.getAuthnState().size(), equalTo(4));
+        assertThat(response.getAuthnState().get("authn_request_id"), equalTo(authnRequest.getID()));
+        assertThat(response.getAuthnState().get("nameid_format"), equalTo(TRANSIENT));
+    }
+
+    public void testValidUnSignedAuthnRequestWhenServiceProviderShouldSign() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp2.kibana.org", new URL("https://sp2.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("must sign authentication requests but no signature was found"));
+    }
+
+    public void testSignedAuthnRequestWithWrongKey() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp2.kibana.org", new URL("https://sp2.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState, true, readCredentials("RSA2", 4096)), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("Unable to validate signature of authentication request"));
+    }
+
+    public void testSignedAuthnRequestWithWrongSizeKey() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp2.kibana.org", new URL("https://sp2.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState, true, readCredentials("RSA", 2048)), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("Unable to validate signature of authentication request"));
+    }
+
+    public void testWrongDestination() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            new URL("https://wrong.destination.org"), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("but the SSO endpoint of this Identity Provider is"));
+        assertThat(e.getMessage(), containsString("wrong.destination.org"));
+    }
+
+    public void testUnregisteredAcsForSp() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://malicious.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("The registered ACS URL for this Service Provider is"));
+        assertThat(e.getMessage(), containsString("https://malicious.kibana.org/saml/acs"));
+    }
+
+    public void testUnregisteredSp()throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://unknown.kibana.org", new URL("https://unknown.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        mockRegisteredServiceProvider(idp, "https://unknown.kibana.org", null);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, future::actionGet);
+        assertThat(e.getMessage(), containsString("is not registered with this Identity Provider"));
+        assertThat(e.getMessage(), containsString("https://unknown.kibana.org"));
+    }
+
+    public void testAuthnRequestWithoutAcsUrl() throws Exception{
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        // remove ACS
+        authnRequest.setAssertionConsumerServiceURL(null);
+        final boolean containsIndex = randomBoolean();
+        if (containsIndex) {
+            authnRequest.setAssertionConsumerServiceIndex(randomInt(10));
+        }
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("SAML authentication does not contain an AssertionConsumerService URL"));
+        if (containsIndex) {
+            assertThat(e.getMessage(), containsString("It contains an Assertion Consumer Service Index "));
+        }
+    }
+
+    public void testAuthnRequestWithoutIssuer() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), TRANSIENT);
+        // remove issuer
+        authnRequest.setIssuer(null);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("SAML authentication request has no issuer"));
+    }
+
+    public void testInvalidNameIDPolicy() throws Exception {
+        final String relayState = randomAlphaOfLength(6);
+        final AuthnRequest authnRequest = buildAuthnRequest("https://sp1.kibana.org", new URL("https://sp1.kibana.org/saml/acs"),
+            idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), PERSISTENT);
+        PlainActionFuture<SamlValidateAuthnRequestResponse> future = new PlainActionFuture<>();
+        validator.processQueryString(getQueryString(authnRequest, relayState), future);
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+            future::actionGet);
+        assertThat(e.getMessage(), containsString("doesn't match the allowed NameID format"));
+    }
+
+    private AuthnRequest buildAuthnRequest(String entityId, URL acs, URL destination, String nameIdFormat) {
+        final Issuer issuer = samlFactory.buildObject(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(entityId);
+        final NameIDPolicy nameIDPolicy = samlFactory.buildObject(NameIDPolicy.class, NameIDPolicy.DEFAULT_ELEMENT_NAME);
+        nameIDPolicy.setFormat(nameIdFormat);
+        final AuthnRequest authnRequest = samlFactory.buildObject(AuthnRequest.class, AuthnRequest.DEFAULT_ELEMENT_NAME);
+        authnRequest.setID(samlFactory.secureIdentifier());
+        authnRequest.setIssuer(issuer);
+        authnRequest.setIssueInstant(now());
+        authnRequest.setAssertionConsumerServiceURL(acs.toString());
+        authnRequest.setDestination(destination.toString());
+        authnRequest.setNameIDPolicy(nameIDPolicy);
+        return authnRequest;
+    }
+
+    private String getQueryString(AuthnRequest authnRequest, String relayState) {
+        return getQueryString(authnRequest, relayState, false, null);
+    }
+
+    private String getQueryString(AuthnRequest authnRequest, String relayState, boolean sign, @Nullable X509Credential credential) {
+        try {
+            final String request = deflateAndBase64Encode(authnRequest);
+            String queryParam = "SAMLRequest=" + urlEncode(request);
+            if (relayState != null) {
+                queryParam += "&RelayState=" + urlEncode(relayState);
+            }
+            if (sign) {
+                final String algo = SignatureConstants.ALGO_ID_SIGNATURE_RSA_SHA256;
+                queryParam += "&SigAlg=" + urlEncode(algo);
+                final byte[] sig = sign(queryParam, algo, credential);
+                queryParam += "&Signature=" + urlEncode(base64Encode(sig));
+            }
+            return queryParam;
+        } catch (Exception e) {
+            throw new ElasticsearchException("Cannot construct SAML redirect", e);
+        }
+    }
+
+    private String base64Encode(byte[] bytes) {
+        return Base64.getEncoder().encodeToString(bytes);
+    }
+
+    private String urlEncode(String param) throws UnsupportedEncodingException {
+        return URLEncoder.encode(param, StandardCharsets.UTF_8.name());
+    }
+
+    private String deflateAndBase64Encode(SAMLObject message)
+        throws Exception {
+        Deflater deflater = new Deflater(Deflater.DEFLATED, true);
+        try (ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+             DeflaterOutputStream deflaterStream = new DeflaterOutputStream(bytesOut, deflater)) {
+            String messageStr = samlFactory.toString(XMLObjectSupport.marshall(message), false);
+            deflaterStream.write(messageStr.getBytes(StandardCharsets.UTF_8));
+            deflaterStream.finish();
+            return base64Encode(bytesOut.toByteArray());
+        }
+    }
+
+    private byte[] sign(String text, String algo, X509Credential credential) throws SecurityException {
+        return sign(text.getBytes(StandardCharsets.UTF_8), algo, credential);
+    }
+
+    private byte[] sign(byte[] content, String algo, X509Credential credential) throws SecurityException {
+        return XMLSigningUtil.signWithURI(credential, algo, content);
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilderTests.java
new file mode 100644
index 0000000000000..07aef6cb21ad7
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/authn/SuccessfulAuthenticationResponseMessageBuilderTests.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.authn;
+
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.support.XmlValidator;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.joda.time.Duration;
+import org.junit.Before;
+import org.opensaml.saml.saml2.core.Response;
+
+import java.net.URL;
+import java.time.Clock;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class SuccessfulAuthenticationResponseMessageBuilderTests extends IdpSamlTestCase {
+
+    private SamlIdentityProvider idp;
+    private XmlValidator validator;
+    private SamlFactory samlFactory;
+
+    @Before
+    public void setupSaml() throws Exception {
+        SamlInit.initialize();
+        samlFactory = new SamlFactory();
+        validator = new XmlValidator("saml-schema-protocol-2.0.xsd");
+
+        idp = mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn("https://cloud.elastic.co/saml/idp");
+        when(idp.getSigningCredential()).thenReturn(readCredentials("RSA", 2048));
+        when(idp.getServiceProviderDefaults())
+            .thenReturn(new ServiceProviderDefaults("elastic-cloud", TRANSIENT, Duration.standardMinutes(5)));
+    }
+
+    public void testSignedResponseIsValidAgainstXmlSchema() throws Exception {
+        final Response response = buildResponse();
+        final String xml = super.toString(response);
+        assertThat(xml, containsString("SignedInfo>"));
+        validator.validate(xml);
+    }
+
+    private Response buildResponse() throws Exception{
+        final Clock clock = Clock.systemUTC();
+
+        final SamlServiceProvider sp = mock(SamlServiceProvider.class);
+        final String baseServiceUrl = "https://" + randomAlphaOfLength(32) + ".us-east-1.aws.found.io/";
+        final String acs = baseServiceUrl + "api/security/saml/callback";
+        when(sp.getEntityId()).thenReturn(baseServiceUrl);
+        when(sp.getAssertionConsumerService()).thenReturn(new URL(acs));
+        when(sp.getAuthnExpiry()).thenReturn(Duration.standardMinutes(10));
+        when(sp.getAttributeNames()).thenReturn(new SamlServiceProvider.AttributeNames("principal", null, null, null));
+
+        final UserServiceAuthentication user = mock(UserServiceAuthentication.class);
+        when(user.getPrincipal()).thenReturn(randomAlphaOfLengthBetween(4, 12));
+        when(user.getRoles()).thenReturn(Set.of(randomArray(1, 5, String[]::new, () -> randomAlphaOfLengthBetween(4, 12))));
+        when(user.getEmail()).thenReturn(randomAlphaOfLength(8) + "@elastic.co");
+        when(user.getName()).thenReturn(randomAlphaOfLength(6) + " " + randomAlphaOfLength(8));
+        when(user.getServiceProvider()).thenReturn(sp);
+
+        final SuccessfulAuthenticationResponseMessageBuilder builder =
+            new SuccessfulAuthenticationResponseMessageBuilder(samlFactory, clock, idp);
+        return builder.build(user, null);
+    }
+
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java
new file mode 100644
index 0000000000000..9fcf28eb98377
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java
@@ -0,0 +1,536 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+import org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.Matchers;
+import org.joda.time.Duration;
+import org.mockito.Mockito;
+import org.opensaml.security.x509.X509Credential;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+import java.util.List;
+
+import static org.elasticsearch.test.TestMatchers.throwableWithMessage;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ALLOWED_NAMEID_FORMATS;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_EMAIL;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_GIVEN_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_SURNAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ENTITY_ID;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_DISPLAY_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_URL;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SLO_POST_ENDPOINT;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SLO_REDIRECT_ENDPOINT;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SSO_POST_ENDPOINT;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SSO_REDIRECT_ENDPOINT;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class SamlIdentityProviderBuilderTests extends IdpSamlTestCase {
+
+    public void testAllSettings() throws Exception {
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final Path destSigningKeyPath = dir.resolve("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+        final Path signingCertPath = getDataPath("signing1.crt");
+        final Path destSigningCertPath = dir.resolve("signing1.crt");
+        final X509Certificate signingCert = CertParsingUtils.readX509Certificates(List.of(signingCertPath))[0];
+        Files.copy(signingKeyPath, destSigningKeyPath);
+        Files.copy(signingCertPath, destSigningCertPath);
+
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_SSO_POST_ENDPOINT.getKey(), "https://idp.org/sso/post")
+            .put(IDP_SLO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/slo/redirect")
+            .put(IDP_SLO_POST_ENDPOINT.getKey(), "https://idp.org/slo/post")
+            .putList(IDP_ALLOWED_NAMEID_FORMATS.getKey(), TRANSIENT)
+            .put(IDP_ORGANIZATION_NAME.getKey(), "organization_name")
+            .put(IDP_ORGANIZATION_DISPLAY_NAME.getKey(), "organization_display_name")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_GIVEN_NAME.getKey(), "Tony")
+            .put(IDP_CONTACT_SURNAME.getKey(), "Stark")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .put(ServiceProviderDefaults.APPLICATION_NAME_SETTING.getKey(), "my_super_idp")
+            .put(ServiceProviderDefaults.NAMEID_FORMAT_SETTING.getKey(), PERSISTENT)
+            .put(ServiceProviderDefaults.AUTHN_EXPIRY_SETTING.getKey(), "2m")
+            .put("xpack.idp.signing.key", destSigningKeyPath)
+            .put("xpack.idp.signing.certificate", destSigningCertPath)
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        final ServiceProviderDefaults defaults = ServiceProviderDefaults.forSettings(settings);
+        final SamlIdentityProvider idp = SamlIdentityProvider.builder(resolver).fromSettings(env).serviceProviderDefaults(defaults).build();
+        assertThat(idp.getEntityId(), equalTo("urn:elastic:cloud:idp"));
+        assertThat(idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI).toString(), equalTo("https://idp.org/sso/redirect"));
+        assertThat(idp.getSingleSignOnEndpoint(SAML2_POST_BINDING_URI).toString(), equalTo("https://idp.org/sso/post"));
+        assertThat(idp.getSingleLogoutEndpoint(SAML2_REDIRECT_BINDING_URI).toString(), equalTo("https://idp.org/slo/redirect"));
+        assertThat(idp.getSingleLogoutEndpoint(SAML2_POST_BINDING_URI).toString(), equalTo("https://idp.org/slo/post"));
+        assertThat(idp.getAllowedNameIdFormats(), hasSize(1));
+        assertThat(idp.getAllowedNameIdFormats(), Matchers.contains(TRANSIENT));
+        assertThat(idp.getOrganization(), equalTo(new SamlIdentityProvider.OrganizationInfo("organization_name",
+            "organization_display_name", "https://idp.org")));
+        assertThat(idp.getServiceProviderDefaults().applicationName, equalTo("my_super_idp"));
+        assertThat(idp.getServiceProviderDefaults().nameIdFormat, equalTo(PERSISTENT));
+        assertThat(idp.getServiceProviderDefaults().authenticationExpiry, equalTo(Duration.standardMinutes(2)));
+        assertThat(idp.getSigningCredential().getEntityCertificate(), equalTo(signingCert));
+        assertThat(idp.getSigningCredential().getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testMissingCredentials() {
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_SSO_POST_ENDPOINT.getKey(), "https://idp.org/sso/post")
+            .put(IDP_SLO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/slo/redirect")
+            .put(IDP_SLO_POST_ENDPOINT.getKey(), "https://idp.org/slo/post")
+            .put(IDP_ALLOWED_NAMEID_FORMATS.getKey(), TRANSIENT)
+            .put(IDP_ORGANIZATION_NAME.getKey(), "organization_name")
+            .put(IDP_ORGANIZATION_DISPLAY_NAME.getKey(), "organization_display_name")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_GIVEN_NAME.getKey(), "Tony")
+            .put(IDP_CONTACT_SURNAME.getKey(), "Stark")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        final ServiceProviderDefaults defaults = new ServiceProviderDefaults(
+            randomAlphaOfLengthBetween(4, 8), randomFrom(TRANSIENT, PERSISTENT),
+            Duration.standardMinutes(randomIntBetween(2, 90)));
+        IllegalArgumentException e = LuceneTestCase.expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProvider.builder(resolver).fromSettings(env).serviceProviderDefaults(defaults).build());
+        assertThat(e, instanceOf(ValidationException.class));
+        assertThat(e.getMessage(), containsString("Signing credential must be specified"));
+    }
+
+    public void testDefaultAllowedNameIdFormats() throws Exception {
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final Path destSigningKeyPath = dir.resolve("signing1.key");
+        final Path signingCertPath = getDataPath("signing1.crt");
+        final Path destSigningCertPath = dir.resolve("signing1.crt");
+        Files.copy(signingKeyPath, destSigningKeyPath);
+        Files.copy(signingCertPath, destSigningCertPath);
+
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_SSO_POST_ENDPOINT.getKey(), "https://idp.org/sso/post")
+            .put(IDP_SLO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/slo/redirect")
+            .put(IDP_SLO_POST_ENDPOINT.getKey(), "https://idp.org/slo/post")
+            .put(IDP_ALLOWED_NAMEID_FORMATS.getKey(), TRANSIENT)
+            .put(IDP_ORGANIZATION_NAME.getKey(), "organization_name")
+            .put(IDP_ORGANIZATION_DISPLAY_NAME.getKey(), "organization_display_name")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_GIVEN_NAME.getKey(), "Tony")
+            .put(IDP_CONTACT_SURNAME.getKey(), "Stark")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .put(ServiceProviderDefaults.APPLICATION_NAME_SETTING.getKey(), "my_super_idp")
+            .put(ServiceProviderDefaults.NAMEID_FORMAT_SETTING.getKey(), PERSISTENT)
+            .put(ServiceProviderDefaults.AUTHN_EXPIRY_SETTING.getKey(), "2m")
+            .put("xpack.idp.signing.key", destSigningKeyPath)
+            .put("xpack.idp.signing.certificate", destSigningCertPath)
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        final ServiceProviderDefaults defaults = ServiceProviderDefaults.forSettings(settings);
+        final SamlIdentityProvider idp = SamlIdentityProvider.builder(resolver).fromSettings(env).serviceProviderDefaults(defaults).build();
+        assertThat(idp.getAllowedNameIdFormats(), hasSize(1));
+        assertThat(idp.getAllowedNameIdFormats(), Matchers.contains(TRANSIENT));
+    }
+
+    public void testConfigurationWithForbiddenAllowedNameIdFormats() throws Exception {
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final Path destSigningKeyPath = dir.resolve("signing1.key");
+        final Path signingCertPath = getDataPath("signing1.crt");
+        final Path destSigningCertPath = dir.resolve("signing1.crt");
+        Files.copy(signingKeyPath, destSigningKeyPath);
+        Files.copy(signingCertPath, destSigningCertPath);
+
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_SSO_POST_ENDPOINT.getKey(), "https://idp.org/sso/post")
+            .put(IDP_SLO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/slo/redirect")
+            .put(IDP_SLO_POST_ENDPOINT.getKey(), "https://idp.org/slo/post")
+            .putList(IDP_ALLOWED_NAMEID_FORMATS.getKey(), TRANSIENT, PERSISTENT)
+            .put(IDP_ORGANIZATION_NAME.getKey(), "organization_name")
+            .put(IDP_ORGANIZATION_DISPLAY_NAME.getKey(), "organization_display_name")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_GIVEN_NAME.getKey(), "Tony")
+            .put(IDP_CONTACT_SURNAME.getKey(), "Stark")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .put(ServiceProviderDefaults.APPLICATION_NAME_SETTING.getKey(), "my_super_idp")
+            .put(ServiceProviderDefaults.NAMEID_FORMAT_SETTING.getKey(), PERSISTENT)
+            .put(ServiceProviderDefaults.AUTHN_EXPIRY_SETTING.getKey(), "2m")
+            .put("xpack.idp.signing.key", destSigningKeyPath)
+            .put("xpack.idp.signing.certificate", destSigningCertPath)
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        final ServiceProviderDefaults defaults = ServiceProviderDefaults.forSettings(settings);
+        IllegalArgumentException e = LuceneTestCase.expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProvider.builder(resolver).fromSettings(env).build());
+        assertThat(e.getMessage(), containsString("are not valid NameID formats. Allowed values are"));
+        assertThat(e.getMessage(), containsString(PERSISTENT));
+    }
+
+    public void testInvalidSsoEndpoint() {
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "not a url")
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        IllegalArgumentException e = LuceneTestCase.expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProvider.builder(resolver).fromSettings(env).build());
+        assertThat(e.getMessage(), containsString(IDP_SSO_REDIRECT_ENDPOINT.getKey()));
+        assertThat(e.getMessage(), containsString("Not a valid URL"));
+    }
+
+    public void testMissingSsoRedirectEndpoint() {
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_ORGANIZATION_NAME.getKey(), "The Organization")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        IllegalArgumentException e = LuceneTestCase.expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProvider.builder(resolver).fromSettings(env).build());
+        assertThat(e.getMessage(), containsString(IDP_SSO_REDIRECT_ENDPOINT.getKey()));
+        assertThat(e.getMessage(), containsString("is required"));
+    }
+
+    public void testMalformedOrganizationUrl() {
+        Settings settings = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_ORGANIZATION_URL.getKey(), "The Org")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .put(IDP_ORGANIZATION_NAME.getKey(), "The Organization")
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final SamlServiceProviderResolver resolver = Mockito.mock(SamlServiceProviderResolver.class);
+        IllegalArgumentException e = LuceneTestCase.expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProvider.builder(resolver).fromSettings(env).build());
+        assertThat(e.getMessage(), containsString(IDP_ORGANIZATION_URL.getKey()));
+        assertThat(e.getMessage(), containsString("Not a valid URL"));
+    }
+
+    public void testCreateSigningCredentialFromPemFiles() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final Path destSigningKeyPath = dir.resolve("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+        final Path signingCertPath = getDataPath("signing1.crt");
+        final Path destSigningCertPath = dir.resolve("signing1.crt");
+        final X509Certificate signingCert = CertParsingUtils.readX509Certificates(List.of(signingCertPath))[0];
+        Files.copy(signingKeyPath, destSigningKeyPath);
+        Files.copy(signingCertPath, destSigningCertPath);
+        builder.put("xpack.idp.signing.key", destSigningKeyPath);
+        builder.put("xpack.idp.signing.certificate", destSigningCertPath);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing.");
+
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+        assertThat(credential.getPublicKey(), equalTo(signingCert.getPublicKey()));
+    }
+
+    public void testCreateSigningCredentialFromΚeystoreWithSingleEntry() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.path", destSigningKeyStorePath);
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing.");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateSigningCredentialFromKeyStoreWithSingleEntryAndConfiguredAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.alias", "signing1");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing.");
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateSigningCredentialFromKeyStoreWithSingleEntryButWrongAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.alias", "some-other");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing."));
+        assertThat(e, throwableWithMessage(
+            "The configured credential [xpack.idp.signing.keystore] with alias [some-other] is not a valid signing key"
+                + " - There is no private key available for this credential"));
+    }
+
+    public void testCreateSigningCredentialFromKeyStoreWithMultipleEntriesAndConfiguredAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+        final String ks = randomFrom("multi_signing.jks", "multi_signing.p12");
+        final String alias = randomFrom("signing1", "signing2", "signing3", "signing4");
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath(ks);
+        final Path destSigningKeyStorePath = dir.resolve(ks);
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.alias", alias);
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing.");
+        final Path signingKeyPath = getDataPath(alias + ".key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateSigningCredentialFromKeyStoreWithMultipleEntriesButWrongAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("multi_signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("multi_signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.signing.keystore.alias", "some-other");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.signing."));
+        assertThat(e, throwableWithMessage(
+            "The configured credential [xpack.idp.signing.keystore] with alias [some-other] is not a valid signing key"
+                + " - There is no private key available for this credential"));
+    }
+
+    public void testCreateMetadataSigningCredentialFromΚeystoreWithSingleEntry() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.path", destSigningKeyStorePath);
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.metadata_signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.metadata_signing.");
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateMetadataSigningCredentialFromKeyStoreWithSingleEntryAndConfiguredAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.alias", "signing1");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.metadata_signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final Path signingKeyPath = getDataPath("signing1.key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.metadata_signing.");
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateMetadataSigningCredentialFromKeyStoreWithSingleEntryButWrongAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.alias", "some-other");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.metadata_signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.metadata_signing."));
+        assertThat(e, throwableWithMessage(
+            "The configured credential [xpack.idp.metadata_signing.keystore] with alias [some-other] is not a valid signing key"
+                + " - There is no private key available for this credential"));
+    }
+
+    public void testCreateMetadataSigningCredentialFromKeyStoreWithMultipleEntriesAndConfiguredAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+        final String ks = randomFrom("multi_signing.jks", "multi_signing.p12");
+        final String alias = randomFrom("signing1", "signing2", "signing3", "signing4");
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath(ks);
+        final Path destSigningKeyStorePath = dir.resolve(ks);
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.alias", alias);
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.metadata_signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        final X509Credential credential = SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.metadata_signing.");
+        final Path signingKeyPath = getDataPath(alias + ".key");
+        final PrivateKey signingKey = PemUtils.readPrivateKey(signingKeyPath, () -> null);
+
+        assertThat(credential, notNullValue());
+        assertThat(credential.getPrivateKey(), equalTo(signingKey));
+    }
+
+    public void testCreateMetadataSigningCredentialFromKeyStoreWithMultipleEntriesButWrongAlias() throws Exception {
+        Settings.Builder builder = Settings.builder()
+            .put("path.home", LuceneTestCase.createTempDir())
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect");
+
+        final Path dir = LuceneTestCase.createTempDir("signing");
+        final Path signingKeyStorePath = getDataPath("multi_signing.p12");
+        final Path destSigningKeyStorePath = dir.resolve("multi_signing.p12");
+        Files.copy(signingKeyStorePath, destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.path", destSigningKeyStorePath);
+        builder.put("xpack.idp.metadata_signing.keystore.alias", "some-other");
+
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.idp.metadata_signing.keystore.secure_password", "signing");
+        builder.setSecureSettings(secureSettings);
+        final Settings settings = builder.build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> SamlIdentityProviderBuilder.buildSigningCredential(env, settings, "xpack.idp.metadata_signing."));
+        assertThat(e, throwableWithMessage(
+            "The configured credential [xpack.idp.metadata_signing.keystore] with alias [some-other] is not a valid signing key"
+                + " - There is no private key available for this credential"));
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdpMetadataBuilderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdpMetadataBuilderTests.java
new file mode 100644
index 0000000000000..d536c201303dd
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdpMetadataBuilderTests.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.elasticsearch.common.util.NamedFormatter;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+import org.opensaml.saml.saml2.metadata.EntityDescriptor;
+import org.opensaml.saml.saml2.metadata.impl.EntityDescriptorMarshaller;
+import org.w3c.dom.Element;
+
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class SamlIdpMetadataBuilderTests extends IdpSamlTestCase {
+
+    private SamlFactory samlFactory;
+
+    @Before
+    public void setup() throws Exception {
+        SamlInit.initialize();
+        samlFactory = new SamlFactory();
+    }
+
+    public void testSimpleMetadataGeneration() throws Exception {
+        final String entityId = "https://idp.org";
+        final EntityDescriptor entityDescriptor = new SamlIdPMetadataBuilder(entityId)
+            .withSingleSignOnServiceUrl(SAML2_REDIRECT_BINDING_URI, new URL(entityId + "/sso/redirect"))
+            .build();
+        final Element element = new EntityDescriptorMarshaller().marshall(entityDescriptor);
+        final String xml = samlFactory.toString(element, false);
+        assertThat(xml, equalTo("<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
+            "<md:EntityDescriptor xmlns:md=\"urn:oasis:names:tc:SAML:2.0:metadata\" entityID=\"https://idp.org\">" +
+            "<md:IDPSSODescriptor WantAuthnRequestsSigned=\"false\" protocolSupportEnumeration=\"urn:oasis:names:tc:SAML:2.0:protocol\">" +
+            "<md:SingleSignOnService Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect\" " +
+            "Location=\"https://idp.org/sso/redirect\"/>" +
+            "</md:IDPSSODescriptor>" +
+            "</md:EntityDescriptor>"));
+    }
+
+    public void testMetadataGenerationWithAllParameters() throws Exception {
+        final String entityId = "https://idp.org";
+        final EntityDescriptor entityDescriptor = new SamlIdPMetadataBuilder(entityId)
+            .withLocale(Locale.forLanguageTag("en"))
+            .withSingleSignOnServiceUrl(SAML2_REDIRECT_BINDING_URI, new URL(entityId + "/sso/redirect"))
+            .withSingleSignOnServiceUrl(SAML2_POST_BINDING_URI, new URL(entityId + "/sso/post"))
+            .withSingleLogoutServiceUrl(SAML2_REDIRECT_BINDING_URI, new URL(entityId + "/slo/redirect"))
+            .withSingleLogoutServiceUrl(SAML2_POST_BINDING_URI, new URL(entityId + "/slo/post"))
+            .withNameIdFormat(PERSISTENT)
+            .withNameIdFormat(TRANSIENT)
+            .wantAuthnRequestsSigned(true)
+            .withSigningCertificate(readCredentials("RSA", 2048).getEntityCertificate())
+            .withSigningCertificate(readCredentials("RSA", 4096).getEntityCertificate())
+            .withContact("technical", "Tony", "Stark", "tony@starkindustries.com")
+            .organization("Avengers", "The Avengers", "https://linktotheidp.org")
+            .build();
+        final Element element = new EntityDescriptorMarshaller().marshall(entityDescriptor);
+        final String xml = samlFactory.toString(element, false);
+
+        // RSA_2048
+        final String signingCertificateOne = joinCertificateLines(
+            "MIIDYzCCAkugAwIBAgIVAITQVqXYYUT0w04Z2gWAZ6pv7gwbMA0GCSqGSIb3DQEBCwUAMDQxMjAw",
+            "BgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMCAXDTIwMDEy",
+            "ODA2MzczNloYDzIxNjgxMDE5MDYzNzM2WjBRMRMwEQYKCZImiZPyLGQBGRYDb3JnMR0wGwYKCZIm",
+            "iZPyLGQBGRYNZWxhc3RpY3NlYXJjaDEMMAoGA1UECxMDaWRwMQ0wCwYDVQQDEwR0ZXN0MIIBIjAN",
+            "BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAujk+mVzI+qmf4gSJZdVVDdhFTi06kikb7FxG5JPu",
+            "+gmU9Ke0LVEpP7Jp3gmhwsa18JUuvaepL1jnKmbbepKkEsvqUj4FuI/gImvFwb7X+xUwzNTYZAEv",
+            "nZ4n16k0sBPuDuDibF0MGniVeLG3bD2VF3crFQrphFr+GZSXbVk5zIcSf6D6nSDcKmCNpVAK3jX9",
+            "iV0nkr8cPtHOgprv1Y7mZgk5jwli9to0QD7r7OG5Db34R06JTMGTji+RULPISH1bc8FHdurRASkG",
+            "msei5GlJqPSuKdViuaKPmDrvKR8OK6gMzUd3pikJgD8veLxEuZ640FHPndPlvwJrSLwhitRgPQID",
+            "AQABo00wSzAdBgNVHQ4EFgQUD87H31WVQNfCc85/H2qhpzs3XfowHwYDVR0jBBgwFoAUVenAN+T0",
+            "6rqNDxjMcvgimnTw+FgwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAIvHYxT30cvoHWUE2",
+            "saDVJ4qs/e0G3WusDyem3e4HkqwLEah06RDSgVCaOfW3ey5Q6CIQW3HHGUYqO0nU8JVCWdAk3+bU",
+            "YJJOeLnwD+SbDxxKBhxLdx+BjWata85lfJTR9+dXs0RXAAN8dSiIaj9NSgnwiJqQQZf7i66S7XB5",
+            "8TDTdZlV3d26STLy5h7Uy6vyCka8Xu8HFQ4hH2qf2L6EhBbzVTB6tuyPQOQwrlLE65nhUNkfBbjZ",
+            "lre45UMc9GuxzHkbvd3HEQaroMHZxnu+/n/JDlgsrCYUEXnZnOXvgUPupPynoRdDN1F6r95TLyU9",
+            "pYjDf/6zNPE854VF6y1TqQ=="
+            );
+        // RSA_4096
+        final String signingCertificateTwo = joinCertificateLines(
+            "MIIFCTCCAvGgAwIBAgIUei1EtkLvWStQusThWwgO14R+gFowDQYJKoZIhvcNAQELBQAwFDESMBAG",
+            "A1UEAwwJc2FtbC10ZXN0MB4XDTIwMDIwNjIwMzA0OFoXDTIwMDMwNzIwMzA0OFowFDESMBAGA1UE",
+            "AwwJc2FtbC10ZXN0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuylnpy9h6cKUz6uC",
+            "Bi6eHBNW5GN6fQb1rU3BLjj5BMbXOa4Ebj84OOsNVtS5/vskMtBH5R+Y1IT49Evd0pKgkJtvbso7",
+            "avq/jfmi18uDI2Q7BjTzWC7aOkfaYXr5aBPEQpapjxV6QxirgMqz5x4WTgfBDfRr+gw2oXWsaVaQ",
+            "7xkECeIIra7j0vgOfTfSR4BQlbEiZktVvaE882kKGZPdDvklGs1/3DMHM09kRpxR/CUc3s8XtjH3",
+            "DjUhK2MxX77gXxRXMMrWNWpk1/oXF/MnTkIiJhYqD7lnudbGXaLnaP38M6/OtHPluqomS5j2tVGj",
+            "hh8vIHA/kXTh6lPH3rmGtHe3PgkARohojPsf62RXk22ycbXeMrvaw/V1cQmDUrmm8ha8qAi48uu6",
+            "oRK3pk81j3LI+Q4SCbP1gllaRnwP7b8xgxbKEzNJp83+a8eM+6zLlLchxjP2MhU41VkzKtomQ+tJ",
+            "lT9mSFLD48b06EKTRUQ9rK5bpv1j+TOBCxfKAB9NSGwxcjlVhokxKKGrJlPb7aPJzqY4lcRzZJyl",
+            "hqz7gM+QvSwOvGbw7m45hbBNGV2VJi00VP6d+O/2p2bg/fniV4rjh5pWgjKh0kLUzp9kqFDnbpZh",
+            "TcBP1YrfR1M2YUP7BBA887AQ5nOD5PRaV58Sz4F+K4pbwG4c5Dld1WNWJTsCAwEAAaNTMFEwHQYD",
+            "VR0OBBYEFFlOCjw+wyUxFg2kpSzLYROZ1WLjMB8GA1UdIwQYMBaAFFlOCjw+wyUxFg2kpSzLYROZ",
+            "1WLjMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAIxs61pAUU0R7j8guc9vOziN",
+            "Cm48KwrrqzSQ7JvgrPdrZ0mv3JN+luAFcA+4itVU5LqO92V5PVuxhbNLOvAx7+qR32zRF863Crpq",
+            "/jAux+3azBsGyGyjqmA+bL1wXGLrn+Tmndwc90iEcRPcOZIDreSF0mRXesBVcXEg1E5gdKg4Hm7O",
+            "dbMZEHITzQ1/uSAQ0WxONsABS0SYgplk20IyWm2nOjtt5zw54mD6TfSgeirQa6Jaoht6n7EfIIMo",
+            "5wWxOXnAXOMdBzN4xf3S6u57fOXDkbHcS0zDYiEFP+FpXB+eabsHfUUKou7YHrzno2vaoAASLpfR",
+            "Xy3WlHqulg0ZaDspY/0mAZxwwUL294yO+/tBsh8y3a9qep/mEl0h0ghj25PeWR+XeNPBth1Y54XD",
+            "KSXGjWXRekn0dLMzazksvJgDN6Ip0hX5StqHbiJX10WpWdSOT06ynh+N6OI+VBj8ndcnLGfSE6gt",
+            "L9JH3IKNtUgodr6Z+CcyZswWKutHyyZE5vteNQFKeTidCQAw9kRW6gtGUVRU0+PrMvD/8WhSd6Wk",
+            "FS4XjN+BXrmruCSGugdL9fgpg21qKcZwkR9rYQXqRPK+nTiVCRrOzUyTFnPmusz8fg7eg6ONaf2x",
+            "MUeWfI+F8kK4NH5GkGggGqQDtes3Y+bWQ28lV7ny44TkMBARz6zH"
+            );
+
+        final String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+            + "<md:EntityDescriptor xmlns:md=\"urn:oasis:names:tc:SAML:2.0:metadata\" entityID=\"https://idp.org\">"
+            + "  <md:IDPSSODescriptor"
+            + "      WantAuthnRequestsSigned=\"true\""
+            + "      protocolSupportEnumeration=\"urn:oasis:names:tc:SAML:2.0:protocol\">"
+            + "    <md:KeyDescriptor use=\"signing\">"
+            + "      <ds:KeyInfo xmlns:ds=\"http://www.w3.org/2000/09/xmldsig#\">"
+            + "        <ds:X509Data>"
+            + "          <ds:X509Certificate>%(signingCertificateOne)</ds:X509Certificate>"
+            + "        </ds:X509Data>"
+            + "      </ds:KeyInfo>"
+            + "    </md:KeyDescriptor>"
+            + "    <md:KeyDescriptor use=\"signing\">"
+            + "      <ds:KeyInfo xmlns:ds=\"http://www.w3.org/2000/09/xmldsig#\">"
+            + "        <ds:X509Data>"
+            + "          <ds:X509Certificate>%(signingCertificateTwo)</ds:X509Certificate>"
+            + "        </ds:X509Data>"
+            + "      </ds:KeyInfo>"
+            + "    </md:KeyDescriptor>"
+            + "    <md:SingleLogoutService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST\""
+            + "        Location=\"https://idp.org/slo/post\"/>"
+            + "    <md:SingleLogoutService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect\""
+            + "        Location=\"https://idp.org/slo/redirect\"/>"
+            + "    <md:NameIDFormat>urn:oasis:names:tc:SAML:2.0:nameid-format:persistent</md:NameIDFormat>"
+            + "    <md:NameIDFormat>urn:oasis:names:tc:SAML:2.0:nameid-format:transient</md:NameIDFormat>"
+            + "    <md:SingleSignOnService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST\""
+            + "        Location=\"https://idp.org/sso/post\"/>"
+            + "    <md:SingleSignOnService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect\""
+            + "        Location=\"https://idp.org/sso/redirect\"/>"
+            + "  </md:IDPSSODescriptor>"
+            + "  <md:Organization>"
+            + "    <md:OrganizationName xml:lang=\"en\">Avengers</md:OrganizationName>"
+            + "    <md:OrganizationDisplayName xml:lang=\"en\">The Avengers</md:OrganizationDisplayName>"
+            + "    <md:OrganizationURL xml:lang=\"en\">https://linktotheidp.org</md:OrganizationURL>"
+            + "  </md:Organization>"
+            + "  <md:ContactPerson contactType=\"technical\">"
+            + "    <md:GivenName>Tony</md:GivenName>"
+            + "    <md:SurName>Stark</md:SurName>"
+            + "    <md:EmailAddress>tony@starkindustries.com</md:EmailAddress>"
+            + "  </md:ContactPerson>"
+            + "</md:EntityDescriptor>";
+
+        final Map<String, Object> replacements = new HashMap<>();
+        replacements.put("signingCertificateOne", signingCertificateOne);
+        replacements.put("signingCertificateTwo", signingCertificateTwo);
+
+        final String expectedXmlWithCertificate = NamedFormatter.format(expectedXml, replacements);
+
+        assertThat(xml, Matchers.equalTo(normaliseXml(expectedXmlWithCertificate)));
+
+        assertValidXml(xml);
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGeneratorTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGeneratorTests.java
new file mode 100644
index 0000000000000..2cc1a78cb51d3
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlMetadataGeneratorTests.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.idp;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.util.NamedFormatter;
+import org.elasticsearch.xpack.idp.action.SamlMetadataResponse;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.Matchers;
+import org.opensaml.saml.saml2.metadata.EntityDescriptor;
+import org.opensaml.saml.security.impl.SAMLSignatureProfileValidator;
+
+import org.opensaml.xmlsec.signature.support.SignatureException;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.xmlsec.signature.Signature;
+import org.opensaml.xmlsec.signature.support.SignatureValidator;
+import org.w3c.dom.Element;
+
+import java.net.URL;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI;
+import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI;
+
+public class SamlMetadataGeneratorTests extends IdpSamlTestCase {
+
+    public void testGenerateMetadata() throws Exception {
+        SamlServiceProvider sp = mock(SamlServiceProvider.class);
+        when(sp.getEntityId()).thenReturn("https://sp.org");
+        when(sp.shouldSignAuthnRequests()).thenReturn(true);
+        SamlIdentityProvider idp = mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn("https://idp.org");
+        when(idp.getMetadataSigningCredential()).thenReturn(null);
+        when(idp.getSigningCredential()).thenReturn(readCredentials("RSA", 2048));
+        when(idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI)).thenReturn(new URL("https://idp.org/sso/redirect"));
+        when(idp.getSingleLogoutEndpoint(SAML2_POST_BINDING_URI)).thenReturn(new URL("https://idp.org/slo/post"));
+        mockRegisteredServiceProvider(idp, "https://sp.org", sp);
+
+        SamlFactory factory = new SamlFactory();
+        SamlMetadataGenerator generator = new SamlMetadataGenerator(factory, idp);
+        PlainActionFuture<SamlMetadataResponse> future = new PlainActionFuture<>();
+        generator.generateMetadata("https://sp.org", future);
+        SamlMetadataResponse response = future.actionGet();
+        final String xml = response.getXmlString();
+
+        // RSA_2048
+        final String signingCertificate = joinCertificateLines(
+            "MIIDYzCCAkugAwIBAgIVAITQVqXYYUT0w04Z2gWAZ6pv7gwbMA0GCSqGSIb3DQEBCwUAMDQxMjAw",
+            "BgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMCAXDTIwMDEy",
+            "ODA2MzczNloYDzIxNjgxMDE5MDYzNzM2WjBRMRMwEQYKCZImiZPyLGQBGRYDb3JnMR0wGwYKCZIm",
+            "iZPyLGQBGRYNZWxhc3RpY3NlYXJjaDEMMAoGA1UECxMDaWRwMQ0wCwYDVQQDEwR0ZXN0MIIBIjAN",
+            "BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAujk+mVzI+qmf4gSJZdVVDdhFTi06kikb7FxG5JPu",
+            "+gmU9Ke0LVEpP7Jp3gmhwsa18JUuvaepL1jnKmbbepKkEsvqUj4FuI/gImvFwb7X+xUwzNTYZAEv",
+            "nZ4n16k0sBPuDuDibF0MGniVeLG3bD2VF3crFQrphFr+GZSXbVk5zIcSf6D6nSDcKmCNpVAK3jX9",
+            "iV0nkr8cPtHOgprv1Y7mZgk5jwli9to0QD7r7OG5Db34R06JTMGTji+RULPISH1bc8FHdurRASkG",
+            "msei5GlJqPSuKdViuaKPmDrvKR8OK6gMzUd3pikJgD8veLxEuZ640FHPndPlvwJrSLwhitRgPQID",
+            "AQABo00wSzAdBgNVHQ4EFgQUD87H31WVQNfCc85/H2qhpzs3XfowHwYDVR0jBBgwFoAUVenAN+T0",
+            "6rqNDxjMcvgimnTw+FgwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAIvHYxT30cvoHWUE2",
+            "saDVJ4qs/e0G3WusDyem3e4HkqwLEah06RDSgVCaOfW3ey5Q6CIQW3HHGUYqO0nU8JVCWdAk3+bU",
+            "YJJOeLnwD+SbDxxKBhxLdx+BjWata85lfJTR9+dXs0RXAAN8dSiIaj9NSgnwiJqQQZf7i66S7XB5",
+            "8TDTdZlV3d26STLy5h7Uy6vyCka8Xu8HFQ4hH2qf2L6EhBbzVTB6tuyPQOQwrlLE65nhUNkfBbjZ",
+            "lre45UMc9GuxzHkbvd3HEQaroMHZxnu+/n/JDlgsrCYUEXnZnOXvgUPupPynoRdDN1F6r95TLyU9",
+            "pYjDf/6zNPE854VF6y1TqQ=="
+        );
+
+        final String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+            + "<md:EntityDescriptor xmlns:md=\"urn:oasis:names:tc:SAML:2.0:metadata\" entityID=\"https://idp.org\">"
+            + "  <md:IDPSSODescriptor"
+            + "      WantAuthnRequestsSigned=\"true\""
+            + "      protocolSupportEnumeration=\"urn:oasis:names:tc:SAML:2.0:protocol\">"
+            + "    <md:KeyDescriptor use=\"signing\">"
+            + "      <ds:KeyInfo xmlns:ds=\"http://www.w3.org/2000/09/xmldsig#\">"
+            + "        <ds:X509Data>"
+            + "          <ds:X509Certificate>%(signingCertificate)</ds:X509Certificate>"
+            + "        </ds:X509Data>"
+            + "      </ds:KeyInfo>"
+            + "    </md:KeyDescriptor>"
+            + "    <md:SingleLogoutService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST\""
+            + "        Location=\"https://idp.org/slo/post\"/>"
+            + "    <md:NameIDFormat>urn:oasis:names:tc:SAML:2.0:nameid-format:persistent</md:NameIDFormat>"
+            + "    <md:NameIDFormat>urn:oasis:names:tc:SAML:2.0:nameid-format:transient</md:NameIDFormat>"
+            + "    <md:SingleSignOnService"
+            + "        Binding=\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect\""
+            + "        Location=\"https://idp.org/sso/redirect\"/>"
+            + "  </md:IDPSSODescriptor>"
+            + "</md:EntityDescriptor>";
+
+        final Map<String, Object> replacements = Map.of("signingCertificate", signingCertificate);
+
+        final String expectedXmlWithCertificate = NamedFormatter.format(expectedXml, replacements);
+        assertThat(xml, Matchers.equalTo(normaliseXml(expectedXmlWithCertificate)));
+        assertValidXml(xml);
+    }
+
+    public void testGenerateAndSignMetadata() throws Exception {
+        SamlServiceProvider sp = mock(SamlServiceProvider.class);
+        when(sp.getEntityId()).thenReturn("https://sp.org");
+        when(sp.shouldSignAuthnRequests()).thenReturn(true);
+        SamlIdentityProvider idp = mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn("https://idp.org");
+        when(idp.getMetadataSigningCredential()).thenReturn(readCredentials("RSA", 4096));
+        when(idp.getSigningCredential()).thenReturn(readCredentials("RSA", 2048));
+        when(idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI)).thenReturn(new URL("https://idp.org/sso/redirect"));
+        when(idp.getSingleLogoutEndpoint(SAML2_POST_BINDING_URI)).thenReturn(new URL("https://idp.org/slo/post"));
+        mockRegisteredServiceProvider(idp, "https://sp.org", sp);
+
+        X509Credential signingCredential = readCredentials("RSA", 4096);
+        SamlFactory factory = new SamlFactory();
+        SamlMetadataGenerator generator = new SamlMetadataGenerator(factory, idp);
+        Element element = generator.possiblySignDescriptor(generator.buildEntityDescriptor(sp), signingCredential);
+        EntityDescriptor descriptor = factory.buildXmlObject(element, EntityDescriptor.class);
+        Signature signature = descriptor.getSignature();
+        assertNotNull(signature);
+        SAMLSignatureProfileValidator profileValidator = new SAMLSignatureProfileValidator();
+        profileValidator.validate(signature);
+        SignatureValidator.validate(signature, signingCredential);
+        //no exception thrown
+        SignatureException e = expectThrows(SignatureException.class,
+            () -> SignatureValidator.validate(signature, readCredentials("RSA", 2048)));
+        assertThat(e.getMessage(), containsString("Unable to evaluate key against signature"));
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandlerTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandlerTests.java
new file mode 100644
index 0000000000000..e820406e62bf9
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/rest/action/IdpBaseRestHandlerTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.rest.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.TestUtils;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class IdpBaseRestHandlerTests extends ESTestCase {
+
+    public void testIdpAvailableOnTrialOrEnterprise() {
+        final IdpBaseRestHandler handler = buildHandler(randomFrom(License.OperationMode.ENTERPRISE, License.OperationMode.TRIAL));
+        assertThat(handler.isIdpFeatureAllowed(), equalTo(true));
+    }
+
+    public void testIdpNotAvailableOnOtherLicenses() {
+        License.OperationMode mode =
+            randomValueOtherThanMany(m -> m == License.OperationMode.ENTERPRISE || m == License.OperationMode.TRIAL,
+                () -> randomFrom(License.OperationMode.values()));
+        final IdpBaseRestHandler handler = buildHandler(mode);
+        assertThat(handler.isIdpFeatureAllowed(), equalTo(false));
+    }
+
+    private IdpBaseRestHandler buildHandler(License.OperationMode licenseMode) {
+        final Settings settings = Settings.builder()
+            .put("xpack.idp.enabled", true)
+            .build();
+        final TestUtils.UpdatableLicenseState licenseState = new TestUtils.UpdatableLicenseState(settings);
+        licenseState.update(licenseMode, true, null);
+        return new IdpBaseRestHandler(licenseState) {
+            @Override
+            protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+                return null;
+            }
+
+            @Override
+            public String getName() {
+                return "idp-rest-test";
+            }
+
+            @Override
+            public List<Route> routes() {
+                return List.of();
+            }
+        };
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java
new file mode 100644
index 0000000000000..1fb78fbe692b2
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java
@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.opensaml.saml.saml2.core.NameID;
+import org.opensaml.security.x509.X509Credential;
+
+import java.io.IOException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class SamlServiceProviderDocumentTests extends IdpSamlTestCase {
+
+    public void testValidationFailuresForMissingFields() throws Exception {
+        final SamlServiceProviderDocument doc = new SamlServiceProviderDocument();
+        doc.setDocId(randomAlphaOfLength(16));
+        final ValidationException validationException = doc.validate();
+        assertThat(validationException, notNullValue());
+        assertThat(validationException.validationErrors(), not(emptyIterable()));
+        assertThat(validationException.validationErrors(), Matchers.containsInAnyOrder(
+            "field [name] is required, but was [null]",
+            "field [entity_id] is required, but was [null]",
+            "field [acs] is required, but was [null]",
+            "field [created] is required, but was [null]",
+            "field [last_modified] is required, but was [null]",
+            "field [privileges.resource] is required, but was [null]",
+            "field [attributes.principal] is required, but was [null]"
+        ));
+    }
+
+    public void testValidationSucceedsWithMinimalFields() throws Exception {
+        final SamlServiceProviderDocument doc = createMinimalDocument();
+        final ValidationException validationException = doc.validate();
+        assertThat(validationException, nullValue());
+    }
+
+    public void testXContentRoundTripWithMinimalFields() throws Exception {
+        final SamlServiceProviderDocument doc1 = createMinimalDocument();
+        final SamlServiceProviderDocument doc2 = assertXContentRoundTrip(doc1);
+        assertThat(assertXContentRoundTrip(doc2), equalTo(doc1));
+    }
+
+    public void testXContentRoundTripWithAllFields() throws Exception {
+        final SamlServiceProviderDocument doc1 = createFullDocument();
+        final SamlServiceProviderDocument doc2 = assertXContentRoundTrip(doc1);
+        assertThat(assertXContentRoundTrip(doc2), equalTo(doc1));
+    }
+
+    public void testStreamRoundTripWithMinimalFields() throws Exception {
+        final SamlServiceProviderDocument doc1 = createMinimalDocument();
+        final SamlServiceProviderDocument doc2 = assertXContentRoundTrip(doc1);
+        assertThat(assertSerializationRoundTrip(doc2), equalTo(doc1));
+    }
+
+    public void testStreamRoundTripWithAllFields() throws Exception {
+        final SamlServiceProviderDocument doc1 = createFullDocument();
+        final SamlServiceProviderDocument doc2 = assertXContentRoundTrip(doc1);
+        assertThat(assertSerializationRoundTrip(doc2), equalTo(doc1));
+    }
+
+    private SamlServiceProviderDocument createFullDocument() throws CertificateException, IOException {
+        final List<X509Credential> credentials = readCredentials();
+        final List<X509Certificate> certificates = credentials.stream()
+            .map(X509Credential::getEntityCertificate)
+            .collect(Collectors.toUnmodifiableList());
+        final List<X509Certificate> spCertificates = randomSubsetOf(certificates);
+        final List<X509Certificate> idpCertificates = randomSubsetOf(certificates);
+        final List<X509Certificate> idpMetadataCertificates = randomSubsetOf(certificates);
+
+        final SamlServiceProviderDocument doc1 = new SamlServiceProviderDocument();
+        doc1.setDocId(randomAlphaOfLength(16));
+        doc1.setName(randomAlphaOfLengthBetween(8, 12));
+        doc1.setEntityId("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        doc1.setAcs("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs");
+        doc1.setCreatedMillis(System.currentTimeMillis() - randomIntBetween(100_000, 1_000_000));
+        doc1.setLastModifiedMillis(System.currentTimeMillis() - randomIntBetween(1_000, 100_000));
+        doc1.setNameIdFormat(randomFrom(NameID.TRANSIENT, NameID.PERSISTENT, NameID.EMAIL));
+        doc1.setAuthenticationExpiryMillis(randomLongBetween(100, 5_000_000));
+        doc1.certificates.setServiceProviderX509SigningCertificates(spCertificates);
+        doc1.certificates.setIdentityProviderX509SigningCertificates(idpCertificates);
+        doc1.certificates.setIdentityProviderX509MetadataSigningCertificates(idpMetadataCertificates);
+
+        doc1.privileges.setResource("service:" + randomAlphaOfLength(12) + ":" + randomAlphaOfLength(12));
+        final Map<String, String> roleActions = new HashMap<>();
+        for (int i = randomIntBetween(1, 6); i > 0; i--) {
+            roleActions.put(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLength(6) + ":" + randomAlphaOfLength(6));
+        }
+        doc1.privileges.setRoleActions(roleActions);
+
+        doc1.attributeNames.setPrincipal("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        doc1.attributeNames.setEmail("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        doc1.attributeNames.setName("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        doc1.attributeNames.setRoles("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        return doc1;
+    }
+
+    private SamlServiceProviderDocument createMinimalDocument() {
+        final SamlServiceProviderDocument doc1 = new SamlServiceProviderDocument();
+        doc1.setDocId(randomAlphaOfLength(16));
+        doc1.setName(randomAlphaOfLengthBetween(8, 12));
+        doc1.setEntityId("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        doc1.setAcs("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs");
+        doc1.setCreatedMillis(System.currentTimeMillis() - randomIntBetween(100_000, 1_000_000));
+        doc1.setLastModifiedMillis(System.currentTimeMillis() - randomIntBetween(1_000, 100_000));
+        doc1.privileges.setResource("service:" + randomAlphaOfLength(12) + ":" + randomAlphaOfLength(12));
+        doc1.attributeNames.setPrincipal("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        return doc1;
+    }
+
+    private SamlServiceProviderDocument assertXContentRoundTrip(SamlServiceProviderDocument obj1) throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final boolean humanReadable = randomBoolean();
+        final BytesReference bytes1 = XContentHelper.toXContent(obj1, xContentType, humanReadable);
+        try (XContentParser parser = XContentHelper.createParser(
+            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes1, xContentType)) {
+            final SamlServiceProviderDocument obj2 = SamlServiceProviderDocument.fromXContent(obj1.docId, parser);
+            assertThat(obj2, equalTo(obj1));
+
+            final BytesReference bytes2 = XContentHelper.toXContent(obj2, xContentType, humanReadable);
+            assertThat(bytes2, equalTo(bytes1));
+
+            return obj2;
+        }
+    }
+
+    private SamlServiceProviderDocument assertSerializationRoundTrip(SamlServiceProviderDocument doc) throws IOException {
+        final Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_7_0, Version.CURRENT);
+        final SamlServiceProviderDocument read = copyWriteable(doc, new NamedWriteableRegistry(List.of()),
+            SamlServiceProviderDocument::new, version);
+        MatcherAssert.assertThat("Serialized document with version [" + version + "] does not match original object", read, equalTo(doc));
+        return read;
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndexTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndexTests.java
new file mode 100644
index 0000000000000..8d3401707349b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderIndexTests.java
@@ -0,0 +1,303 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.idp.IdentityProviderPlugin;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ENTITY_ID;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SSO_REDIRECT_ENDPOINT;
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.iterableWithSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public class SamlServiceProviderIndexTests extends ESSingleNodeTestCase {
+
+    private ClusterService clusterService;
+    private SamlServiceProviderIndex serviceProviderIndex;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return List.of(LocalStateCompositeXPackPlugin.class, IdentityProviderPlugin.class);
+    }
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder()
+            .put(IDP_ENTITY_ID.getKey(), "urn:idp:org")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/init")
+            .put(super.nodeSettings())
+            .build();
+    }
+
+    @Before
+    public void setupComponents() throws Exception {
+        clusterService = super.getInstanceFromNode(ClusterService.class);
+        serviceProviderIndex = new SamlServiceProviderIndex(client(), clusterService);
+    }
+
+    @After
+    public void deleteTemplateAndIndex() {
+        client().admin().indices().delete(new DeleteIndexRequest(SamlServiceProviderIndex.INDEX_NAME + "*")).actionGet();
+        client().admin().indices().deleteTemplate(new DeleteIndexTemplateRequest(SamlServiceProviderIndex.TEMPLATE_NAME)).actionGet();
+        serviceProviderIndex.close();
+    }
+
+    public void testWriteAndFindServiceProvidersFromIndex() {
+        final int count = randomIntBetween(3, 5);
+        List<SamlServiceProviderDocument> documents = new ArrayList<>(count);
+
+        final ClusterService clusterService = super.getInstanceFromNode(ClusterService.class);
+        // Install the template
+        assertTrue("Template should have been installed", installTemplate());
+        // No need to install it again
+        assertFalse("Template should not have been installed a second time", installTemplate());
+
+        // Index should not exist yet
+        assertThat(clusterService.state().metaData().index(SamlServiceProviderIndex.INDEX_NAME), nullValue());
+
+        for (int i = 0; i < count; i++) {
+            final SamlServiceProviderDocument doc = randomDocument(i);
+            writeDocument(doc);
+            documents.add(doc);
+        }
+
+        final IndexMetaData indexMetaData = clusterService.state().metaData().index(SamlServiceProviderIndex.INDEX_NAME);
+        assertThat(indexMetaData, notNullValue());
+        assertThat(indexMetaData.getSettings().get("index.format"), equalTo("1"));
+        assertThat(indexMetaData.getAliases().size(), equalTo(1));
+        assertThat(indexMetaData.getAliases().keys().toArray(), arrayContainingInAnyOrder(SamlServiceProviderIndex.ALIAS_NAME));
+
+        refresh();
+
+        final Set<SamlServiceProviderDocument> allDocs = getAllDocs();
+        assertThat(allDocs, iterableWithSize(count));
+        for (SamlServiceProviderDocument doc : documents) {
+            assertThat(allDocs, hasItem(Matchers.equalTo(doc)));
+        }
+
+        final SamlServiceProviderDocument readDoc = randomFrom(documents);
+        assertThat(readDocument(readDoc.docId), equalTo(readDoc));
+
+        final SamlServiceProviderDocument findDoc = randomFrom(documents);
+        assertThat(findByEntityId(findDoc.entityId), equalTo(findDoc));
+
+        final SamlServiceProviderDocument deleteDoc = randomFrom(documents);
+        final DeleteResponse deleteResponse = deleteDocument(deleteDoc);
+        assertThat(deleteResponse.getId(), equalTo(deleteDoc.docId));
+        assertThat(deleteResponse.getResult(), equalTo(DocWriteResponse.Result.DELETED));
+
+        refresh();
+
+        assertThat(readDocument(deleteDoc.docId), nullValue());
+        assertThat(findAllByEntityId(deleteDoc.entityId), emptyIterable());
+    }
+
+    public void testWritesViaAliasIfItExists() {
+        final PlainActionFuture<Boolean> installTemplate = new PlainActionFuture<>();
+        serviceProviderIndex.installIndexTemplate(installTemplate);
+        assertTrue(installTemplate.actionGet());
+
+        // Create an index that will trigger the template, but isn't the standard index name
+        final String customIndexName = SamlServiceProviderIndex.INDEX_NAME + "-test";
+        client().admin().indices().create(new CreateIndexRequest(customIndexName)).actionGet();
+
+        final IndexMetaData indexMetaData = clusterService.state().metaData().index(customIndexName);
+        assertThat(indexMetaData, notNullValue());
+        assertThat(indexMetaData.getSettings().get("index.format"), equalTo("1"));
+        assertThat(indexMetaData.getAliases().size(), equalTo(1));
+        assertThat(indexMetaData.getAliases().keys().toArray(), arrayContainingInAnyOrder(SamlServiceProviderIndex.ALIAS_NAME));
+
+        SamlServiceProviderDocument document = randomDocument(1);
+        writeDocument(document);
+
+        // Index should not exist because we created an alternate index, and the alias points to that.
+        assertThat(clusterService.state().metaData().index(SamlServiceProviderIndex.INDEX_NAME), nullValue());
+
+        refresh();
+
+        final Set<SamlServiceProviderDocument> allDocs = getAllDocs();
+        assertThat(allDocs, iterableWithSize(1));
+        assertThat(allDocs, hasItem(Matchers.equalTo(document)));
+
+        assertThat(readDocument(document.docId), equalTo(document));
+    }
+
+    public void testInstallTemplateAutomaticallyOnClusterChange() throws Exception {
+        // Create an index that will trigger a cluster state change
+        final String indexName = randomAlphaOfLength(7).toLowerCase(Locale.ROOT);
+        client().admin().indices().create(new CreateIndexRequest(indexName)).actionGet();
+
+        ensureGreen(indexName);
+
+        IndexTemplateMetaData templateMeta = clusterService.state().metaData().templates().get(SamlServiceProviderIndex.TEMPLATE_NAME);
+
+        assertBusy(() -> assertThat("template should have been installed", templateMeta, notNullValue()));
+
+        final PlainActionFuture<Boolean> installTemplate = new PlainActionFuture<>();
+        serviceProviderIndex.installIndexTemplate(installTemplate);
+        assertFalse("Template is already installed, should not install again", installTemplate.actionGet());
+    }
+
+    public void testInstallTemplateAutomaticallyOnDocumentWrite() {
+        final SamlServiceProviderDocument doc = randomDocument(1);
+        writeDocument(doc);
+
+        assertThat(readDocument(doc.docId), equalTo(doc));
+
+        IndexTemplateMetaData templateMeta = clusterService.state().metaData().templates().get(SamlServiceProviderIndex.TEMPLATE_NAME);
+        assertThat("template should have been installed", templateMeta, notNullValue());
+
+        final PlainActionFuture<Boolean> installTemplate = new PlainActionFuture<>();
+        serviceProviderIndex.installIndexTemplate(installTemplate);
+        assertFalse("Template is already installed, should not install again", installTemplate.actionGet());
+    }
+
+    private boolean installTemplate() {
+        final PlainActionFuture<Boolean> installTemplate = new PlainActionFuture<>();
+        serviceProviderIndex.installIndexTemplate(installTemplate);
+        return installTemplate.actionGet();
+    }
+
+    private Set<SamlServiceProviderDocument> getAllDocs() {
+        final PlainActionFuture<Set<SamlServiceProviderDocument>> future = new PlainActionFuture<>();
+        serviceProviderIndex.findAll(ActionListener.wrap(
+            set -> future.onResponse(set.stream().map(doc -> doc.document.get()).collect(Collectors.toUnmodifiableSet())),
+            future::onFailure
+        ));
+        return future.actionGet();
+    }
+
+    private SamlServiceProviderDocument readDocument(String docId) {
+        final PlainActionFuture<SamlServiceProviderIndex.DocumentSupplier> future = new PlainActionFuture<>();
+        serviceProviderIndex.readDocument(docId, future);
+        final SamlServiceProviderIndex.DocumentSupplier supplier = future.actionGet();
+        return supplier == null ? null : supplier.getDocument();
+    }
+
+    private void writeDocument(SamlServiceProviderDocument doc) {
+        final PlainActionFuture<DocWriteResponse> future = new PlainActionFuture<>();
+        serviceProviderIndex.writeDocument(doc, DocWriteRequest.OpType.INDEX, WriteRequest.RefreshPolicy.WAIT_UNTIL, future);
+        doc.setDocId(future.actionGet().getId());
+    }
+
+    private DeleteResponse deleteDocument(SamlServiceProviderDocument doc) {
+        final PlainActionFuture<DeleteResponse> future = new PlainActionFuture<>();
+        serviceProviderIndex.readDocument(doc.docId, ActionListener.wrap(
+            info -> serviceProviderIndex.deleteDocument(info.version, WriteRequest.RefreshPolicy.IMMEDIATE, future),
+            future::onFailure));
+        return future.actionGet();
+    }
+
+    private SamlServiceProviderDocument findByEntityId(String entityId) {
+        final Set<SamlServiceProviderDocument> docs = findAllByEntityId(entityId);
+        assertThat(docs, iterableWithSize(1));
+        return docs.iterator().next();
+    }
+
+    private Set<SamlServiceProviderDocument> findAllByEntityId(String entityId) {
+        final PlainActionFuture<Set<SamlServiceProviderDocument>> future = new PlainActionFuture<>();
+        serviceProviderIndex.findByEntityId(entityId, ActionListener.wrap(
+            set -> future.onResponse(set.stream().map(doc -> doc.document.get()).collect(Collectors.toUnmodifiableSet())),
+            future::onFailure
+        ));
+        return future.actionGet();
+    }
+
+    private void refresh() {
+        PlainActionFuture<Void> future = new PlainActionFuture<>();
+        serviceProviderIndex.refresh(future);
+        future.actionGet();
+    }
+
+    public static SamlServiceProviderDocument randomDocument() {
+        return randomDocument(randomIntBetween(1, 999_999));
+    }
+
+    public static SamlServiceProviderDocument randomDocument(int index) {
+        final SamlServiceProviderDocument document = new SamlServiceProviderDocument();
+        document.setName(randomAlphaOfLengthBetween(5, 12));
+        document.setEntityId(randomUri() + index);
+        document.setAcs(randomUri("https") + index + "/saml/acs");
+
+        document.setEnabled(randomBoolean());
+        document.setCreatedMillis(System.currentTimeMillis() - TimeValue.timeValueDays(randomIntBetween(2, 90)).millis());
+        document.setLastModifiedMillis(System.currentTimeMillis() - TimeValue.timeValueHours(randomIntBetween(1, 36)).millis());
+
+        if (randomBoolean()) {
+            document.setNameIdFormat(TRANSIENT);
+        }
+        if (randomBoolean()) {
+            document.setAuthenticationExpiryMillis(TimeValue.timeValueMinutes(randomIntBetween(1, 15)).millis());
+        }
+
+        document.privileges.setResource("app:" + randomAlphaOfLengthBetween(3, 6) + ":" + Math.abs(randomLong()));
+        final int roleCount = randomIntBetween(0, 4);
+        final Map<String, String> roles = new HashMap<>();
+        for (int i = 0; i < roleCount; i++) {
+            roles.put(randomAlphaOfLengthBetween(4, 8), randomAlphaOfLengthBetween(3, 6) + ":" + randomAlphaOfLengthBetween(3, 6));
+        }
+        document.privileges.setRoleActions(roles);
+
+        document.attributeNames.setPrincipal(randomUri());
+        if (randomBoolean()) {
+            document.attributeNames.setName(randomUri());
+        }
+        if (randomBoolean()) {
+            document.attributeNames.setEmail(randomUri());
+        }
+        if (roles.isEmpty() == false) {
+            document.attributeNames.setRoles(randomUri());
+        }
+
+        assertThat(document.validate(), nullValue());
+        return document;
+    }
+
+    private static String randomUri() {
+        return randomUri(randomFrom("urn", "http", "https"));
+    }
+
+    private static String randomUri(String scheme) {
+        return scheme + "://" + randomAlphaOfLengthBetween(2, 6) + "."
+            + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(2, 4) + "/";
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolverTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolverTests.java
new file mode 100644
index 0000000000000..f637e9b517f85
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderResolverTests.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.sp;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderIndex.DocumentVersion;
+import org.hamcrest.Matchers;
+import org.joda.time.Duration;
+import org.junit.Before;
+import org.opensaml.saml.saml2.core.NameID;
+
+import java.net.URL;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SamlServiceProviderResolverTests extends ESTestCase {
+
+    private SamlServiceProviderIndex index;
+    private ServiceProviderDefaults serviceProviderDefaults;
+    private SamlIdentityProvider identityProvider;
+    private SamlServiceProviderResolver resolver;
+
+    @Before
+    public void setupMocks() {
+        index = mock(SamlServiceProviderIndex.class);
+        identityProvider = mock(SamlIdentityProvider.class);
+        serviceProviderDefaults = configureIdentityProviderDefaults();
+        resolver = new SamlServiceProviderResolver(Settings.EMPTY, index, serviceProviderDefaults);
+    }
+
+    public void testResolveWithoutCache() throws Exception {
+
+        final String entityId = "https://" + randomAlphaOfLength(12) + ".elastic-cloud.com/";
+        final URL acs = new URL(entityId + "saml/acs");
+
+        final String principalAttribute = randomAlphaOfLengthBetween(6, 36);
+        final String rolesAttribute = randomAlphaOfLengthBetween(6, 36);
+        final String resource = "ece:" + randomAlphaOfLengthBetween(6, 12);
+        final Map<String, String> rolePrivileges = Map.of(randomAlphaOfLengthBetween(3, 6), "role:" + randomAlphaOfLengthBetween(4, 8));
+
+        final DocumentVersion docVersion = new DocumentVersion(
+            randomAlphaOfLength(12), randomNonNegativeLong(), randomNonNegativeLong());
+        final SamlServiceProviderDocument document = new SamlServiceProviderDocument();
+        document.setEntityId(entityId);
+        document.setAuthenticationExpiry(null);
+        document.setAcs(acs.toString());
+        document.privileges.setResource(resource);
+        document.privileges.setRoleActions(rolePrivileges);
+        document.attributeNames.setPrincipal(principalAttribute);
+        document.attributeNames.setRoles(rolesAttribute);
+
+        mockDocument(entityId, docVersion, document);
+
+        final SamlServiceProvider serviceProvider = resolveServiceProvider(entityId);
+        assertThat(serviceProvider.getEntityId(), equalTo(entityId));
+        assertThat(serviceProvider.getAssertionConsumerService(), equalTo(acs));
+        assertThat(serviceProvider.getAllowedNameIdFormat(), equalTo(serviceProviderDefaults.nameIdFormat));
+        assertThat(serviceProvider.getAuthnExpiry(), equalTo(serviceProviderDefaults.authenticationExpiry));
+        assertThat(serviceProvider.getSpSigningCredentials(), emptyIterable());
+        assertThat(serviceProvider.shouldSignAuthnRequests(), equalTo(false));
+        assertThat(serviceProvider.shouldSignLogoutRequests(), equalTo(false));
+
+        assertThat(serviceProvider.getAttributeNames(), notNullValue());
+        assertThat(serviceProvider.getAttributeNames().principal, equalTo(principalAttribute));
+        assertThat(serviceProvider.getAttributeNames().name, nullValue());
+        assertThat(serviceProvider.getAttributeNames().email, nullValue());
+        assertThat(serviceProvider.getAttributeNames().roles, equalTo(rolesAttribute));
+
+        assertThat(serviceProvider.getPrivileges(), notNullValue());
+        assertThat(serviceProvider.getPrivileges().getApplicationName(), equalTo(serviceProviderDefaults.applicationName));
+        assertThat(serviceProvider.getPrivileges().getResource(), equalTo(resource));
+        assertThat(serviceProvider.getPrivileges().getRoleActions(), equalTo(rolePrivileges));
+    }
+
+    public void testResolveReturnsCachedObject() throws Exception {
+        final SamlServiceProviderDocument document1 = SamlServiceProviderIndexTests.randomDocument(1);
+        final SamlServiceProviderDocument document2 = SamlServiceProviderIndexTests.randomDocument(2);
+        document2.entityId = document1.entityId;
+
+        final DocumentVersion docVersion = new DocumentVersion(randomAlphaOfLength(12), 1, 1);
+
+        mockDocument(document1.entityId, docVersion, document1);
+        final SamlServiceProvider serviceProvider1 = resolveServiceProvider(document1.entityId);
+
+        mockDocument(document1.entityId, docVersion, document2);
+        final SamlServiceProvider serviceProvider2 = resolveServiceProvider(document1.entityId);
+
+        assertThat(serviceProvider2, sameInstance(serviceProvider1));
+    }
+
+    public void testResolveIgnoresCacheWhenDocumentVersionChanges() throws Exception {
+        final SamlServiceProviderDocument document1 = SamlServiceProviderIndexTests.randomDocument(1);
+        final SamlServiceProviderDocument document2 = SamlServiceProviderIndexTests.randomDocument(2);
+        document2.entityId = document1.entityId;
+
+        final DocumentVersion docVersion1 = new DocumentVersion(randomAlphaOfLength(12), 1, 1);
+        final DocumentVersion docVersion2 = new DocumentVersion(randomAlphaOfLength(12), randomIntBetween(2, 10), randomIntBetween(1, 10));
+
+        mockDocument(document1.entityId, docVersion1, document1);
+        final SamlServiceProvider serviceProvider1a = resolveServiceProvider(document1.entityId);
+        final SamlServiceProvider serviceProvider1b = resolveServiceProvider(document1.entityId);
+        assertThat(serviceProvider1b, sameInstance(serviceProvider1a));
+
+        mockDocument(document1.entityId, docVersion2, document2);
+        final SamlServiceProvider serviceProvider2 = resolveServiceProvider(document1.entityId);
+
+        assertThat(serviceProvider2, not(sameInstance(serviceProvider1a)));
+        assertThat(serviceProvider2.getEntityId(), equalTo(document2.entityId));
+        assertThat(serviceProvider2.getAssertionConsumerService().toString(), equalTo(document2.acs));
+        assertThat(serviceProvider2.getAttributeNames().principal, equalTo(document2.attributeNames.principal));
+        assertThat(serviceProvider2.getAttributeNames().name, equalTo(document2.attributeNames.name));
+        assertThat(serviceProvider2.getAttributeNames().email, equalTo(document2.attributeNames.email));
+        assertThat(serviceProvider2.getAttributeNames().roles, equalTo(document2.attributeNames.roles));
+        assertThat(serviceProvider2.getPrivileges().getResource(), equalTo(document2.privileges.resource));
+    }
+
+    private SamlServiceProvider resolveServiceProvider(String entityId) {
+        final PlainActionFuture<SamlServiceProvider> future = new PlainActionFuture<>();
+        resolver.resolve(entityId, future);
+
+        final SamlServiceProvider serviceProvider = future.actionGet();
+        assertThat(serviceProvider, notNullValue());
+        return serviceProvider;
+    }
+
+    private ServiceProviderDefaults configureIdentityProviderDefaults() {
+        final String defaultNameId = NameID.TRANSIENT;
+        final String defaultApplication = randomAlphaOfLengthBetween(4, 12);
+        final Duration defaultExpiry = Duration.standardMinutes(12);
+        final ServiceProviderDefaults defaults = new ServiceProviderDefaults(
+            defaultApplication, defaultNameId, defaultExpiry);
+        when(identityProvider.getServiceProviderDefaults()).thenReturn(defaults);
+        return defaults;
+    }
+
+    private void mockDocument(String entityId, DocumentVersion docVersion, SamlServiceProviderDocument document) {
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, Matchers.arrayWithSize(2));
+
+            assertThat(args[0], equalTo(entityId));
+
+            ActionListener<Set<SamlServiceProviderIndex.DocumentSupplier>> listener
+                = (ActionListener<Set<SamlServiceProviderIndex.DocumentSupplier>>) args[args.length - 1];
+            listener.onResponse(Set.of(new SamlServiceProviderIndex.DocumentSupplier(docVersion, () -> document)));
+            return null;
+        }).when(index).findByEntityId(anyString(), any(ActionListener.class));
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java
new file mode 100644
index 0000000000000..a5d92a8b82b40
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.opensaml.saml.saml2.core.NameID;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class SamlAuthenticationStateTests extends IdpSamlTestCase {
+
+    public void testValidation() {
+        final SamlAuthenticationState state = new SamlAuthenticationState();
+        final ValidationException validationException = state.validate();
+        assertThat(validationException, notNullValue());
+        assertThat(validationException.validationErrors(), not(emptyIterable()));
+        assertThat(validationException.validationErrors(), Matchers.containsInAnyOrder(
+            "field [entity_id] is required, but was [null]",
+            "field [acs_url] is required, but was [null]"
+        ));
+    }
+
+    public void testXContentRoundTrip() throws Exception {
+        final SamlAuthenticationState state1 = generateAuthnState();
+        final SamlAuthenticationState state2 = assertXContentRoundTrip(state1);
+        assertThat(assertXContentRoundTrip(state2), equalTo(state1));
+    }
+
+    public void testNullStateXContentRoundTrip() throws Exception {
+        final SamlAuthenticationState state1 = generateMinimalAuthnState();
+        final SamlAuthenticationState state2 = assertXContentRoundTrip(state1);
+        assertThat(assertXContentRoundTrip(state2), equalTo(state1));
+    }
+
+    public void testStreamRoundTrip() throws Exception {
+        final SamlAuthenticationState state1 = generateAuthnState();
+        final SamlAuthenticationState state2 = assertSerializationRoundTrip(state1);
+        assertThat(assertSerializationRoundTrip(state2), equalTo(state1));
+    }
+
+    public void testNullStateStreamRoundTrip() throws Exception {
+        final SamlAuthenticationState state1 = generateMinimalAuthnState();
+        final SamlAuthenticationState state2 = assertSerializationRoundTrip(state1);
+        assertThat(assertSerializationRoundTrip(state2), equalTo(state1));
+    }
+
+    private SamlAuthenticationState generateAuthnState() {
+        SamlAuthenticationState authnState = new SamlAuthenticationState();
+        authnState.setRequestedAcsUrl("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs");
+        authnState.setEntityId("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        authnState.setAuthnRequestId(randomAlphaOfLength(12));
+        authnState.setRequestedNameidFormat(randomFrom(NameID.TRANSIENT, NameID.PERSISTENT, NameID.EMAIL));
+        return authnState;
+    }
+
+    private SamlAuthenticationState generateMinimalAuthnState() {
+        SamlAuthenticationState authnState = new SamlAuthenticationState();
+        authnState.setRequestedAcsUrl("https://" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8) + "/saml/acs");
+        authnState.setEntityId("urn:" + randomAlphaOfLengthBetween(4, 8) + "." + randomAlphaOfLengthBetween(4, 8));
+        authnState.setAuthnRequestId(null);
+        authnState.setRequestedNameidFormat(null);
+        return authnState;
+    }
+
+    private SamlAuthenticationState assertXContentRoundTrip(SamlAuthenticationState obj1) throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final boolean humanReadable = randomBoolean();
+        final BytesReference bytes1 = XContentHelper.toXContent(obj1, xContentType, humanReadable);
+        try (XContentParser parser = XContentHelper.createParser(
+            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes1, xContentType)) {
+            final SamlAuthenticationState obj2 = SamlAuthenticationState.fromXContent(parser);
+            assertThat(obj2, equalTo(obj1));
+
+            final BytesReference bytes2 = XContentHelper.toXContent(obj2, xContentType, humanReadable);
+            assertThat(bytes2, equalTo(bytes1));
+
+            return obj2;
+        }
+    }
+
+    private SamlAuthenticationState assertSerializationRoundTrip(SamlAuthenticationState state) throws IOException {
+        final Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_7_0, Version.CURRENT);
+        final SamlAuthenticationState read = copyWriteable(state, new NamedWriteableRegistry(List.of()),
+            SamlAuthenticationState::new, version);
+        MatcherAssert.assertThat("Serialized state with version [" + version + "] does not match original object", read, equalTo(state));
+        return read;
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSignerTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSignerTests.java
new file mode 100644
index 0000000000000..7fe52a51baacf
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlObjectSignerTests.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.support;
+
+import org.elasticsearch.xpack.core.security.support.RestorableContextClassLoader;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase;
+import org.hamcrest.Matchers;
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.opensaml.saml.common.SignableSAMLObject;
+import org.opensaml.saml.saml2.core.Issuer;
+import org.opensaml.saml.saml2.core.LogoutRequest;
+import org.opensaml.saml.saml2.core.NameID;
+import org.opensaml.saml.saml2.core.NameIDType;
+import org.opensaml.saml.saml2.core.Response;
+import org.opensaml.saml.saml2.core.Status;
+import org.opensaml.saml.saml2.core.StatusCode;
+import org.opensaml.security.x509.X509Credential;
+import org.opensaml.xmlsec.signature.support.SignatureException;
+import org.opensaml.xmlsec.signature.support.SignatureValidator;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.iterableWithSize;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SamlObjectSignerTests extends IdpSamlTestCase {
+
+    private SamlFactory samlFactory;
+    @Before
+    public void setupState() {
+        SamlInit.initialize();
+        samlFactory = new SamlFactory();
+    }
+
+    public void testSignLogoutRequest() throws Exception {
+        final List<X509Credential> credentials = readCredentials();
+        assertThat(credentials.size(), greaterThanOrEqualTo(2));
+        final X509Credential credential = credentials.get(0);
+        final X509Credential alternateCredential = credentials.get(1);
+
+        final String entityId = "https://" + randomAlphaOfLengthBetween(3, 8) + ".example.com/" + randomAlphaOfLengthBetween(2, 12);
+        final LogoutRequest request = createLogoutRequest(entityId);
+        final Element signedElement = sign(request, entityId, credential);
+        verifySignatureExists(signedElement);
+
+        final LogoutRequest signedRequest = domElementToXmlObject(signedElement, LogoutRequest.class);
+        try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(SignatureValidator.class)) {
+            // verify with correct credential
+            SignatureValidator.validate(signedRequest.getSignature(), credential);
+            // fail with incorrect credential
+            expectThrows(SignatureException.class,
+                () -> SignatureValidator.validate(signedRequest.getSignature(), alternateCredential)
+            );
+        }
+    }
+
+    public void testSignAuthResponse() throws Exception {
+        final List<X509Credential> credentials = readCredentials();
+        assertThat(credentials.size(), greaterThanOrEqualTo(2));
+        final X509Credential credential = credentials.get(0);
+        final X509Credential alternateCredential = credentials.get(1);
+
+        final String entityId = "uri://" + randomAlphaOfLengthBetween(3, 8) + ".example.com/" + randomAlphaOfLengthBetween(2, 12);
+        final Response request = createAuthnResponse(entityId);
+        final Element signedElement = sign(request, entityId, credential);
+        verifySignatureExists(signedElement);
+
+        final Response signedResponse = domElementToXmlObject(signedElement, Response.class);
+        try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(SignatureValidator.class)) {
+            // verify with correct credential
+            SignatureValidator.validate(signedResponse.getSignature(), credential);
+            // fail with incorrect credential
+            expectThrows(SignatureException.class,
+                () -> SignatureValidator.validate(signedResponse.getSignature(), alternateCredential)
+            );
+        }
+    }
+
+    private void verifySignatureExists(Element signedElement) {
+        final NodeList signatures = signedElement.getElementsByTagNameNS("http://www.w3.org/2000/09/xmldsig#", "Signature");
+        assertThat(signatures.getLength(), Matchers.equalTo(1));
+
+        final Node sigNode = signatures.item(0);
+        assertThat(sigNode, Matchers.instanceOf(Element.class));
+        assertThat(sigNode.getLocalName(), Matchers.equalTo("Signature"));
+
+        final List<Element> children = getChildren(sigNode, Element.class);
+        assertThat(children, iterableWithSize(2));
+        assertThat(children.get(0).getLocalName(), Matchers.equalTo("SignedInfo"));
+        assertThat(children.get(1).getLocalName(), Matchers.equalTo("SignatureValue"));
+    }
+
+    private Element sign(SignableSAMLObject request, String entityId, X509Credential credential) {
+        SamlIdentityProvider idp = buildIdP(entityId, credential);
+
+        SamlObjectSigner signer = new SamlObjectSigner(samlFactory, idp);
+        return signer.sign(request);
+    }
+
+    private SamlIdentityProvider buildIdP(String entityId, X509Credential credential) {
+        SamlIdentityProvider idp = mock(SamlIdentityProvider.class);
+        when(idp.getEntityId()).thenReturn(entityId);
+        when(idp.getSigningCredential()).thenReturn(credential);
+        when(idp.getEntityId()).thenReturn(entityId);
+        return idp;
+    }
+
+    private LogoutRequest createLogoutRequest(String entityId) {
+        final LogoutRequest request = samlFactory.buildObject(LogoutRequest.class, LogoutRequest.DEFAULT_ELEMENT_NAME);
+        request.setNotOnOrAfter(DateTime.now().plusMinutes(15));
+
+        final NameID nameID = samlFactory.buildObject(NameID.class, NameID.DEFAULT_ELEMENT_NAME);
+        nameID.setFormat(NameIDType.TRANSIENT);
+        nameID.setValue(randomAlphaOfLengthBetween(24, 64));
+        request.setNameID(nameID);
+
+        final Issuer issuer = samlFactory.buildObject(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(entityId);
+        request.setIssuer(issuer);
+        return request;
+    }
+
+    private Response createAuthnResponse(String entityId) {
+        final Response response = samlFactory.buildObject(Response.class, Response.DEFAULT_ELEMENT_NAME);
+        final DateTime now = DateTime.now();
+        response.setIssueInstant(now);
+        response.setID(randomAlphaOfLength(24));
+
+        final StatusCode code = samlFactory.buildObject(StatusCode.class, StatusCode.DEFAULT_ELEMENT_NAME);
+        code.setValue(StatusCode.AUTHN_FAILED);
+
+        Status status = samlFactory.buildObject(Status.class, Status.DEFAULT_ELEMENT_NAME);
+        status.setStatusCode(code);
+        response.setStatus(status);
+
+        final Issuer issuer = samlFactory.buildObject(Issuer.class, Issuer.DEFAULT_ELEMENT_NAME);
+        issuer.setValue(entityId);
+        response.setIssuer(issuer);
+        return response;
+    }
+
+    private <T extends Node> List<T> getChildren(Node node, Class<T> type) {
+        final NodeList childNodes = node.getChildNodes();
+        final List<T> result = new ArrayList<>(childNodes.getLength());
+        for (int i = 0; i < childNodes.getLength(); i++) {
+            final Node child = childNodes.item(i);
+            if (type.isInstance(child)) {
+                result.add(type.cast(child));
+            }
+        }
+        return result;
+    }
+
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java
new file mode 100644
index 0000000000000..404f58fa6a3c1
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java
@@ -0,0 +1,297 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.test;
+
+import io.netty.util.ThreadDeathWatcher;
+import io.netty.util.concurrent.GlobalEventExecutor;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.Netty4Plugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.SecurityField;
+import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.support.Hasher;
+import org.elasticsearch.xpack.idp.LocalStateIdentityProviderPlugin;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.ExternalResource;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
+import static java.nio.file.StandardOpenOption.WRITE;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_EMAIL;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_GIVEN_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_CONTACT_SURNAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ENTITY_ID;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_DISPLAY_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_NAME;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_ORGANIZATION_URL;
+import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SSO_REDIRECT_ENDPOINT;
+import static org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults.APPLICATION_NAME_SETTING;
+import static org.elasticsearch.xpack.idp.saml.sp.ServiceProviderDefaults.NAMEID_FORMAT_SETTING;
+import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT;
+
+public abstract class IdentityProviderIntegTestCase extends ESIntegTestCase {
+
+    // Local Security Cluster user
+    public static final String SAMPLE_USER_NAME = "es_user";
+    public static final String SAMPLE_USER_PASSWORD = "es_user_password";
+    public static final String SAMPLE_USER_PASSWORD_HASHED =
+        new String(Hasher.resolve("bcrypt9").hash(new SecureString(SAMPLE_USER_PASSWORD.toCharArray())));
+    public static final String SAMPLE_USER_ROLE = "es_user_role";
+    // User that is authenticated to the Security Cluster in order to perform SSO to cloud resources
+    public static final String SAMPLE_IDPUSER_NAME = "idp_user";
+    public static final String SAMPLE_IDPUSER_PASSWORD = "idp_user_password";
+    public static final String SAMPLE_IDPUSER_PASSWORD_HASHED =
+        new String(Hasher.resolve("bcrypt9").hash(new SecureString(SAMPLE_IDPUSER_PASSWORD.toCharArray())));
+    public static final String SAMPLE_IDPUSER_ROLE = "idp_user_role";
+    // Cloud console user that calls all IDP related APIs
+    public static final String CONSOLE_USER_NAME = "console_user";
+    public static final String CONSOLE_USER_PASSWORD = "console_user_password";
+    public static final String CONSOLE_USER_PASSWORD_HASHED =
+        new String(Hasher.resolve("bcrypt9").hash(new SecureString(CONSOLE_USER_PASSWORD.toCharArray())));
+    public static final String CONSOLE_USER_ROLE = "console_user_role";
+    public static final String SP_ENTITY_ID = "ec:abcdef:123456";
+    public static final RequestOptions REQUEST_OPTIONS_AS_CONSOLE_USER = RequestOptions.DEFAULT.toBuilder()
+        .addHeader("Authorization", basicAuthHeaderValue(CONSOLE_USER_NAME,
+            new SecureString(CONSOLE_USER_PASSWORD.toCharArray())))
+        .build();
+    private static Path PARENT_DIR;
+
+    @BeforeClass
+    public static void setup() {
+        PARENT_DIR = createTempDir();
+    }
+
+    /**
+     * A JUnit class level rule that runs after the AfterClass method in {@link ESIntegTestCase},
+     * which stops the cluster. After the cluster is stopped, there are a few netty threads that
+     * can linger, so we wait for them to finish otherwise these lingering threads can intermittently
+     * trigger the thread leak detector
+     */
+    @ClassRule
+    public static final ExternalResource STOP_NETTY_RESOURCE = new ExternalResource() {
+        @Override
+        protected void after() {
+            try {
+                GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            } catch (IllegalStateException e) {
+                if (e.getMessage().equals("thread was not started") == false) {
+                    throw e;
+                }
+                // ignore since the thread was never started
+            }
+
+            try {
+                ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        }
+    };
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        final Path home = nodePath(PARENT_DIR, nodeOrdinal);
+        final Path xpackConf = home.resolve("config");
+        try {
+            Files.createDirectories(xpackConf);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        writeFile(xpackConf, "roles.yml", configRoles());
+        writeFile(xpackConf, "users", configUsers());
+        writeFile(xpackConf, "users_roles", configUsersRoles());
+        Settings.Builder builder = Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(XPackSettings.SECURITY_ENABLED.getKey(), true)
+            .put(NetworkModule.TRANSPORT_TYPE_KEY, randomBoolean() ? SecurityField.NAME4 : SecurityField.NIO)
+            .put(NetworkModule.HTTP_TYPE_KEY, randomBoolean() ? SecurityField.NAME4 : SecurityField.NIO)
+            .put("xpack.idp.enabled", true)
+            .put(IDP_ENTITY_ID.getKey(), "urn:elastic:cloud:idp")
+            .put(IDP_SSO_REDIRECT_ENDPOINT.getKey(), "https://idp.org/sso/redirect")
+            .put(IDP_ORGANIZATION_NAME.getKey(), "Identity Provider")
+            .put(IDP_ORGANIZATION_DISPLAY_NAME.getKey(), "Identity Provider")
+            .put(IDP_ORGANIZATION_URL.getKey(), "https://idp.org")
+            .put(IDP_CONTACT_GIVEN_NAME.getKey(), "Tony")
+            .put(IDP_CONTACT_SURNAME.getKey(), "Stark")
+            .put(IDP_CONTACT_EMAIL.getKey(), "tony@starkindustries.com")
+            .put(APPLICATION_NAME_SETTING.getKey(), "elastic-cloud")
+            .put(NAMEID_FORMAT_SETTING.getKey(), TRANSIENT)
+            .put("xpack.idp.signing.key", resolveResourcePath("/keypair/keypair_RSA_2048.key"))
+            .put("xpack.idp.signing.certificate", resolveResourcePath("/keypair/keypair_RSA_2048.crt"))
+            .put("xpack.security.authc.realms." + FileRealmSettings.TYPE + ".file.order", 0)
+            .put("xpack.security.authc.realms." + NativeRealmSettings.TYPE + ".index.order", "1")
+            .put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true)
+            .put("xpack.license.self_generated.type", "trial");
+        return builder.build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(LocalStateIdentityProviderPlugin.class, Netty4Plugin.class, CommonAnalysisPlugin.class);
+    }
+
+    @Override
+    protected boolean addMockTransportService() {
+        return false; // security has its own transport service
+    }
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable http
+    }
+
+    @Override
+    protected Function<Client, Client> getClientWrapper() {
+        Map<String, String> headers = Collections.singletonMap("Authorization",
+            basicAuthHeaderValue(SAMPLE_USER_NAME, new SecureString(SAMPLE_USER_PASSWORD.toCharArray())));
+        // we need to wrap node clients because we do not specify a user for nodes and all requests will use the system
+        // user. This is ok for internal n2n stuff but the test framework does other things like wiping indices, repositories, etc
+        // that the system user cannot do. so we wrap the node client with a user that can do these things since the client() calls
+        // return a node client
+        return client -> (client instanceof NodeClient) ? client.filterWithHeader(headers) : client;
+    }
+
+    @Override
+    protected Path nodeConfigPath(int nodeOrdinal) {
+        return nodePath(PARENT_DIR, nodeOrdinal).resolve("config");
+    }
+
+    private String configRoles() {
+        // test role allows for everything
+        return SAMPLE_USER_ROLE + ":\n" +
+            "  cluster: [ ALL ]\n" +
+            "  indices:\n" +
+            "    - names: '*'\n" +
+            "      allow_restricted_indices: true\n" +
+            "      privileges: [ ALL ]\n" +
+            "\n" +
+            // IDP end user doesn't need any privileges on the security cluster
+            SAMPLE_IDPUSER_ROLE + ":\n" +
+            // Could switch to grant apikey for user and call this as console_user
+            "  cluster: ['cluster:admin/xpack/security/api_key/create']\n" +
+            "  indices: []\n" +
+            "  applications:\n " +
+            "    - application: elastic-cloud\n" +
+            "       resources: [ '" + SP_ENTITY_ID + "' ]\n" +
+            "       privileges: [ 'sso:superuser' ]\n" +
+            "\n" +
+            // Console user should be able to call all IDP related endpoints
+            CONSOLE_USER_ROLE + ":\n" +
+            "  cluster: ['cluster:admin/idp/*']\n" +
+            "  indices: []\n";
+    }
+
+    private String configUsers() {
+        return SAMPLE_USER_NAME + ":" + SAMPLE_USER_PASSWORD_HASHED + "\n" +
+            SAMPLE_IDPUSER_NAME + ":" + SAMPLE_IDPUSER_PASSWORD_HASHED + "\n" +
+            CONSOLE_USER_NAME + ":" + CONSOLE_USER_PASSWORD_HASHED + "\n";
+    }
+
+    private String configUsersRoles() {
+        return SAMPLE_USER_ROLE + ":" + SAMPLE_USER_NAME + "\n" +
+            SAMPLE_IDPUSER_ROLE + ":" + SAMPLE_IDPUSER_NAME + "\n" +
+            CONSOLE_USER_ROLE + ":" + CONSOLE_USER_NAME + "\n";
+    }
+
+    Path nodePath(Path confDir, final int nodeOrdinal) {
+        return confDir.resolve(getCurrentClusterScope() + "-" + nodeOrdinal);
+    }
+
+    protected Scope getCurrentClusterScope() {
+        return getCurrentClusterScope(this.getClass());
+    }
+
+    private static Scope getCurrentClusterScope(Class<?> clazz) {
+        ClusterScope annotation = getAnnotation(clazz);
+        return annotation == null ? Scope.SUITE : annotation.scope();
+    }
+
+    private static ClusterScope getAnnotation(Class<?> clazz) {
+        if (clazz == Object.class || clazz == IdentityProviderIntegTestCase.class) {
+            return null;
+        }
+        ClusterScope annotation = clazz.getAnnotation(ClusterScope.class);
+        if (annotation != null) {
+            return annotation;
+        }
+        return getAnnotation(clazz.getSuperclass());
+    }
+
+    private static String writeFile(Path folder, String name, byte[] content) {
+        final Path path = folder.resolve(name);
+        Path tempFile = null;
+        try {
+            tempFile = Files.createTempFile(path.getParent(), path.getFileName().toString(), "tmp");
+            try (OutputStream os = Files.newOutputStream(tempFile, CREATE, TRUNCATE_EXISTING, WRITE)) {
+                Streams.copy(content, os);
+            }
+
+            try {
+                Files.move(tempFile, path, REPLACE_EXISTING, ATOMIC_MOVE);
+            } catch (final AtomicMoveNotSupportedException e) {
+                Files.move(tempFile, path, REPLACE_EXISTING);
+            }
+        } catch (final IOException e) {
+            throw new UncheckedIOException(String.format(Locale.ROOT, "could not write file [%s]", path.toAbsolutePath()), e);
+        } finally {
+            // we are ignoring exceptions here, so we do not need handle whether or not tempFile was initialized nor if the file exists
+            IOUtils.deleteFilesIgnoringExceptions(tempFile);
+        }
+        return path.toAbsolutePath().toString();
+    }
+
+    private static String writeFile(Path folder, String name, String content) {
+        return writeFile(folder, name, content.getBytes(StandardCharsets.UTF_8));
+    }
+
+    private Path resolveResourcePath(String resourcePathToFile) {
+        try {
+            Path path = createTempFile();
+            try (InputStream resourceInput = IdentityProviderIntegTestCase.class.getResourceAsStream(resourcePathToFile)) {
+                Files.copy(resourceInput, path, StandardCopyOption.REPLACE_EXISTING);
+            }
+            return path;
+        } catch (IOException e) {
+            throw new ElasticsearchException("Failed to resolve resource (Path=[{}])", e, resourcePathToFile);
+        }
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdpSamlTestCase.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdpSamlTestCase.java
new file mode 100644
index 0000000000000..54ba329dedc1b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/test/IdpSamlTestCase.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.idp.saml.test;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.FileMatchers;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
+import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProvider;
+import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderResolver;
+import org.elasticsearch.xpack.idp.saml.support.SamlFactory;
+import org.elasticsearch.xpack.idp.saml.support.SamlInit;
+import org.elasticsearch.xpack.idp.saml.support.XmlValidator;
+import org.hamcrest.Matchers;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.mockito.Mockito;
+import org.opensaml.core.xml.XMLObject;
+import org.opensaml.core.xml.io.MarshallingException;
+import org.opensaml.core.xml.io.Unmarshaller;
+import org.opensaml.core.xml.io.UnmarshallerFactory;
+import org.opensaml.core.xml.io.UnmarshallingException;
+import org.opensaml.core.xml.util.XMLObjectSupport;
+import org.opensaml.security.x509.BasicX509Credential;
+import org.opensaml.security.x509.X509Credential;
+import org.w3c.dom.Element;
+
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.UncheckedIOException;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.security.PrivateKey;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.stream.Collectors;
+
+import static org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport.getUnmarshallerFactory;
+
+public abstract class IdpSamlTestCase extends ESTestCase {
+
+    private static Locale restoreLocale;
+
+    @BeforeClass
+    public static void localeChecks() throws Exception {
+        Logger logger = LogManager.getLogger(IdpSamlTestCase.class);
+        SamlInit.initialize();
+        if (isTurkishLocale()) {
+            // See: https://github.com/elastic/elasticsearch/issues/29824
+            logger.warn("Attempting to run SAML test on turkish-like locale, but that breaks OpenSAML. Switching to English.");
+            restoreLocale = Locale.getDefault();
+            Locale.setDefault(Locale.ENGLISH);
+        }
+    }
+
+    private static boolean isTurkishLocale() {
+        return Locale.getDefault().getLanguage().equals(new Locale("tr").getLanguage())
+            || Locale.getDefault().getLanguage().equals(new Locale("az").getLanguage());
+    }
+
+    @AfterClass
+    public static void restoreLocale() {
+        if (restoreLocale != null) {
+            Locale.setDefault(restoreLocale);
+            restoreLocale = null;
+        }
+    }
+
+    protected static void mockRegisteredServiceProvider(SamlIdentityProvider idp, String entityId, SamlServiceProvider sp) {
+        Mockito.doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, Matchers.arrayWithSize(3));
+            assertThat(args[0], Matchers.equalTo(entityId));
+            assertThat(args[args.length-1], Matchers.instanceOf(ActionListener.class));
+            ActionListener<SamlServiceProvider> listener = (ActionListener<SamlServiceProvider>) args[args.length-1];
+
+            listener.onResponse(sp);
+            return null;
+        }).when(idp).getRegisteredServiceProvider(Mockito.eq(entityId), Mockito.anyBoolean(), Mockito.any(ActionListener.class));
+    }
+
+    protected static void mockRegisteredServiceProvider(SamlServiceProviderResolver resolverMock, String entityId,
+                                                        SamlServiceProvider sp) {
+        Mockito.doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            assertThat(args, Matchers.arrayWithSize(2));
+            assertThat(args[0], Matchers.equalTo(entityId));
+            assertThat(args[args.length-1], Matchers.instanceOf(ActionListener.class));
+            ActionListener<SamlServiceProvider> listener = (ActionListener<SamlServiceProvider>) args[args.length-1];
+
+            listener.onResponse(sp);
+            return null;
+        }).when(resolverMock).resolve(Mockito.eq(entityId), Mockito.any(ActionListener.class));
+    }
+
+    protected List<X509Credential> readCredentials() throws CertificateException, IOException {
+        List<X509Credential> list = new ArrayList<>(2);
+        list.add(readCredentials("RSA", 1024));
+        list.add(readCredentials("RSA", 2048));
+        Collections.shuffle(list, random());
+        return list;
+    }
+
+    protected X509Credential readCredentials(String type, int size) throws CertificateException, IOException {
+        Path certPath = getDataPath("/keypair/keypair_" + type + "_" + size + ".crt");
+        Path keyPath = getDataPath("/keypair/keypair_" + type + "_" + size + ".key");
+        assertThat(certPath, FileMatchers.isRegularFile());
+        assertThat(keyPath, FileMatchers.isRegularFile());
+
+        final X509Certificate[] certificates = CertParsingUtils.readX509Certificates(List.of(certPath));
+        assertThat("Incorrect number of certificates in " + certPath, certificates, Matchers.arrayWithSize(1));
+
+        final PrivateKey privateKey = PemUtils.readPrivateKey(keyPath, () -> new char[0]);
+        return new BasicX509Credential(certificates[0], privateKey);
+    }
+
+    protected <T extends XMLObject> T domElementToXmlObject(Element element, Class<T> type) throws UnmarshallingException {
+        final UnmarshallerFactory unmarshallerFactory = getUnmarshallerFactory();
+        Unmarshaller unmarshaller = unmarshallerFactory.getUnmarshaller(element);
+        assertThat(unmarshaller, Matchers.notNullValue());
+        final XMLObject object = unmarshaller.unmarshall(element);
+        assertThat(object, Matchers.instanceOf(type));
+        return type.cast(object);
+    }
+
+    protected void print(Element element, Writer writer, boolean pretty) throws TransformerException {
+        final Transformer serializer = new SamlFactory().getHardenedXMLTransformer();
+        if (pretty) {
+            serializer.setOutputProperty(OutputKeys.INDENT, "yes");
+        }
+        serializer.transform(new DOMSource(element), new StreamResult(writer));
+    }
+
+    protected String toString(XMLObject object) {
+        try {
+            return toString(XMLObjectSupport.marshall(object));
+        } catch (MarshallingException e) {
+            throw new RuntimeException("cannot marshall XML object to DOM", e);
+        }
+    }
+
+    protected String toString(Element element) {
+        try (StringWriter writer = new StringWriter()) {
+            print(element, writer, true);
+            return writer.toString();
+        } catch (TransformerException e) {
+            throw new RuntimeException("cannot transform XML element to string", e);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    protected void assertValidXml(String xml) throws Exception {
+        new XmlValidator( "saml-schema-metadata-2.0.xsd").validate(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)));
+    }
+
+    protected String joinCertificateLines(String... lines) {
+        return Arrays.stream(lines).collect(Collectors.joining(System.lineSeparator()));
+    }
+
+    protected String normaliseXml(String input) {
+        // Remove spaces between elements, and compress other spaces. These patterns don't use \s because
+        // that would match newlines.
+        return input.replaceAll("> +<", "><").replaceAll(" +", " ");
+    }
+}
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/README.txt b/x-pack/plugin/identity-provider/src/test/resources/keypair/README.txt
new file mode 100644
index 0000000000000..9e7dc1180c97f
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/README.txt
@@ -0,0 +1,6 @@
+# The RSA keypair certificates were generated with:
+
+elasticsearch-certutil cert --pem --out ${PWD}/keypair-rsa-2048.zip --days 54321 --keysize 2048 --name "CN=test,OU=idp,DC=elasticsearch,DC=org"
+
+elasticsearch-certutil cert --pem --out ${PWD}/keypair-rsa-1024.zip --days 54321 --keysize 1024 --name "CN=test-1024,OU=idp,DC=elasticsearch,DC=org"
+
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.crt b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.crt
new file mode 100644
index 0000000000000..77bef513f58e7
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.crt
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFCTCCAvGgAwIBAgIUC6eGyW4ccdmz4bbQHVtbogQ5Ip0wDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJc2FtbC10ZXN0MB4XDTIwMDIxMzA2MjEzNVoXDTIwMDMx
+NDA2MjEzNVowFDESMBAGA1UEAwwJc2FtbC10ZXN0MIICIjANBgkqhkiG9w0BAQEF
+AAOCAg8AMIICCgKCAgEAmy/NbPNOgv4YCA+RvTyWwS/pneWiVeqGEbCxTWetVK8k
+83hypxjSWZleQk8bVcfRVKwswYnIK1XwR1W0bnJyjXwo4dVT5nSOrzyLa3Z+BEIf
+KgRAk+rQ/o5u46HxXDifoiY45hP+AOA/0btvRXmQYkMI7R+/BjpntD3hNnh+8Z4V
+O+5+nTvlpVbOqEsanMG9VdeA8hL6rzve0mBU8zZkOkTlDOiWzIJST5n84phowt8a
+cNucrKyHxa90PJBC9vyva5o460QMiwRWe5SByP+h5j5n5mHMb3e8km22fctzMOct
+fCcIEAJMbw347euE7/ZjaueXtic5QCxcaoyDvf1v+Y/DJFb3xDnqXHwS+xzD4lM/
+3dScqHI5atpkghnhWSaFhEuH9BtQWwsTCojcckUbTFS/yQM5PJqiLhWk6QpazIOj
+S8kP9T9ta0zJDsfPKUNgrLWUn0cj8L6eZ87ET1E4fkJiD1WfeuvDmsPhImYCBWKY
+Iv3SXncs7uwe3EyJCuiJmncFXTJ2FruR6OkIOpWi6qh7V/ocTvSnP2MHJ08M8NwJ
+vv+ZIdjsAcEhn0kosfyI3ZkDsAwxait3ipG/asMp97T4qvDrnhHQXVYLsILq/+SV
+bZx42g29vzIFjGflqw0RxtxQvfb3xM9V71mnbJd5WHmN1IBp02Ct9dHi58Bh1z8C
+AwEAAaNTMFEwHQYDVR0OBBYEFFTTlGzPZb0FlDrRKRp1jB7e1J5HMB8GA1UdIwQY
+MBaAFFTTlGzPZb0FlDrRKRp1jB7e1J5HMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggIBAAM3Uz89I/7tfxtWW4lsPdE1jVJEzhufgu5Kj3JX7BfUpR1O
+6lz3RNHr55hjffAoiUd2//ZprmRgDnOcGx7P82fR7DXVsIBR6Jj19hO4w1Mwz2Ae
+kmdQ0Uiz1UYzul+U5o0+2WwH8sdgoNi+0U/LbgMFC3OT9XPAYvWfmJh07OKCJct1
++Eq0bVEZKbyQUzpwN1+fuX6DM7e21uOsZZgELi+pMx3UM8siw1veU6umbK6o1pML
+9px9j3rYGs8jP+s7bkrdfMam4tWmkcA9rYNHcjaTODuemaN5MH/q4HtvGUFyXTHH
+Z9cEc0LWg+83LmCIzCevmHmPpUHo285DFbs9gjC7effnXait7RH/SZ5ygR2c9bfX
+pOp5laVCnmgSg/EMkA47XnJI11iowlzqKZ/Bv6sydmB5V15ZJmyr61p/i7u0J6xj
+cCg0hZHZHBf2Wfihu2Cf1oK7NB7iwfJNHlKCcwc2RxIWAdCarF0e7yeLQdzbCy94
+orTQGIGgFu/ct8ip44yg5/HcJMvruDende9NKb7rjnSTxfqhBjjT6gnrxroKksWH
+eiMTg6g+ICTsAj4LHQ8LCoqdCBvGLOqmQeeDKZrYu1LRXbMD0PJchopGnCg0DAxP
+ndxNt7jpXjkrKH/C31LLRBD94ZSjeRCPqkoxeGTBTu0Uru3HNdlSnaxmr6gn
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.key b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.key
new file mode 100644
index 0000000000000..8d9454d33df4b
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA2_4096.key
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCbL81s806C/hgI
+D5G9PJbBL+md5aJV6oYRsLFNZ61UryTzeHKnGNJZmV5CTxtVx9FUrCzBicgrVfBH
+VbRucnKNfCjh1VPmdI6vPItrdn4EQh8qBECT6tD+jm7jofFcOJ+iJjjmE/4A4D/R
+u29FeZBiQwjtH78GOme0PeE2eH7xnhU77n6dO+WlVs6oSxqcwb1V14DyEvqvO97S
+YFTzNmQ6ROUM6JbMglJPmfzimGjC3xpw25ysrIfFr3Q8kEL2/K9rmjjrRAyLBFZ7
+lIHI/6HmPmfmYcxvd7ySbbZ9y3Mw5y18JwgQAkxvDfjt64Tv9mNq55e2JzlALFxq
+jIO9/W/5j8MkVvfEOepcfBL7HMPiUz/d1Jyocjlq2mSCGeFZJoWES4f0G1BbCxMK
+iNxyRRtMVL/JAzk8mqIuFaTpClrMg6NLyQ/1P21rTMkOx88pQ2CstZSfRyPwvp5n
+zsRPUTh+QmIPVZ9668Oaw+EiZgIFYpgi/dJedyzu7B7cTIkK6ImadwVdMnYWu5Ho
+6Qg6laLqqHtX+hxO9Kc/YwcnTwzw3Am+/5kh2OwBwSGfSSix/IjdmQOwDDFqK3eK
+kb9qwyn3tPiq8OueEdBdVguwgur/5JVtnHjaDb2/MgWMZ+WrDRHG3FC99vfEz1Xv
+Wadsl3lYeY3UgGnTYK310eLnwGHXPwIDAQABAoICAH5j8GtLviXxzKDEDW6SajXt
+T9fJru1KlObTgZQJXKIcA3xIHRj6nldbEenvg9PZaiQMFmeKT/z7gTaYFtvdWm7h
+MGq91Bdd1tfh96sOVpQhRkByNiZCmPS6DJZYzrrNYzYs6yp4HeeYvGGUxotO9skQ
+z7RQpsPrzYp+224BtWIT3jdxhq2ImwCOMsHeaU5CfHHtkpeV3ZXarR+qVYXARuEu
+O17IEAmAGom6YLwsZSZRjrdcJb58xhe7TnAFDoUaR8TbVoBGa/DNF6KO0SVjDfj3
+2qdH/7jJTEv2DytlB+xMXD9Bis9/D8FjH9qmdx75DP+En4s7ZuVdO+eLicCzKP5T
+2Na8bJ+QRo20k4P0hQLsT2oMag7jaTi9n3Qv8/Ia3V6108uGWtsT9xH9pbVxptMt
+4Y2TpGqe2ljs6nGoO4wVE55I++exutUpXyGC/NUS9TBhqwsjMtDd+43JI9hhZvFY
+0r6f4MJgr23WYPyVSNFi1c6p4wun3oJNd8HLxMLwtYCB3eS4+SgdFKfYRXXAZqIP
+dc0X4TtE90VzyCtOSmUSQVRJ3mDz9ey80ZH7bsGg6ZZ4JefdGNpCD6/QUqR4eHLy
+GDyr0jAkLWAG5q0TbnbhsihSaCQGgcPYSagTZUUDMAyQ7BkxoX+T12xMmgcXqGme
+ATu2I+sMT1mmMIW1HvABAoIBAQDLr1jhBRWObR6JFQr0frotimKsdlt1ENKzIbYj
+pP6lf2eFr7x0qie5gkpcWJizEyFs2mWmNtsdgHSpj8L0+PHnnRczSnYk/eo0AS+I
+vtci9/9dTJb7X+nwQ49LsijQt3UVqNfdSiFpxphqixB1C0R0ISxF4CfxpmGH4YYN
+TIBmhZwFS7/Dutrw/MtzTg7kpa/IK72CTn9b+Re9vWhFghgCUW12jnsUXKriYCRr
+73kUCHk6h8L7ZA4HseIDzKOzAwr/pyD1wl+wh2YPchy/FFIm4tMFYVZZnMFw7jIX
+cR3q1G0vI4pIzuze2x33Sad6VGL3eymatSJZUQ0ZQLBKctCRAoIBAQDDC5wWeBLA
+fHv6MxO5uToFZAfqWvdItQthxe8wDsLMJ0tWfB/j2ySqzVh6Nt3BeWkeR/E6ZAAr
+JqzNXvpoCwq4udWZVE3FjPBY/YlxQj0tjwfJaoDcWeCpxIj9AC+veKh8NH/5t7e8
+YLMgeoG4JsKQWUI7FZAZlRV36cWnkc3ICZcKv+A9Kxz9PtZWge5Fudd4cib8zMxX
++jrG/NvIhHYuQVee5jm2BflQB7OWntabjj8Q141IDPDiMIvarNiBliQQH5aL+Orv
+Q2uq8R3sOXfyTAh1P3blD3H3DOoi9kalqyoYCCpUP1J3H1hFe3M5xS7gt2D2b8ZI
+xWYCQQItURLPAoIBAGOVpZTO0Y1DGIzZNvP3Cbu+TyQW8fw4b2uZbd3bi40Kz2hz
+VWaM0sNGSmkAABh9n/gCh3ROma7A4UkNurnfBRbCnVc8bqsDYgBvG/h1peW47qeF
+DMR2ToBj56mQv5fcOmiOlovg9JksudRNjYxMJ/nHRkgboivptwGiZ9IUtypo5LFK
+KcXek7EDR81mOq3bvCfYCt2s12P2U383HcXuJyLrAOPYIaEA9aOccfI7sSz0P6E4
+Q0J51so4VFKbOOYK9NOtuNWvZO2AJYwjtJ5PpyLP/3A/+OHzXDiEnQUNDx2DIqbL
+b47NbM5Av1PqNPIAVCq/ksXLDbIxiH/yOr8Lh6ECggEBAKzYtwR6eDO4na3GYe6u
+pSpt0U3wO0BloKaO4D74dcyx7ePAsAofBEmRHJ2BzddNHsjE/JPAVx3mcjC1wLc/
+QcsedJoiyaEOG1jhplGGX+zl2gK2rWeZBq2sC7IZ3ihkhvs7E0Zbdyorj9Jyfk6R
+ms6NK1Js83yYT6QquQyc+34QcZgHHlNWx6PtccjL8Do+TSLiFoblfxlgGYKKRsNW
+D91B7sJUSER02tH/YTlTnd/QmTb1rrBxN29bkjTCBEQip+bZ93InrxtF5j5g2yGi
+dBd3Je+xxE2N+HL1MPC2yzi5jN4rLkfmrc//KRy6IAzMH5TpLbZ2q4OF27aXobRD
+KqUCggEATNxcmKNCEKBtpRHtcXa5qi4dEDIPYb2eeIrJMc0O/fD/SCiDzZL8Ewp3
+C4A7h1fTLxVCeAM2A+LYTLZXdZ04M4keOiBmagOMkTfl5UYcSJfP3UjHYf71HSFY
+pbJmow5iNlLizGExH+ZgV+3bKz2mJmBQVmVU46/1i5CeoJECllS/NBB9Qc571eG4
+ENJ1EcIMVcYsdAgB82L9mQtUYdxZJNy1QRMNbinuISqew6+NEzZyJlyyOtI0kEGm
+gOo6hf27PjiHHGvERgHVLLy4ZPz6JOVITWaYRVs5Oxemk/Q+v1Lv60MEYxzh9rni
+eJirRnJDCpyAo0tJ/FyjbG07Mu+dcg==
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.crt b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.crt
new file mode 100644
index 0000000000000..33d1a7d03bf2e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.crt
@@ -0,0 +1,15 @@
+-----BEGIN CERTIFICATE-----
+MIICYjCCAcugAwIBAgIUL9T74f75INSxmpirmjJTZHJXOGAwDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwIBcNMjAwMTI4MDgwNzA1WhgPMjE2ODEwMTkwODA3MDVaMFYxEzAR
+BgoJkiaJk/IsZAEZFgNvcmcxHTAbBgoJkiaJk/IsZAEZFg1lbGFzdGljc2VhcmNo
+MQwwCgYDVQQLEwNpZHAxEjAQBgNVBAMTCXRlc3QtMTAyNDCBnzANBgkqhkiG9w0B
+AQEFAAOBjQAwgYkCgYEAt2bngLBAj+gq7rBNSdfetiTemIAxtGZiSuXJj6+C/2A/
+PAA9iMh9zGxtyv5YEw7zz5dDHf3v3pp3LZOm1N16/J5V1kmZrkpLFYkIb+xmxVmp
+H0hOdtHhvEKUMoi77oFGLf/pP+TR+2TGW1v9O8kGzcaTV8gwPVVF6trJdpmzdkMC
+AwEAAaNNMEswHQYDVR0OBBYEFG932Q/6totkjHTHRdwkXOfxyjl5MB8GA1UdIwQY
+MBaAFGRvsxxsx22dsQX8cVcBaNlNz5uDMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQEL
+BQADgYEAhVvFReQd4rmd7gHOSJyZz8gEfHKNPY9G7CX82ojdHpHBQb2m5JSctHfx
+uk4G3CKimRTxgkA91JYoD7z6eHCVWfSuqSEqi8DX0RTkQ6/iEeU75uFPktMmqwR9
+Oy0uR7N64MOPBa7jcyPlNFxKsi0VTCvqUa4CeArghjheCrTuhZE=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.key b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.key
new file mode 100644
index 0000000000000..2cd0756824103
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_1024.key
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXAIBAAKBgQC3ZueAsECP6CrusE1J1962JN6YgDG0ZmJK5cmPr4L/YD88AD2I
+yH3MbG3K/lgTDvPPl0Md/e/emnctk6bU3Xr8nlXWSZmuSksViQhv7GbFWakfSE52
+0eG8QpQyiLvugUYt/+k/5NH7ZMZbW/07yQbNxpNXyDA9VUXq2sl2mbN2QwIDAQAB
+AoGBAIDiqD4fnr+DAOjMrQP/2FtYfbXeIRAOA3VvFr8LMJwXH1kjMhs2CnHmoSHO
+rEZ4c46nVlt1YWSmWKqG4O8m1o4QiESSbLNxP1AJddWBXeosDgK7pHNNZKprGGUz
+aef3IRYBsoVwo76TsdFkFskFe1Ibq16qdH/JZgEEom/3Y65BAkEA2mcAIiusLdkl
+WMIkX+JbJequ7s3jCqC4NSB5YSVQjAAyUENhX2dOqjwHj8280LK0Nh32E2Yg1OjL
+U2ks+dWp7wJBANb5b5jFQ/qiZpn8ZUzSFICJ1yZHtYTdL+YghdFe4V1/5VkRCntn
+p8BdB/KPYlqIh4nNah9nvI1Q7PRY1f+hHO0CQHuJM2wgo92zLArOleCemswRuS1e
+zAPg8REr8zxd7PejzNHVqyhZuQdVuTlAOIMb8g+CVwSWChAkNY5SuZv/nYUCQBqa
+N3MiRi1H14aKs0Z0YNEfPZ6hFRdh/8FsmOK16RGW+ByzfSaURzAI0yjzEWq+pncp
+Ek7cmXqrXwdFzwWZPGkCQC18L7C5mnT4it6z5DHRimCDkwzbNOEEl8xjt1k55lm0
+dy3nq3Ca4FQF5Nbz4SL7e0STIzT5zbd/N8BsRXe1npA=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.crt b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.crt
new file mode 100644
index 0000000000000..f7d5e0dd71bf9
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.crt
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDYzCCAkugAwIBAgIVAITQVqXYYUT0w04Z2gWAZ6pv7gwbMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMCAXDTIwMDEyODA2MzczNloYDzIxNjgxMDE5MDYzNzM2WjBRMRMw
+EQYKCZImiZPyLGQBGRYDb3JnMR0wGwYKCZImiZPyLGQBGRYNZWxhc3RpY3NlYXJj
+aDEMMAoGA1UECxMDaWRwMQ0wCwYDVQQDEwR0ZXN0MIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAujk+mVzI+qmf4gSJZdVVDdhFTi06kikb7FxG5JPu+gmU
+9Ke0LVEpP7Jp3gmhwsa18JUuvaepL1jnKmbbepKkEsvqUj4FuI/gImvFwb7X+xUw
+zNTYZAEvnZ4n16k0sBPuDuDibF0MGniVeLG3bD2VF3crFQrphFr+GZSXbVk5zIcS
+f6D6nSDcKmCNpVAK3jX9iV0nkr8cPtHOgprv1Y7mZgk5jwli9to0QD7r7OG5Db34
+R06JTMGTji+RULPISH1bc8FHdurRASkGmsei5GlJqPSuKdViuaKPmDrvKR8OK6gM
+zUd3pikJgD8veLxEuZ640FHPndPlvwJrSLwhitRgPQIDAQABo00wSzAdBgNVHQ4E
+FgQUD87H31WVQNfCc85/H2qhpzs3XfowHwYDVR0jBBgwFoAUVenAN+T06rqNDxjM
+cvgimnTw+FgwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAIvHYxT30cvoH
+WUE2saDVJ4qs/e0G3WusDyem3e4HkqwLEah06RDSgVCaOfW3ey5Q6CIQW3HHGUYq
+O0nU8JVCWdAk3+bUYJJOeLnwD+SbDxxKBhxLdx+BjWata85lfJTR9+dXs0RXAAN8
+dSiIaj9NSgnwiJqQQZf7i66S7XB58TDTdZlV3d26STLy5h7Uy6vyCka8Xu8HFQ4h
+H2qf2L6EhBbzVTB6tuyPQOQwrlLE65nhUNkfBbjZlre45UMc9GuxzHkbvd3HEQar
+oMHZxnu+/n/JDlgsrCYUEXnZnOXvgUPupPynoRdDN1F6r95TLyU9pYjDf/6zNPE8
+54VF6y1TqQ==
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.key b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.key
new file mode 100644
index 0000000000000..52eb2d494d60e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_2048.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAujk+mVzI+qmf4gSJZdVVDdhFTi06kikb7FxG5JPu+gmU9Ke0
+LVEpP7Jp3gmhwsa18JUuvaepL1jnKmbbepKkEsvqUj4FuI/gImvFwb7X+xUwzNTY
+ZAEvnZ4n16k0sBPuDuDibF0MGniVeLG3bD2VF3crFQrphFr+GZSXbVk5zIcSf6D6
+nSDcKmCNpVAK3jX9iV0nkr8cPtHOgprv1Y7mZgk5jwli9to0QD7r7OG5Db34R06J
+TMGTji+RULPISH1bc8FHdurRASkGmsei5GlJqPSuKdViuaKPmDrvKR8OK6gMzUd3
+pikJgD8veLxEuZ640FHPndPlvwJrSLwhitRgPQIDAQABAoIBAFLSQMf+Xxg0NnpT
+4DOdFG3K4M/rB6CXjeaqB5GDRYaImmnQ6C47aWf91l281OeGbsZn4tjQZ0s6c90E
+z9oMu9HsRPN8yBiUTG+ilNodIXt2qjR++RpI+L9V5oxgoaEpy/dsBueA3nArbva1
+u0j1fohoJI4AQ2js+xOvZUVpqG4QZcSmcYf8xM2t3hTE+jYXfOuNnX4PIU3fDqTp
+N0BP8A0sTpzPhuoSiICbqVSWiBe2UXojbhLp1D17R1Jm/PWuGYp8mUBVs5xJe4e+
+DGoek5JfC1H18b8/lmJZbN7njEsdA9XYOg67F02bgU0BGif3d0T5Sr0Yol19v2/o
+NNJhbFUCgYEA7Ts6kW/WRaZrRdBC3jhtOcZa1YiaoRFKk6m5hBpsmxmuOdIMtfgP
+94TN3hhi3XDrnG59vIukT0FSJy04qjbkIXToEi3C8BR6ALv228N34GLbq2zwfyqK
+1BnJkoGqCp8X3pScrg8nX11pSeS4ZGITJApSTnMT09BwwFqA48rpenMCgYEAyPTt
+05j4Fyg2jvXvS2oAJE4KUAo5kOy5PrHHkiJ7jaCC+ztCiebTa1V6QlwAeywhohBD
+kPcLDuNAtXlHq/9JC0Oym99fedQ7Fowk7VDjzNkPL2ZTbSTbzMwPeQVBrNP8AH06
+Kb9LCbvNlrymnciGbqRsnD3k4nVV2wwY0qBSno8CgYEAv8JjqlIbgvtttOYOVzCH
+Ye8rV3XvgAjJHqlQEy+E4K6WfZiDu8fDqFFKU5+kstjnKvDJZFIARCxJ6u8L4dMy
+/rRQLvmcnJIAuH03oxbS3KcsZ9pMYI6T37Tkki+LssA8/CdmqvAG5HhOdnOSdVot
+nLFXLKCqw0m3ClVd4EQSDG8CgYEAoSWYGZiPSbKbnaSmEemc14AlUO4CH69VFWtm
+yE8SmQ3gOyrYPtF1NJtFATXZxOwsxFMae4M/wN6Ho37v9IMwxfGxpsFHRODlyTz0
+18GIC1r8jFwYBmwKPn5c4gVPFxKfAJ1rKNMXj2Skhn4ddmW5N7xuxBP2GYtSH+fh
+d/XMiw0CgYEA6QjLGwgDJ3zmnrkJ+qcMjSS1yMWlhLpqHuSGj9P6hiF+szl3H9LU
+aH6ZpFWumzHPSqr3MJG0sHo/+Q1qQQduhiC5u5CXM1YEpnLQ6foK4TS5UXfOYfUJ
+lp0VuNrnPrycQ7tCtJ7s8uqWlFh3QufiUvCOLD5XS9Sw2rXs/vzI3aM=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.crt b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.crt
new file mode 100644
index 0000000000000..191f5ceb8db13
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.crt
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFCTCCAvGgAwIBAgIUei1EtkLvWStQusThWwgO14R+gFowDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJc2FtbC10ZXN0MB4XDTIwMDIwNjIwMzA0OFoXDTIwMDMw
+NzIwMzA0OFowFDESMBAGA1UEAwwJc2FtbC10ZXN0MIICIjANBgkqhkiG9w0BAQEF
+AAOCAg8AMIICCgKCAgEAuylnpy9h6cKUz6uCBi6eHBNW5GN6fQb1rU3BLjj5BMbX
+Oa4Ebj84OOsNVtS5/vskMtBH5R+Y1IT49Evd0pKgkJtvbso7avq/jfmi18uDI2Q7
+BjTzWC7aOkfaYXr5aBPEQpapjxV6QxirgMqz5x4WTgfBDfRr+gw2oXWsaVaQ7xkE
+CeIIra7j0vgOfTfSR4BQlbEiZktVvaE882kKGZPdDvklGs1/3DMHM09kRpxR/CUc
+3s8XtjH3DjUhK2MxX77gXxRXMMrWNWpk1/oXF/MnTkIiJhYqD7lnudbGXaLnaP38
+M6/OtHPluqomS5j2tVGjhh8vIHA/kXTh6lPH3rmGtHe3PgkARohojPsf62RXk22y
+cbXeMrvaw/V1cQmDUrmm8ha8qAi48uu6oRK3pk81j3LI+Q4SCbP1gllaRnwP7b8x
+gxbKEzNJp83+a8eM+6zLlLchxjP2MhU41VkzKtomQ+tJlT9mSFLD48b06EKTRUQ9
+rK5bpv1j+TOBCxfKAB9NSGwxcjlVhokxKKGrJlPb7aPJzqY4lcRzZJylhqz7gM+Q
+vSwOvGbw7m45hbBNGV2VJi00VP6d+O/2p2bg/fniV4rjh5pWgjKh0kLUzp9kqFDn
+bpZhTcBP1YrfR1M2YUP7BBA887AQ5nOD5PRaV58Sz4F+K4pbwG4c5Dld1WNWJTsC
+AwEAAaNTMFEwHQYDVR0OBBYEFFlOCjw+wyUxFg2kpSzLYROZ1WLjMB8GA1UdIwQY
+MBaAFFlOCjw+wyUxFg2kpSzLYROZ1WLjMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggIBAIxs61pAUU0R7j8guc9vOziNCm48KwrrqzSQ7JvgrPdrZ0mv
+3JN+luAFcA+4itVU5LqO92V5PVuxhbNLOvAx7+qR32zRF863Crpq/jAux+3azBsG
+yGyjqmA+bL1wXGLrn+Tmndwc90iEcRPcOZIDreSF0mRXesBVcXEg1E5gdKg4Hm7O
+dbMZEHITzQ1/uSAQ0WxONsABS0SYgplk20IyWm2nOjtt5zw54mD6TfSgeirQa6Ja
+oht6n7EfIIMo5wWxOXnAXOMdBzN4xf3S6u57fOXDkbHcS0zDYiEFP+FpXB+eabsH
+fUUKou7YHrzno2vaoAASLpfRXy3WlHqulg0ZaDspY/0mAZxwwUL294yO+/tBsh8y
+3a9qep/mEl0h0ghj25PeWR+XeNPBth1Y54XDKSXGjWXRekn0dLMzazksvJgDN6Ip
+0hX5StqHbiJX10WpWdSOT06ynh+N6OI+VBj8ndcnLGfSE6gtL9JH3IKNtUgodr6Z
++CcyZswWKutHyyZE5vteNQFKeTidCQAw9kRW6gtGUVRU0+PrMvD/8WhSd6WkFS4X
+jN+BXrmruCSGugdL9fgpg21qKcZwkR9rYQXqRPK+nTiVCRrOzUyTFnPmusz8fg7e
+g6ONaf2xMUeWfI+F8kK4NH5GkGggGqQDtes3Y+bWQ28lV7ny44TkMBARz6zH
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.key b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.key
new file mode 100644
index 0000000000000..1d83738b7a2b9
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/keypair/keypair_RSA_4096.key
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC7KWenL2HpwpTP
+q4IGLp4cE1bkY3p9BvWtTcEuOPkExtc5rgRuPzg46w1W1Ln++yQy0EflH5jUhPj0
+S93SkqCQm29uyjtq+r+N+aLXy4MjZDsGNPNYLto6R9phevloE8RClqmPFXpDGKuA
+yrPnHhZOB8EN9Gv6DDahdaxpVpDvGQQJ4gitruPS+A59N9JHgFCVsSJmS1W9oTzz
+aQoZk90O+SUazX/cMwczT2RGnFH8JRzezxe2MfcONSErYzFfvuBfFFcwytY1amTX
++hcX8ydOQiImFioPuWe51sZdoudo/fwzr860c+W6qiZLmPa1UaOGHy8gcD+RdOHq
+U8feuYa0d7c+CQBGiGiM+x/rZFeTbbJxtd4yu9rD9XVxCYNSuabyFryoCLjy67qh
+EremTzWPcsj5DhIJs/WCWVpGfA/tvzGDFsoTM0mnzf5rx4z7rMuUtyHGM/YyFTjV
+WTMq2iZD60mVP2ZIUsPjxvToQpNFRD2srlum/WP5M4ELF8oAH01IbDFyOVWGiTEo
+oasmU9vto8nOpjiVxHNknKWGrPuAz5C9LA68ZvDubjmFsE0ZXZUmLTRU/p347/an
+ZuD9+eJXiuOHmlaCMqHSQtTOn2SoUOdulmFNwE/Vit9HUzZhQ/sEEDzzsBDmc4Pk
+9FpXnxLPgX4rilvAbhzkOV3VY1YlOwIDAQABAoICAFLRl5RbWzBdcgwTEI47wqsZ
+w7F8c48vrTbq2Tji7Q44DrTvU/aU8wP8vwJVT5iM+Q+jKq0wtigUTzWK/LVZPMPA
+hCa6RmCoZGsms/BZlcXrbFLqy2OSF+8CLJhGGmb7mDT/BjjSgC+AkyOCjukOX0BY
+Hg1WwxD6ppH7yDc0nx2uLCNTahOw+A86xO6T7PDGzuHuaBJr85zd5GKxcE6xJ3ig
+ttKNbK67xcvmrbCxK3Yv1f6iFRQCOBiJWgwg8JA1noN0NMDagL9SPR11BRML/bCb
+gxDnbeVXXZbWeyCkHVJQUXFtskIXpuhasmdxYHoLjhijY+8uvHJZDj7qo97iqcAf
+jzbvnDq+59hDrcYiLAeYY9VPCyhzMTF1dybdqVRLslwGsryzIGAqVGZi7Suf6Ufw
+8HuLd2uJmkRf/g/hFxVcvkavylwpNKYniaw8O0lHqqh1jTVZFufF/E6MvH63y+wB
+2lj1PyXgg27Lh2ehm0btu/9/iuLYyNUSEuKJ9lTSHLdzHlWxmk7Wg3jOus/9B++X
+NQpCtpE8IZA4KoTkvnmsepPwzzGXgmSwnEraveV4c/59JOyVa7V8hEYGQMdeQcsa
+av7PZkgdSlYygfjyK0gu/d6tpTdbVWk6Wo2suTWvTHVosisqUdODiaK5rqwIeAyD
+wHWsTSlDiqZyuIqoyB3BAoIBAQDsUqYM8izDFLXhcstgdLfaU1drMDZze0FLqWyw
+ItcCeQNpAbev1w5FgaJn41NdYDSKmfH3yJ1rDlUwz7qI4L0PZO7mHe3AgVDvNbJO
+9w1cooQrv17zG2wPGbFosmiEzJdTl0Wbtd44AxSwSjCMmO5v0SJWbxnz/hkM47pG
+98TB2QMY30hh9fJTguVDzf8uo0Ba4IdK59k59JyBai3hIuVhwEpA+hzRMmX0IMHN
+QphmKcsM13B8mxJQtrw5WOeXiJypaEQ3g5z+fIxAP4ZPHFKLu6CR5+dYnbvRn7Qs
+v54JanujCLaQoY5lvMlAtc/ahQohuxHdiPY22JqeshdKfg4FAoIBAQDKvtw27L81
+GOEY6rC3qtcEVZbXWheytp2CzxYITXgRzDIcr64ftJijN107/n9R+bnfSCEKZXfb
+h+1MHM2ieVsGfT3WjRviEtmNXCWuMOUJJt/QeNXKLJ2D3G82P9QFdHn6KALOxE1L
+VCE6xRoTmGIw1zO7MhdZqURD7CiE5XOTQOpwJ1wxxJAJK1agbV8mI9+lGdpSDoex
+Ye4VxRVKFg+lAImy6NE+I7ILx08Wuo8tAe25lGKdnxaST4DfFSNQ97JMawJ259+7
+/4f/Z1k528UwiCBmmkPd32YRES1AQBS4dlyYo7b5d7JCfGgnluu0Ij4IAd6YG0Sq
+fDOX/wcJWoo/AoIBAQCScdT8iMY8TYU4JA7rwuh7ntG5VrMndOXNoTv9Glnq6lq2
+o53UlYv+forJnhF297sXeLZ9OhBy2vrnuJJ7Z+lAtOdmKn2+oWQqGDhavrXFBeCp
+y3lnZULdSKqrmoMyaCYOYtt71oPSafZUGGVlew+msw54SXjnvjp/xk8U001WYsiM
+jTEpo2N7ZM316p2LpRaoOiMcw/fU9kRV3hp4PUuv+50IDvX0nSqY0gPG0tHl3Vg6
+qPbIAMCky9hwISvCt6F9SWOcky6idUPkPHlaZM6yUJ78S50NSRGuoSzh3gxtAVOE
+gE3B6pt3aDm+o+weGfthU6+KfOk2q5uFZA8S1nKRAoIBAAiLZqiB7cUFGWd9bJkx
+AALUwJiu2VTU0WqT46/R4/+MBYij9sirScuHJ8zBewgvaB/64DmZ8SU1Y1DT3e69
+ApMRcCu8orIybSzB92yR7vUlyET28rukibbm8WDt67eXIxTh3sxsBX3pO8VITUMK
+Jwk76Jg5EjbbJV9fco23Rdms4LR9VCtcx107kFkOgvYOiPxiTIssxZc6M1iDh9Qp
+fdtMXIFHwpnF01kfuzuP/Ty+WhB6KnXOIJtl8l50e5qL9d22nNH6D7nSmR6wclg3
+5OWYtxTb54RQR/LsMGYGzVjA1EIylCGLY93ddF5GxARBdS/NNF5gt9IfjeUYAGnq
+gO0CggEAMUDRZs1YYiuwCMQRuvi3a9JtoJxQ05YdN04Njn5bC/0E4CFDT2eTFozQ
+JBLgxhwZ+UxFVMRLCGLzL7zaCYvwnnpzIHkTnfdPQJF5/LEEbYTlvKDdk6jqrwAW
+BhIuLxJk1r/opNWKOyAEsCADXnpPgbb+0jYuJgEHCIFeyExx8nZcCRVhZoZeFsJ3
+fO2fqc+mV+UN4r1wHejlKp10OrNEeuPE5tJuwO1/QtKVTFIxOUn0PqftoCUUwzI7
+R/2dJC5AFueMbVQlu7Rx2rI0S816sapYniPZrnv66QsDAYRic3/m84c3mVj3YCrp
+vkIxxqLwG5fIEleJmcvzZGyM1zAPew==
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/README.txt b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/README.txt
new file mode 100644
index 0000000000000..af802e68a2236
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/README.txt
@@ -0,0 +1,24 @@
+## Create JKS keystore with multiple entries
+keytool -genkey -alias signing1 -keyalg RSA -keysize 2048 -keystore multi_signing.jks -storepass signing -dname "CN=saml1-test"
+
+keytool -genkey -alias signing2 -keyalg RSA -keysize 2048 -keystore multi_signing.jks -storepass signing -dname "CN=saml2-test"
+
+keytool -genkey -alias signing3 -keyalg RSA -keysize 2048 -keystore multi_signing.jks -storepass signing -dname "CN=saml3-test"
+
+keytool -genkey -alias signing4 -keyalg EC -keysize 256 -keystore multi_signing.jks -storepass signing -dname "CN=saml4-test"
+
+## Create a PKCS12 with only signing1
+keytool -importkeystore -srckeystore multi_signing.jks  -destkeystore signing.p12 -deststoretype PKCS12 -deststorepass signing -destkeypass signing -alias signing1
+
+## Convert the keystore to a PKCS#12 one
+keytool -importkeystore -srckeystore multi_signing.jks  -destkeystore multi_signing.p12 -deststoretype PKCS12 -deststorepass signing -destkeypass signing
+
+## Export all keys (openssl pkcs12 doesn't have an `-alias` flag :( )
+openssl pkcs12 -in multi_signing.p12 -nocerts -nodes -out all_keys
+
+## Copy and paste the private keys files from all_keys into signing{1,2,3,4}.key
+
+## Export all certificates (openssl pkcs12 doesn't have an `-alias` flag :( )
+openssl pkcs12 -in multi_signing.p12 -nokeys -out all_certs
+
+## Copy and paste the certificates into signing{1,2,3,4}.crt
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.jks b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.jks
new file mode 100644
index 0000000000000000000000000000000000000000..6c5d73771197afa5aadf62e9afba59a175dc6530
GIT binary patch
literal 6640
zcmajjbx<75wg7N;aredDEi7)q-5r9%qDycM?hxF91%f-lH8_Fb5IjJFOJH#i1bMvg
z-aqaq_r0p=nmJQ*rhEEK)vvlAj~<WU;NXDI4dJ-~;85J5woXtdTi)lu`MM{(^Z9NN
zH~@@=gvN#hKmf1;aRC5CIE2@fUx7w5+p)Dw(o-xnx}gZxxisOnQ$P7>6CTh*5`BdD
z9^&hbaD?Qxdk&{#F9ye0v&L*>$32W0VpxCi1|3kNbMGoFf2y7qj<I3C=B0AnkefDS
z5AP|OhsJCZLx2gHtSb&n>D8{cR{uLd037107kpr$=PPM&C_o$>vXc)F1OR~ukc0^&
zFa{C|Yox-YC;~hX5e`hqg%8F>g4acW$47IwaCG41@UV9G08<iSa)ZHKJkJN85SYvG
zkC2yJ=-=UgOEUoDBO(9sb|B#2-M|Bo;icdZZbP!g$M027nZ4}fFx)^D%0w6+Z3rvp
zB}Lw%=-aqJ`Yft$<JQ(se7(eP@J^HZR*Wda&$f|5EQ}j?MqDwaifF)8V2bB~QeYu~
z5D04Ot3_yK``=oG3>nbMH_e{vTyFmP%i!hVz%oGUz?XewW5HeIv%@pLG<;_)#``28
zm{P`5QvICA<eF>cKx&oQeX?Q8ww?OpeC|E^{B62j5d|kX$B*Kf=tH&0?65zTn&*#F
zCuY2ZgF^&@LlGYTR_ss&SR73pXYw)WHK4%ow(-*$E4uno%)+eTkYv%0lJAD3xlEzv
z%Ws7aCM6Y{spDHKFFz;q4GV)8Mr-dY23T%v7nzPoI?(dHSIa*R`+c#KaNGVedFf;^
zaxgHF<^zdyhBxCqD__ugTB|)&lIU{W3KToE&Qd4GH@R6EX5obpRDjM#%BBu7rG<iv
zX6RA^W~Dx=m!J%Z|7@zrf%baebrXat44b09t$WzhOd<+*B_eG1iC}j($<P-Z^zSeR
zLM<81l?L$0ut)O51RX6qugSAByjAl+i073HkzI*`B;5#tmx7$_<<5wMmUrK7a%tDQ
zD{G9c(a%5TB8Kx6{E&;=P^6M^G^U`SuyMfr5-4_A5UI9kM5pHOLZ;*U8p7T(ZN_wJ
zCVem#5FK+z+{j&pl|R9;j!LwBi6IciG-E9;T1uU)Xepl8k>y^#DbfD)IW($I+~6gB
zGym;RAeW4vfFK=)nJ;7YJ<e;ObtG1niOpn&nH?${PbKREys}_IG+UILIwk^|4h&eV
zN=oyXP*f)PJ-J<EYDh6AfG_;&Me=JzeuTl}@CMU%T-gRN!#L+8dnnYyQG-MZJK=$s
z;{2x!n02Q-*=WyIi3AiN!6uup?8K+usi|w)E-+h3U7XoB!M1L9Tef8@l(AcyFo|RK
zMks-or!wgq!e=qI_g9>GLDL^XNG<4MflhcBlf{c50(%PeQlwe?jC+U5lx|v#qh3ZT
zt)XqIUKH%oh;C)x74PmBYHAmrXQUQ4+(UJk-bmh%H?Qon90*~L=-&ie_~kKUl#lTs
zfDu@Uhj^~@WO;@5m>yGkipzE#C7ouNXR?{_xpXXI0R0o)V@3v!X-3RY!wJcp$SHTm
zbh*YNG>4+K{BhD8B^h=J8J=m}f5RM%ENZSRkTVzx*;NyES7L*$3oKFUy{9Fq^Y0FR
zWVebC?);gQ_c0wZ`*J&eMxc}qaif`9hu@od@jA$gApQtE#^NGwux!*-7LB2|BT`t*
zY1^Y~w+XzS7?umvjddxb25pA9B&Olf;efirs$;DDhLx4F8x(RzW0>+W`evAB_#Amu
zs6wyl94<0TkxeR$#t;*?tSKe(`4wjN*OPlzsa@a?xMJF_+{)`cTXRI+1bWZ#a083i
zV!U)K4nBS4D9v0u(s83k&Nm}>?o+d%>6M1%hC^-p$IY%UXVSD0Yizni(MYNDPv(iT
zwMnMn0UdifCocq>nI_0Fk!tqR#nr@c`C01^H{}OMZwUq(cDx#c(=V5ji4~klW#{w<
z#u@OdV|LTie3=E@CcpdY{~GF&t8Kxv(0;<ckjXP1$X{{xSW?IEcWu=&74>2rP)3bp
za{Dk69BdlZ&kkhrgu9KU6?E(5vjd)Wm@puseXW1L4IR!Pxhr>po!y}bTcLWqW6;{x
z4C`@Qp+J~ubhJyfRAH|4&}><}pFCA!L7Jp3@kDn0Ky2~SBk70#Hq6Z=1@}}ZGGXMh
zGNEl&LRS5OzgUQs5pw=|Ii6NkL*)Z!bUfO(_d^7q$J3tHe58M5q&pVBFtJ$1pt0sO
zUc<!M%tkM%iye({Q9HjmkAs+VHR+;4+m9KBn96*OworTc`5YL?yvnfxX8HjYv8|3@
z-+reI9eF``t;+;mN&es#497<FmEC|puW}BB>QCALC%~b=k!Nx0|IH2h!p$8R6AASh
zgvc9A008|Tyy5wqH(-8V?*H)SkMM8a{5vfaK>ME^JYx$V3c!YY9s?RM6aauLm=k0*
zCW0*L?VamuC*d5JVi<|_uz>;^kMEFeve5OmkVcnMxsH~hjt&oP9BcOD36<kEk${G)
zTkQL9yUZ6C0*hR~?w3jZS_-T70q{&7L<)^DHk3gm>G+1^*@yE3BmUB)uJ9AR#+HL}
zwHZ-nAA(0^e@IW`LIb9Ei0A}QA6GNqhif3zZ<z(T5)^2K-5E$-DMSw@OMIQq?&PBs
zwoxZ?iV9`DJQJsD<nHaWdc20gBn(hXn?ow1@0JzE!^*Dd-w54>yiQ3LG$Tn7843}n
za)xC9^?lTxX``1vrE^kiR>YfMnqkux@+K_%V7Gre5&W6GDDr(sRDw4f2oHb*{Dsa>
zZjLf1sd_Zq-9Cc^dOB-`Q{p}UIf>6zdY(i89Ot#ElSe$QB5JIdhuKAUi95fXH;c>1
zv8##iso~pG%cS%RH;>rnka=}RklE}K1micv8mFymIOzvLx&Jn>Hkn+GBLV1No?t8i
z6>gSMOO;p-p@<3@YKTTtS}o-|mUEO2w3u2XUt$aq<efX7q3#A=Dti!T8T|a7p~7jf
zkJoZsCYSAhf@(!M{!`{+fG@)SvRg<UM@ZBW0AbJmoCIKAI4@EUns}9@v|!O;Yf3zz
z-yG#kM!p_dv(veCfW85_U>ry`<3gdw(=X)!3eDxeXxbi1<S*F2(>^D)ErkCBx8X<C
z8{TGFC&Kpd*=th1e`zi8_8sGr8|h84%%9xn{+IjC&*$f#I27UjAKc%2X}d8^6u@Ch
z_lC4e&}gw(H+A%?q)E=eApIp4574=&dmOdr$XH(1x8o5TnZC<lxHY!zPGSg8WS;28
z+TmqsEZIwrhn@{LRU#7qfqvK(_{^4E(?$IIOd}rxTda(Ud+~4ur8lO^>Op>Mwy1BF
zK76Az`m#z?R!50<TLkBI$;`)SSD_lT19t`++(w1bh~gjwkzy&8>$>;h33S5Aw{9jP
z*a*Qup5k!VCVnB_<6V?+n%bB9b>|>#<x?k@%I?j~ktw<$xz6^&6%%`VGY)y)My&WE
z<=W8Y=m-L8RU|r!-CNk?3u@7BEB?`o;x~c05mf5@#j&~FbvsH-(;-73`n=jL10~h1
zGR3I{!y7N{E4q%;Wa(=Ug;7N9FeZFKXlYKQ?a{<9F2=7(z*=d2Pdib{zDmWx@*m9b
zp99Oen~+03e}(6BhZ1Ok@vb6Jwcy!d&?ntcv%Yfl*v2-J((Ao-=pOUo%5RG<CZV3X
z><71m`H0|kWm8O38*$~V&FDGH(Lz^^%fq+t8D(u<4^3ITR_jG+)i6!oo?=67HBVC`
zK|UT&>X3-yMLrY)!Zr@yV1X%W!5fo0?nrB{dHU3pK#e+SvePIn{++olNF@MiuitYF
zsae|AQn{^-lGdtrcd&_cZaw2Ny^Vzwt{t@3B!$>HsNErDK&WJAL@E3$OTk*q(<O*v
zwGK!2DNj6JWvQ*S>SeqZHxPrzZw$SSh!`c3b@NpEy}V^Pb5MXzjkNg?_-F+Y-W9}#
zz-}<egx%Zims3U!Br)~ei*>J)?6j})VwHPU7lAt2LR19XXo|8?$tEcOlC<r*8x_kE
zB`17$|M9vbEur4uDRrGq_#qfSfSCp7tbj|*LCP$X6A-$yIl)hIkKN3HvmmI}HrgvE
zCbe~@iP0MO#al8+$(SOj&DPWYW#(@-AB`QB4385*oYg~E@we#R!*NSi0u#aCB|L$l
zF~sgAc$*@eb!8WWbRLclih&CQB_V|G0H5ul=-)c;8r^U6F=hz36d4>zFsAvEVJ+i*
zESw`@3n;I9(1^vmwu;V$e|553?Bk`Ll@vSxs2j%&v9`<sI(mEE+9Tn*HlJ+2^~}vB
zD%flmsgs)cr<HMt$NCLlMOSYrUndB!+(fi{NLY5+*-Ra8Y$)WuWzk}~VRfZxOcTZb
z#ZbfeIe!ICJGgsOSZuCJ0Y(-QS$p&0$B&#j$L{mC`#buq2R;$S;a~<A^}vF>iWB@;
zpRBF@&rd|};1-o*7)VX5WpQKHCwCFPxE+VdmuioE=^r`3llAA~YM7`;Ad5<W6JZpO
zb02}3LAf=;(?R<R9$#MmizVa>iRg}pvzB~hjm)szD_{N>st42{*~e9Q#ZUvLUm-Wa
zqW3Su1G3w^f=Kwdz4vtky;NnNUdKO4=rr^h@uuDuTv#ZpmEi{lar@O!f0pi-k&|&f
zm+aMX4{j_|b1bI%CKf^RUMt+hs=`;N*4e8g+I)1WWMtlLynZ!y)7-?bO5`mes@p41
z`?mJHS%ScsQw8tbxa&UO9NdY(92Z=C(Z$u4%!3KE#Ru<V2Yyhj^_@5@6KPSix{+?@
zOWW1)EYJJ$+AkdhR|Yq_9kOd@0Sl{$rG^K0(vd>!z%fmGXyH6XX?cb$quC2#zP8i|
zWhA!$TYXu?`Hz3`<Zpa)|AYEGe^&oF{BLRhpZc|oMVFOy+Y(OI8prvwo=aO^g&1zo
z&&A$&)S`KjXqYuDDC%Q-25c6gNfq)fP}FA2xrQDYKiLziAWGk*+6ea91_Y(e&dXUP
z>tJF1_l0Z*du}sGuOV-#g*`s$D{DQ9YZkWG%QrP2dU{rUiMdPOQjYSqY!XVpzmenc
zAUc|!U_}b=@KK|x7{@=fISIojJ7k?GN9R(n<Jv?*innM&j0prWF(T)S+5z=w`E#zK
zdJ!MsF%L*5B#29cL4-JIY&2=FcSI<J+j_$XTP72jk$OD~p-#iDscW;^D_Q~ex#f=7
z!_kqBsAQ&HIWB12@1kqGT`{y#K_Rvmb-4ee{_oVH6yahIcg+-in9W5lY*Vsb?=MyQ
zL;Vki6UH{b9Y0BVl6n?(Ac?u8Xm-w8ABxHO#T||yr%tSKO`Bq8r0CKyxZ*s?cO+&9
z!^Axc9T7#{E-5Kxe{sCD!FuxiI=3@&BCJ4LKv2XV8gjn3*R7E{H5brRm`bQ>j}F`x
z*+JePbux@v<3GTG-)pJaAcTQ`T^^jQ>CE41ru4WxZRj%SDRcv?oF#b?s}e^j0>SQ8
z4_-o3!|hE$JwMFwEXeX++^8rtiQ3dj+Jw`&wN5US{wCqL+XU@*G)>WdnWXeS1fVk$
z<LFcy7*ZqjQnXX=vqM?LuCKnaaT$(0y((ksH)z|1(gN)!IUHj)efE*nM)x*<E<Ug8
z|6^l^>tE_~xjw58cvk=6AJpGJn6t~TUQn^z;$50=x=YA^S;Kx@iMc2KVsMq&NCa(|
z%@8e_eRxS!ljR{knlH${Q2b{ikjkh$$0v^4D#|UR0gnWCj>Rz`O^fw6DFbZ`PV3}n
zO8gLMQVPN2In7qoWLErcxdNkXvYH(`t>y)kEfD_ri-1@^tcVS}K?qnWsOnZIjCJf6
zVH$l`uHd*A#z`wS){5pQP8)&Ns&lo;A9dAHH#@5<hZ9LU9o)?+KGz!C<9gQK3uY!B
z)=Q3}OLhZgk-}q9hzwO}s8_T<8fh^1R_{n*Z|zc;dOMi!v*q+m<7lMN$u^*ho#Xm0
zIy&K0I^?jVCL*55q4_q9>vjsBUyU@ytw*1FHFR}U0DOmpxg>VBF$kwnm~qMG+Qpe)
z<u&1c^SH{LrEw)w@4?YF(&{)iB^rw69xh$gxIJwYq!?7IIFsm19}&T9=8h9Vki}IK
zyRhD>2Pc(34Z(_mVpG{u(>Sx3Tlx<2qtNdOlT3s2ApJvF1z%-#mv{B6Y-Zf(h_Uze
zRhsAf<*n)SoD?Mjmu#843LPFw%jxPrrQWj^$u(RDYFRVxBj(E8XdTvEn%>9w6$ao@
zmL%cmXk7}0c@DqtS<I!v_?b$xe)P-1{zY{)eSBDmpMZ6aeB}_LH%H=<2%IMEx*<rJ
zhOsp=%Bu3!lBlj+v*~N-`7Pd64qOaD@q|@j;5)=D8lk2Ox7SIX1l>I&1dKjt$cGho
z{8qdtOpUvA0l9}8bC65n=vCoejg?qZfzeQO8Km(f><f8EmSzxVui%EoV2cV)ON;ob
zkvI;MYExlHQmw+uyhNmcm-He8$w-}$uK}7P?~7U|!1&E_sTwP?Xsg&g6e*b53zQj;
z%cO{H;Wd9^rJ}`%3JV=^H8`|VBBl9iuzK!K=krq)1DDt{+)%X`5!1|(%$_w9;oemt
zr*gw^VLj<=I#52D;6Z3mMpz4<wOUV)`#b@v(od#E&pz?R%Z;J%8=;i{q>~&*Cc+-Y
zjKx={(b+rm3vPNTLrN>aSLM6`4eyh$Bzi30(-*I*V8wdtPC=%}I_>F3(cM_NV~Rcv
z?^o1(ROx$+cq*6;XIBcBkROm-U(kdQrJ;nj!rpo4N)mzS$#D2mV_l6n;C>~TVhR8|
zXzL5=%C^yET$O!(m7HmaCV5lZW*Dv@MdTrq21zaGLz(CJ{WO|dVUmjdEFJ!k2Oe#P
z;)48^4=lziKRSG{egbcrRL7z{L<aTtqp~o57=mmB5)?i(2Xc{bbIFQ$3hIr}O5<fH
z>NZYO`#Sm>J-IwEdpA;xT=J|fWsv{8@lSLJ5>nF#6?Jg%HGsX7EjCWIali)hiOs*;
z)_zXF)px$jY*-)9dw$?ZwtPDI9@^x)7?f-?e~w7C^}z@8*x$C_&-_YIs(CLgG}j+%
zZ8L-aiC8a)J!uvVOGa%c=TikD)6KQ4YDBc0xewuoEiwF*^n!XWWj<Q2SzItS@5FL?
z-tUk3es}}t)rB>hmk#_@?t_&4nM{C&iHQ0=OXDs4GqRi2B`Zk3kZ>0-Oo62BhIM0b
z<|+hPi3Ib$x3v3vJNbf4Q;rbhm&X_2P`Qj{d#thH(DQ2bGY!f9gV%X)q87+vh+u*b
z+Ezolx~WNKReiz1!Kxz{@@`LqLo!iUg;X`_WdAq&$R3jK{sPM1{O0-x_PKcewAg<R
z|6AJs$Ns7kiomUWscghh0Y(7xUJ)a(_Pb6iPux7Yo3YUhm*sWY>|DEFT5w!i3va^+
z>*A*$2r-*6pwJubG(_un^SOC&KjiWzmPC8%coYJo>AYw*zQ%3cAK!RB#%wAiBWo9N
zOlTEu<H6^51~d)3C9{A&p*QV#aDpex15`4bS{&EL-ys(C*l)F;YWoiUc$x=&z?mgf
zy&jB^#Li-SB)U4}avskmY^rw0DzLTTTz2049ZA<h5R)S?pnAyzg8L|G=Cc${Xy;t$
zX@`U*P#oFml5%i{_(-;EKK6>2XcYa!@+z`ac5ZK{eKGG>-<-FRQ(SxMDr@8r^dhz7
z4CTguV!z43F)@2IiO9{ojY&`j<2|j%Tf)Co=@0fht5;^FFK};JMrUj#6G%`!(@OZM
zw}G3@5hPV_0yZpyp?a9(KQ(f2fC}8bQ5z(vmNm6;J7o=9Cqy<}wdyMb@t`JR-UT_C
zg72=ZxBe+w_A4~&i^6G)pZyhY1JlLh5XLT4uTnSO4&@OazWd#JS)<-8wPkn2!tPC<
zOrJruX;rU~DgCWXF4s`tf=ui8kb1aJ-&%?fI)1oY!AFV(8`W{KAkIuXhk$uH`Y9FM
zMhp@>85;=uL&obcC1X-lu_g-QijbPOGpEfINK=TeZzP8$pHK>y6v9S=*{GZ-V`r&`
zS`+E}V-Nd8lx_;xX{+MD&-K?|49*CD?^Q|PeUP)u8gWp0C*sep`=w$?{7Qhg@_zs@
CU7=b4

literal 0
HcmV?d00001

diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.p12 b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/multi_signing.p12
new file mode 100644
index 0000000000000000000000000000000000000000..a08be5d0a7206abd2b5cadbf2b456660983996bb
GIT binary patch
literal 7462
zcmb7|RZtw<wylB2-6gndpn=9ExCeK4cY>1ujYDt^mc|Jlg1cLA4UN0I`^h=?-o0zr
zefaCVtXVbYoU7J&`D)bwg2`#%U;#id*%BfOOSoM49R@5CECft83J)e5`YZPV!Pxu%
z(ZXJY2V>9um8XEgqljq#cL^O877pSK^aL^^qW(YW@4ck(_%;--7A08@qIbY9>-c%_
zE*E2GGaLY>R~24vGbb5}MZIse<w(tnjAKN%-dX)c|7tCVXPl~N<k@m2Oz5HJq&KLp
zU-w?61UhF`$vb#ydKL@%CM_Gb)vb?ry}c%$vT%gODFvkb*BeprIdLhlVLV{$U~FKV
z|DMi&D+oya?*Rf%e0W?<J0~kDP9A<Bw*Z#_kQXSx2?Qgk|EC`?{1XrifA{};`lhkK
z*XlFrdN{YWt0-2d_--0G{{MIa!w*S=q(JAKOZC6Z;La4dKaOdyCecShZ>KR1jo{^}
zx3->MPv2T#QE+jruYr~)T-dE38txwJgiN#1q5^-u6iE(D7jryh_;=novu%+CJN1ND
zeGSqy*eRKkclJg@;x(_>Uu7zxKnv?%G)7Cs<||f-fLrBaW>qT0;Ewn+{O(`&jh9Oa
zM)B2-2CeL_c%5NE#uVIRxia2ypvcg&Ey$v#Ipl(uNn7+1hYi@9N_xV(F6ir-rja86
zGt~2>DA0FUimLMFjKeV<Eo%8}NsP%ms42OiwDW=D`>^oFaTW?{D+;9pQH0M{oG|&L
z<xg%n414>5-@wMwAN+RajQbi667_n!I~MD;U85TKzYZby#ILx4{JpOHuDLte?KBUI
z4|&=*3NhQuXnZrX{-FJn9MD-$8p_-2IrqkN{twf7`>Wx08M__vC<L$ihPd*qYiwN$
z4)Os_>{($E42tJD;U1bEJu1+r_EPPniV!N4s)+z{KRsD71t@{IG}mmwZ$3<oP+8lr
zz_5L?hn7oCaha~n(kiX*vUsQCLpN(FRiuGAtfKY9r&tiL6qpzHWl7Xj5va$SA07HE
z{N-2UhF-NbXuff&J2QXvbu3>LxnQ7fGMIPe4-1;%$J0kFw^g%*<BpU5`8A>T{f<vW
z;Y^6>?t1Sa_q-#?{<-swD;zuHKhfU%teHu|LM^*W;@K$aW;aQ@(nBK8^TuOVM{Q1{
zLW>VwRMf`j56t-XT(PlxdJ@ahZbq+(yHb`1Gpe!#Ga$Dnq%mlN4Jga<8p9YVc>K<q
zlxZ4RpYK#<&~M~W*<3x|><BZHQ<}?1fo0BimbFihV+A{aOV@<qHDlkZ7PKZw)O2%<
zR<EB@znWTW3M#x4h5HIPd)GZQ<Rsib&6HX+I63**EZ7j1Rlf=ZB~VvbS2g*8lXVYG
zv%Q>CRc#__co@}~4nDEqczQHS+?Ppr)V}8}rKN6MY%wFKMuNA)jQ60AK<XJYh&hGM
z(I}G2{+xF>m|SD>m+-TM;fnMSQ;MeQ9;{qLxW_eX0SGuId5N#QcGccY!#SWkt^z}0
ztCCa0!0vNbNnmNw?^2krSk)JbkfUeA?AzifbU{4FrF?xAm#GTeYCi_HHES7Y(@+38
z!zKURejIgI*c=%(%HC*MJ>n}VY(N%e)`k~F#t8yt(yad={D&Rz&QO`id;ynz4MoPI
z@J4QW!wQjNvyB4@Kt;u1iLN~ZJ|1WK?DjeKeinYr`xk}nSlR2CbFPs@6j!KDw`0e?
zp$z5`&2RTkb2Embcjk4}vz_*4(d;+;bs14zgR`i1M-?MWd^{?)6ZAIKPbMWod=tvs
zs>`<R3(kH>L+BcX6`rsGnwc{V^BRTGw0gGUf7H<S<$jpMi9dW)y;tkiwFA>xT*9Xr
zmk`eWJYCK0$S^rJC$5v19Lt<6hx%?8v3kY6`)mU5t&po``^g~?t;t}5gI2h7A<fi-
ze$5r3_|mP&3O&9n{+%D>c>qdCpufeddZ$uLqF>8*ujGxaFCt?Jl3IJi++-CWxX{D-
z_EC8s%wnLEJS#XFogO^bzu|1_*kU!M5l-szV>y$FTr?Mxk6>>m6iT-25sR+QJ`wTH
zROS9Bs&a7){4c1Qvf0Is2Y?4XL^!Q!Gu&Ap0i5dpMb&0q0p!s&<j!nrP=AaG-<cb6
z+86Ko03&FjmSa7V|9bWG9Pq=>eR+|F>IAgN-<?BqgEF(&=ng|XeJYvha~cmxHD<`M
z<n7G}f^J7~TtTs3A_5mjzF(nm3w>5%wEB*>^RfjmXIE`Elgq4uxHs*|>M;CwUkz!=
zmhKqS2^G4CGG_3$iKT04=kRKize4p=RiH{+M0-E^2^*%4RiV|>5kfCN0=~p$lr7_M
z@!IAb{`onsDt&xg_9R4CoF*8)F1fQ|ks1OU%UXc#D)5!#9E3UWL%G1h160h)&Ht3Y
zi8)@yrRW&2msnYWPimjKpj|AP3L_#%Q$CbfU3yYZpHXABRaC9rq5TXiR8VOD7?P!4
zpT(yi7G9?KqqPUFvfMz5L*zRX9viGQE7KcGb!10E(V5+!y7v_-1Y%X^Uq?sZ$@Q2!
zzV3k^?E>1$8+^YW7ljUz4jwfObD^t7#GWl1a0}UEupxLArxrl+@U?-I-0aiceM*v*
zMe9p&GT}LHqWUB&Cf)~anGvQ4f%<biJL($OU;2soCR$R3A~vobqMEHK+M|Hz^#+od
z&a|%<?c=z5xQYR7KLER<y`78dqokM>0X23CgNw3aYXLsnx8N@(5661z!M=wCoLA@X
zJ!kI7V6(+ho<3p465Gw6#qP(B2FWNGe2@ZryQAEue2X>=Y9bu5upk-NSGzoIM2df0
zNZ&(YlalQFiARr?9!}>YoE}PfP=L=$_)P$0MtMl~WoE$mHa2kS<++{KM2>RKRT9HL
zTp+zbVrz0Ph+UT4UbQO<kcMIBUgtuC2eG)ahHonm_BVJcj@TLUsi0%_fv{GEK$wJ?
zzrfb81w9viGeP@6J5(?gxo1LdNh93qTjz6WucKr|E1&hA@<kX2)<E+^)la2C=3u6-
z$~Dh)PFVKO{{1B}T{XEoLu@;Mno8Tu=S=t<ifZZ(ZsW`g!uyzDbv5CyS<ef+Y&vq=
z68<lyO^YAI6*sXSuFCMb6_viE@l7OpXmckSPKwa5&}{9ZA*+Qv92PaI;}~oB@coWa
z66bc?3>PYB!#_-_@g$Eg9DMbCd%$XVoI%dW%Mj|ObedM-MC*U=d2pHNFq3EDVQBCD
z0Hw^YT6e-7&?#S7HV6;?K`W11bnx_19_awHoJ66%a<g@JV+_5bh)1cn|F(xzS&#kI
zI`RN}R#8I#(k`cSVXD~oG)@E5^SM1WfLP}<8}Q{eNXnLDMA(ecqpR{mI#wL5?Wg)o
zPvXlH1RX3k|Arn0`RsZv!i#A20{P;Ex6cvahO7j^XqLNJi+<1t(o2z;)3z6B^0R{C
zqAo~v46SMMN$LkX6rp_0rPA+eb;NpFle!K2hGObe)<vNy=PM%Ab#znR(^;A0p}JFO
zEWAWQ0Ht_s?4lmJ3ezTYe`wk)H5_VE3$u=yQnIdeWCj71wDRtiy7Ok~^_=B>9O$D+
zuLwf?T>PvJ*o1Ot4UK_8*+|_VCu^T(6-xA}wJB(fsHl)q@JDAyvw1wzU^8tA6U7>p
z2C1M6mSk^Fr)iHq8q?Vo@{d2cVB*Sb12<Hb#h2+UgU@kEuNSkGc4TIly$`P1;`XrT
z8&Op})i%NPxq#mVrvFS;u79K||Nn-n_g8O3`mpL0lntju@eLrV$*RQae^GVSK3h9~
zhG&@eeV@B{q!NK^aq99aCQLS)#roEcF^VXekW()UG9Wx-N6yC^c!&SzDT!St1;<2;
z<s&D`pacM0_BIm*q?R6;bG{rVcQn}QyYKWzA+eggBWtlqy)az)-5CBhalP63?Y&?9
z-uABP@a#c$Ws+^@`O{KXMW88QQV-G4nbO~ap4IgA=uiXF`jU!HyJ_#eq$7%~_Bs#u
z2|{kY_DodbVJ}zatUL8g)sr_oiC&`5%Ru@WNYSDz!u72_Y&4qD8NBU~u5z%2Fb8q{
zE~;Jg-H2XouM#a=Ztjj=Mdt)rS?|?Sa9rnwML!)hNVEO@=;vXFl{1ufoo<xb_xh`U
z1~_nW#;LbXt8PmhMFv%zw5sn(YHK|{4Kki_sQZS3fX>dTxG=Q?rIYwrA^+5`eAalr
zRKZw??yCl|lFx^IM|scwp~c#MEia=zu*<4wUAAe(@`jqjN_*q82K@mdD5P#stDR{k
z_TZ_WwgSCpQB3v*Jq>!Mo*iA<xb!*RUa|`9R$l_d6+RA99Q1NrjnnF16>|H*n=Cb+
zSD~w}bS6X3ri7xz)&w-9?FSbosWWX@i3o7r5#Yd;Z=u;#-+h-O=y`ErBx~mktvzat
zR>V$1icjp!c-}D7uW?olP~AyIxpJF{U^VqL*&MuJF2-Q0m9;B|=fdn64o)dV!_l`H
zh@}(7Ph&&lT3&H%CB1e!(m|aURbe<ni!H0)i7o{$W57IFdXVi8&?-qJMg5vdhkGlS
zW*;9BV{aTn4cR326rF$HYH1YoDbX}5eY1vcD#+Sloy0pFzaki_o=E@<yI%IdAwv?x
zJq292-g*QcwVo)b;*9ZijxaCn0CvnI%pk~D+zCY{059(Hhu;SnHIoQi{zkB~PcDe@
zdL$jofnUc8S+wV{JK3Db&KdWmvJI){luxWck14m4PgltW9_*Ak`%ap+K8MrZl*s7Y
z)>iSi`tg}KeZHbya>})q3$vD`N!Fd)1%p6;L)`Pm<v~5WCqlvrX}TLTl=3`;tIC(+
z<baYdzY6sGLJY6$zF+0^cZyVUlI1oo<OKr4($BKpV`;A0ifbRF&G1QL3K#}HFM%Ij
z`0)qa<xzliC8>7x0LnhUF<+ZhK}!4T7N7df661wHt7uH`wC^dT8H@B(v}*X83^rGv
zoyn8w69)NWTtW>H+igShz3>7HX(@~h`x&~;@-*SX7hGc*>PQv0t8Kk2FV;jRwf0mm
zlJXT+198z=>`U({pF6A$c0G*}+u>E1Ga^YHpSzauuFPH3k5y%=6eH|$rROo!#8?Yo
z$&R$YWx%B74Y=^#B}^ETi7sd4aaxYqzt6x}X;x$!A&i2vdALut;uxfP+#Jl|1iT*@
z)>YRp4@}0Y>RUm+`_41ne0NMO-zQwO^;7Cvi_$_=`H<l$E4al_BiiU-?<>n}PgNZ%
z++|Dxt#lS;culQ|-N-QJ<uga-m%6h9Ox|yrlGKydjAmy%^<&Vo0F&h`NjS2k6>L#I
zR^8Bo^4KF-hC_?ih?Z8@yq=2K`2rw7%&J^~s7n&B#%tptnyo6$wI|rKe?m_MR?8%`
zQGiW+iFpjcI*1(zdz5LG6uAnda5nj8s&f7lRk^uAKrq_uzhVL;L<ktI|F6^y2MhDJ
zwEVjR`yYt_;(w<cKiMnrd<46)W`4)^I59x4<HX1D{)e(bV6>n{+QZ$*j<YX)HV!*c
z&b!>kt^@>@?g3}z<2igPJ?l(wt$YiGo6^?N!t-NUH$bHs>Iwlf_iAS&B}tXTsF;z-
z#qKXYtWz)m5&N;<v%&&P7n4D4MnU_#W!{W>w>raZvRI@_A#8CuSuP3}f{HFnnO*L;
zre{Spk$Un(?D_X2^7N_w>Ju_Koxa-lrN@A}jW;j6PW`zB1@4@QLIK*m+|D4|WsyG?
zUkPq?NEDmNl_e^Av{!+x-X^%{#Vbykh|V%8@B3JEjG5VnaMIRaW~!#;gz*b>1Uur@
zleQS4o{!$pzkhQtv^V+qHI6NZ@e0BX_7dk`>XQ@91-e_-=_crUo?An4C+>{A@0%za
z;Nbh}X6Lpq^YU1h=WDHAy-qXFc5O<=y6h3C_KORnH^Rv1>+f<rNeu+D0C3+wZoY+x
zm@;XrP0%ca2hm2bH}H-98S2g2;(kXa*9Ku{i#hDL#^{riB+B=ibDf>ylJ$mJ^g`6Y
znOqWDEo6?Z<MOXtFuA%OnctWVI%B&m<KqTR6&;mErE!M@>!YISY@Nr6+bTXt^{Dm4
zcWJh*=Z3*tYjPx_&y^mv)ZV?A%khmRE|_VZ&D-?lO(|~VE%&ThT2MpzfE(u72ciM0
z*^RphtmHT!u)=JZ=PP<_hvJmR)PGZ!4H<3RZswPeuFEs<;$=GbUc$|(2&N#Q*i0%C
zL}Iw$U$^6;?@Pf+KBV#1%?nbIWfIsN@A!klx<nI#1E8}`yZArzP1O98x(VWg_Z9Cp
z0K-GQ?-~s3YtyP2$#nI4*<!qG=RPdeZ%dzTT>sGu^l)dmLl?clc!%UN%WE=-Gl#F2
zwe2Di)GCfB>u%L+-)FWDl87aqybS*4n?2W76eZvu>xi9vN{fQS++WV?K8RsWgSNQi
z(~&6T*v7y`@Z|dB^c{w-<qP^Pz)hMd>Y!VAXTh{&&x5NjwKz(Qbh8Q`cF6y`7D9vE
zBbnCx=T3r5c1iD*Ejr3iD^qIfHxzjE1GmV^tJ?!9{Q5jZ39BhzRlUfkR3JwQZfq$K
zHFf&0De~=+6&MWqU9bgc=7p3CZZ#3@gw1q6brUW@IX?Npem_2I9{_Lfkk)a}zR=Ky
zN=4bkdr&^cj6O586}IF7VfJT-YW6$7ot<-klTG`r<%YKs5`Vn@+V4qMe^}!`bJ(os
zBAJV7WKs)1=geYDpr2qzl+D@KHv5xJh?_o%gn8F}>_ms=Gnsb%{%8x|OB;Ktu1&Jr
zlKb%C<Xw`iVH=qe+ag7Kt?f6+YrF&D$8+n~X#|6K3X?sb?rsYEJ=RwRYpu#m6CW%6
z@^t0jT*D-t{5eZ4qZH62?EI6l<3YPfg5(2|ltbNyHr^{Om4ptWO9z8#GLcw4MP@-t
z@5>JTbCdRAy~-u)nC9FTncnA2M8DsN^nu)6Bu!UcBr70+bfny^0(W(0x=H6d=U#$i
z(S;7el^c?0XIaFy<nCb~Y@CQsSgn-9cbNUk4Dx*+WQ&($dl*EU;uBAdf<v4GZKOL0
zh+$+C(NhG?uqS!yuh4XRZ-`GWmY3Oe_J8K~+pW&;sC{okPf$gOZVM0(C6c#X{FNoh
zKA|9w?Fqrf=<2%I&4U1ts`&OVuRhZ$01pN9lxcBDRT5Kf26?7@hNW9>9E~spBrRKV
zVh?PySqG843dH*;YAVL!wwBf&zs=2ziH3V{U5nZCh2OtVAb`_ZfPBSwI+Qs?jjf}C
z@KbxQ&D}aOcv)_*i6-yQ3QJ;OX~n8K65@YkCF6?Vv2dLA_@q@;&iMLk6PAJleMke4
zDiA3aMrU^~E5kYSS)+1gkBn=LbD{rK6!#I`YyI1r#~v@KA<}a(<ej4!CNsQF+JG4I
zv^n$U4>uA^5}O0r;pAwKV1^DSlA*{GIMCHjQy8~vAFG10HsreQ`a8RJzIaz*8Av)Z
zg}SmvCl@{e@d+gLdXaBDXuKz7uDDcU9Zg98(z|_pF~uElOv?VFpl3mq4zC>7cMe4S
z=6raZPuu>B)J(lj8=aKD8Ww<luX0|h<NEyi3Oj*U3%0v<mt_&70%t>YO(fsn?;~)+
ztNg^;;D54Z#~y*P3@DPuFI}>0*^ex`XLbFso2Pk}L|sb9FK`Ty61ULiNd}lYTJ-p%
zVUOV%uEZ!&O|!LE+)=By2Q5je*FM;}xY;J{J%oK}TCOV3Cv9s-ait<(>@C3E-(eHS
zh?OwNM8w=&rQFd{{L<p{!t8`zvij9zk_^=3=ML+-=NZW9U>e>=el0+Grd7I#btZ_)
zc&22O<Ng}JgpiGqD7nho_kEXafXBJZhiuAt1R%(*P~g0cheByYYJFI9>fH9`uSj^w
z<nF26eX{V-(G<#H)ErG4JkcE|jauA@J<qz$=$iCp8Jm2>=2CIR`z~+Xu|tN^TIYF~
z=(nJij|*~2&U=Mgfj9DU7Z6VYi;04R<nKhG+d4~vWtDd}yN|<1>R-_N{XVcIQnIRJ
z;o-=;eHzpL^iYL4^x%nw3c0lVP}3T;ws4uR)>1J7v;r)N#U1%bd=Gw)$YhFtZgiLp
zp%sr@shC8U8YVSjff^P}eG6sfOm~A7I#BY!vubwC3-lEmzWUa@l^DG9M9;*-(2X&t
zG-*?XqPF8^bMJ*8s^KAktMp!zZQM?9Qm24@r}?6o-RIj^UBbz8X3QKG0so1aRK9lY
zG=WpU_A^cJePL4Xi&<JYma|0__L#gM10HhA7z>AF@@-;q`AAHOB?iO7Z+Hb@A%;AC
zid2<nN1W=-k!huGx8K-(O^PyC4H1vGgQsv#_QNkem5>J&a6jByl|z$j*kqM^HAO%|
z$T#c!bhQQt{@0TZW%EDR9?xm_USPb$;tul?OJR83OF<QWy$3mcLdze?Ula{+P~9So
z-KVg#F{Fu*N}nlQk>x+HIE}nzu2mqm(GWd1Y6b9w1>aoUdcxrD5m`SS2OU8<!BEF%
zhcZj)3;wKQtcuV$k3qqY{Ao;68~YP3{aqYSpvI#?xjxZo7e&C&dWuQ&STD5~9pW>n
zUg$=;%JlP5kflNafv?&%g_Ik%v<Kh1yiPE8R9SkrRYb5uy4lF2T5R=*0oH6z%M#b7
zO@n96W}o5NjQU9$OmlzE?Y3(F!@;p(b7+|THD)%Fl`5<7T!1*1n`H~Km2_k->(&6`
z>Sn(D;si_6;;akQl%xCv_F{<a_sq3nrdpLiRbl51q7T)BDyjttnXvb09<UIidz#8b
z!)F^6H(65NN6CoDhD)Lw-B<W<6v62G4ESYEd<iZWmECI=@yZlBEr|ytTHr8!guKf=
zC3(H-k(VuVhn0jud-Rc;K*}|7pta<TciN@kS6h92Bs@2RabC#T<W#&!PCO&h$`kAL
zdns7Toj5XB;@g|%*f||Vw|~U!s!rwGYKH6-Pybn&fOriuu0b!?xuf|veeqo>+6x6L
z)+sthJbqgl=g^VbFMDX*PWOUDnvF8$LLEagGK38QS1Ht*>^2Fui;M%rWA}g99k*Wc
zwi>J9S#qq8`8gJdilsUW=k}!S&Wy7x8Ww46AB$(vdmZ=<pH=%l#>h>~ko;gK=%=0u
zrd<NuA5=-&7vrZ^@SuzP&aQ1Q8sR8gGe)KOgCYo~EGu!%Lx1UugmU!0e~N30#0&L5
zdOOA02bN%kB(q~#G}O~PG!>%Wlc3CNY2ps_Z{OIb<8eY*MCQ%fBx*mZxIwZ@2w!PD
zTrI)yzTC;bo}(E!Dxy$Lb!xjd#L3<6<WfQ!U4IPF<+@mge|CPrE;5cl(2x0>q60sX
z9CDqB6@mmu;V?bsb|jQE|E8c6XFiP`N?cG2-TSrbhTGny94Y*%$BBVVA~FUO%KQx)
zdgnZw3`w(D;xwZ5_ZlC_>NQJ?g~@@UKx!ZgA_6NW0PY<E3_Lyw;^f3$qTW`}<vFNI
m|I;BA6pI`l-|vqJy`insU0La!PVQy0#Q`5U5e@(w4*4(U;r<N(

literal 0
HcmV?d00001

diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing.p12 b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing.p12
new file mode 100644
index 0000000000000000000000000000000000000000..5e8f06d3d616a08495b6e2a486386303beac37c2
GIT binary patch
literal 2419
zcmY+EXH?S(7RA#cNRduLks=5N@J|Uy1Xg;JDn&Yjv{0ge(m}~#kY+$B5|JiKm!YGh
z^j@Tf3}6_VAkuq4Da-EJojLR2y?f5@-uFIUfQY;Y0Yd>IQWAl@8mk{mX8|*S^N2_k
zjELku$Ls(R?)MK1?gS&k?as0F`B)$>{O^jH2@J_2!hQlo7#)y8F#R8&Kj(*WbvFyO
znW5u?QXqHP@~3w-o*iPYLZF~_GZ+!pldTjcb=M}*Y3BRcbdS*&_;tseObOK(Wy9Js
zS$z|4vVuYJt8jnCzGw16)?M`yWI^Y|$;8<W<?xAaO9w9hXf21+JujmkyR$Fp*0dOn
zE6)e|T1~3h<P%gs1(jcJR`Ubz>+J>SR84nlTYAPY^!k)OKXOv|#alz&r1QhEVMo>)
zA_+v0vZ~qwk{~iO8-B4{uz9(q6QRdx;03+8(mt1Z`Ma8SgC&Oe*^tvBHXxWBU6MKK
z3mfg|Vs^+)>)uHOVt#VN%GQ#F!u(vRR{3V@`<0%TW~+5;(>Fhq8?#0?d0(hQq_WQA
zhrXJo-bU@^LyWieKK)>zu<fic*7L&FGGFGUip5*7W(p5q)M7lc+<Er^EEuT&4vl_v
zqWpsXQNDUqjgUq5Qt|D3?@r24Y8-xq-qhyXJRz*)5V|BA#r6{ENjXE+i-Clz3;6Ha
zg`;2NpgrNWH#0-{6l@NT!lq~fHjAaF=7+m3H>73ZimZT?C1=X^*l|{D7M|>zVau^L
zBW1m}9lmE#u`pP)$blfOY!I4J5e?8#2Ky5#(wC_rAlhm(sjSDmYk<eA01r4MB!0jO
zH{ARU&azZZBPT<TdJF7mq;cLBP*=-j`gFWDuGU(z{eG^Ed4i`u>Ic3jw@tz$!d6^{
z&X^<^5TIG8RLYDXSt!LVRbhwg^x_M2C5AEdA#1ax^iLVf=^bXpZGj(+D@jmNf>Thl
z&T^cokIpq^aF4iw_SPBo;SN8)G29}#=u{`N<+MjqgOVA?bp_lOW7#F7v%%_f7VH?g
z8YNYWFwwbm5br}FjM9_^LF+Cxe&C1ZP4~j+qH1C#4m6)vIFw^*?vPlMz2OxPfeyvp
z(ky&FVYcx>M^r@AGS+mYA8NvSXcX3p{6=`{BSv1CY^Y7wZ2#gF?1$p~o;5+|c`ZJ;
z4xBzcrZ06CK3<WnAO998F7cvUH*eg2-FbaN;C_wGy1pa#%?*MT`AFKiU0AcFZ|Y+f
zN4C<V46xVKsi%%@q?ra^=aM{UH5zxlbJ5Rqs}(6=Q7_$n^pISxJ7DusmW3txNxRt?
zE_x%wQs8n?G`10ku|<W)@JMXQHXN609O)&;<y<vPPa1WhR@U-N=}fuQTyc72N<>v7
z{}qQdAUk?0n+`ZOf4&xI5xz24rtz4SH-{pkoEKq9amUBrc{UJQg|_mUyoLf_G*g<l
zNDYf$MW_YqsYoV6c03grTSDK}3nOpt+)URGo#v}9=MGY?i`SuO%0pOXDRI5)m)!)b
zFbh$knhs)eM@orJ{7172E{2|q+ujD;ZR5cOnNu60CLXIN<D=}Tg3_qIhC^{~YHjLE
z_KKiI+sc`Q$7IOYf#>b~yP=<gtuhjE-`KuhHtV$nL}p*UKS<f7Bs>$}j1j#PC(hF0
z;H&!Q%3RgX92<<ZVw+Cekoffs^SsN&Ly}`~$mllY)$#Y=Gt%FTs&1zlPRM69jqoV?
zhOo+~-mmp?QvZpmDGKTTEV%rmNpATzC5?AAqT>>~xCR;${X&L4ZE)0WS=BDA*W&mC
zSdVtQxlc4~_`z1IxTrJRSim<>fGd2qtfx}if@iNK0A%WBjgt)7m?kZeI{n2Z<+OoI
ze^n)tOO8Y6A}A1q2YG@B=bLb@a)8L+0d6@i7>6a^&s|s!dmT_v0RTCyva$+5guePK
z#DK^nLZ6<acnBDDF2uhb@IP7>@t>Bx20VxIHe<rGFO0?6XAd5x6sgny+p>>|P<_`+
zAg<=2`%W)>1B^<R^RDqsc!(^R$?O~S(DYB;jFZdfYi9+_Nw|?rMSne#SWrELw`BOK
z$FEjj2<)ShBagNC-VPDd-<C!6)4Phn3@HK9jFcx;(<4@IsDJ1=K4(xn^>k(|3VyJe
zdZjRJHMD*F7j1U#iZ!&Pt!<ESV55F(cnIk5%ysu-FPgRbWRc5YUu%^YteJiIvSNxt
zQ^c7>XuUZoRK=MDhDrgIO(SC^Y_5u%F}d&Q#qK6{8Ll<89sGp_Z6hMK?vvw1h721^
zQ?J3D{cfLQI(hF=Fadg;xC*1LxTujHU`QL>p~{rw`S8b5!2pv$)^+QI4qNwx<f}JM
z_G<7l8rKIVRu^tT8pE{a5UFR<*-UO+hHlg<#FOyKYneW(uCJm#o;1$$uyK}P>^O8d
zIlCTRy)p268;_H;K30?}^L2QULuTHDLy9{*WWPCA?g=0d`sui_dxMnh%SW8>P3M<z
zRf$wR_Nd12yVvE&k?<KmNlrr9@RU2SR{{Me)kkB>ZK^N>ejOgh?(83x<y4fMGAx{#
zDzxEDn9oo!<W_$Vw#>p~n4Cmc^X+%*U07Npfp5FUV#TIeY?4x!VV&l;Mvv7)(7X;`
z?JD-7`}OKHi!Tg;{QHt3p@-iMXzkJuKjt&ziN_kC*#NJjN3oQ&3l$q=itfdV#0b~d
zhEqYy8l~{MKo5~XeH_i4S3BRlUd>|Z@F)N0&f<g?`*jAV`VIy$hv4`TZP9+t@et2F
z^y=>(@tA`zvML9h%xZf^vO+@dQ<<A5)*aVGEia+y>@!L3?psgNL?~(w2T~tGxq}5g
zH<jJ<OB%lcSQop=atn;tv3JXKz*q}ft@9INLA~3kNy2RQ<!eI;&h37&eYc40{9>Y8
zhB%87IBBwtV_^B9u`U$xY)<3Ht-^*+@BM3ytABf#I|pHm#qO(UHZ8y!g1@WAHkAoU
zSW47kNu2QZVViC5Lm{id<+%GCbPv?qTQB{rWbc^h)^4%=Nd)}9NLyJm;6Qv1K`B@=
zl}qflRv*sV8>{<Pj1y&gPkm4CUDXVj%^tK8zh*k#dm|i4N|D~2&v&9bV}zj}3OZJA
zF}a!|0d+tGKqBB6Rwx7w2f?^R{3Aj`i|#5+lWIgOquBrOqv>wLxK?jbr!lRd$m|5!
SnMOB%w{8=vJOm1k&HFbeab7$C

literal 0
HcmV?d00001

diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.crt b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.crt
new file mode 100644
index 0000000000000..fe8e062bb7d6a
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.crt
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyTCCAbGgAwIBAgIEB3RBlDANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwpz
+YW1sMS10ZXN0MB4XDTIwMDEzMDEzMDYxM1oXDTIwMDQyOTEzMDYxM1owFTETMBEG
+A1UEAxMKc2FtbDEtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AN5KCDf5R6ZFhcehDH0o6KMmAq/bVLczjNUcclxb0jFj9qfT1Lp7lYCMlbY0sskq
+2LncBIriihaC91OVBWWU1Z6fANxGn83ZPb+wM0h+iSN2IeHPjOP87vd1/oriSJEH
+U6MtzVKi5BIDoLx9tMhykSkWuQu05XQvMM6ofUyYtLZt38yUBaG/auO68FN6xtz/
+1RadENMVTfbGhUEPmSv+F/XsMW/MnhW0rXMOoWdmL9pv4vuGI7YUipw3xE30MxYA
+rEFfedmJFWhvonVoBg43pIa8cJLr8QX+G+Zgy1g0F8oLndreB0KbnsC8aqQ0z3qc
+iIbQMehPTN8cYgujTpGXS7MCAwEAAaMhMB8wHQYDVR0OBBYEFLRsbY+bypAXcmC4
+JzhEDJoidIgVMA0GCSqGSIb3DQEBCwUAA4IBAQC8rd3TQ/MR+SnK0mdBjhkJdZWl
+NiDkAuK1hRmreH3hYYBpVg3M51CcEAJIMsCH4RkJY66vjOWnsuPvF2Yxr0/dFI0W
+tBg01kZEodtxKvl8klJq3SHg2DqVJsJ8Sfl+lj+MBMvzTfWT4YjHnxjslPu39K5P
+tULjaO0pLHckkSSXH+JksUiYQ9enRp5aN/MbUvvHT4R5wd+SeQsThHKhrB3WZk3M
+Pn8vmGhsfdUjJNBMEbMMGRJEZmIs/Zdrg0pcGgk+tB0YqYGuiNLwtR0G0AVxeEHI
+Qf8IotlIg8mAKx9uHovTH8e3jyTDi+zBO0r3oSwiq43b1MOx88bSOtvATJbm
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.key b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.key
new file mode 100644
index 0000000000000..29908feeee4ce
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing1.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDeSgg3+UemRYXH
+oQx9KOijJgKv21S3M4zVHHJcW9IxY/an09S6e5WAjJW2NLLJKti53ASK4ooWgvdT
+lQVllNWenwDcRp/N2T2/sDNIfokjdiHhz4zj/O73df6K4kiRB1OjLc1SouQSA6C8
+fbTIcpEpFrkLtOV0LzDOqH1MmLS2bd/MlAWhv2rjuvBTesbc/9UWnRDTFU32xoVB
+D5kr/hf17DFvzJ4VtK1zDqFnZi/ab+L7hiO2FIqcN8RN9DMWAKxBX3nZiRVob6J1
+aAYON6SGvHCS6/EF/hvmYMtYNBfKC53a3gdCm57AvGqkNM96nIiG0DHoT0zfHGIL
+o06Rl0uzAgMBAAECggEAV/2KH04HtKOUfx8a6HhHDj2/lokFkbdQKhOmmTSnkXF2
+c5yAeojs9wOvR364vrS9MWUMT0W5T4vsByPYDQvLO4zJNGioXT1/Gd/PjQS5mFXs
+t4wGw0Cd/0qvxZ4pB2+VBeTdf7aG/TJ+b956r1XabB9YhG/I3CLJ+SPs5WMVNZiv
+kmHldz/XWaa0zIIodk3eZYUM6B45DpVEE+oU2KpL+as4MiO5EhO5razMZARUwUH/
+D5dx6pWBMrkVtITEOJlUbIrjkPoN2uIVL+DjIXgI4t0pEVI8tP2M18xcb1s9C/2v
+wWkZkowaGXYADlvWtWRZSxRv8RAaIt7f+K9UkexRMQKBgQD5PfW5rI+fZr1xlto5
+NP0F8S5nM7Giw+L5DL5hPRhFX3TJR+mloItoIn2wBt5p7viJev1t+eFEITTQ/gnT
+3qj/EUFfjl3mCPCv/bGVFyRuKG9z5921BWEhyxQscQW47nahaeTfsVkqjBKuPQym
+FIa4d+TlIITdhQgwvKKDwmS+vQKBgQDkUPytZ4u9lEJAhBK37U1Oke2pZNwlwON7
+YJqhUV7rHtAffjHECjjOQFaU9qhUmQo0XMRMeyKjxZyxOBbPlNaDCEvj0pppK+fd
+Y70fNraWFKn8Oq8LtxfCJAzjySA8P3DKyyjtMNnONSzZZGAGVburjTv94SbKB933
+9YuQ1NVTLwKBgGqS0tp07eiz7rd+UIt7m+GJklTwprk5V3k0uMctb45iVbaneEGp
+F1MMxH+bQEk9tKpcm7cFzQvrwwDGT393yv1d7H0mjAILupcWPHPKtHoh8h42e/em
+eDUR9+tU3KSJ2tIETKhV1hG6WVaFpj3xsvV31rkeoZFzC+CDGF6MpbVlAoGBAMZD
+6dhvdbMQXyQFTPf5k0tqf7a9BkumoCSQtFboAYQtws/GV1MFCWVaC5XaSD0GlQz5
+jeFYgLE1NqfSrERg3JL3Dmfth/dSkEv5g87qRU447ZBCKaYDvhbyFVttWZfYA4un
+NogbBtkB5kt11clKg+kqgXErJvMEN+ZOjX8cw48zAoGAcW4ntY69UNo2XEm+bRTK
+Bde4WrlnLVztNckZOj6YbhqYzIkXIm9HRl7oOjPnK/M2D8f2UB8H9J8PEyNrFuis
+6MR0eoc4VXuMmi90VYL3yXjUiPYVKjEe1VcA6wDVwudK0v3okVSRjGIp0cJt2OIS
+5E+FvmJ79EQcNbRs/fGVyec=
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.crt b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.crt
new file mode 100644
index 0000000000000..425bb42a2f487
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.crt
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyTCCAbGgAwIBAgIERQXVrDANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwpz
+YW1sMi10ZXN0MB4XDTIwMDEzMDEzMDYzMloXDTIwMDQyOTEzMDYzMlowFTETMBEG
+A1UEAxMKc2FtbDItdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AK8mo/SqI+RAbq1Q7qDTddnjdqIMcmnCpHcSID2fhgoNrikIT8voWSthgpCpR7Zp
+CbVj1LK/RHtF7x9/HnrZr4Us07IESuK89NNKZYA6V5qiK1noctIGa2KIIDp0uVdL
+Uv4/UaK6sUe0tex1davFivqR40uHemO0OZb890YtdBft0c0qBoS7eU4fqcwT7Gbv
+gxMb7CrNqAsxT7Ax4gYGjWG0BYp+FicmB6A9aAJWIjac9YfABf0DDesazUAYpoAW
+FRCVKyGVa+U7HTq4wITGts6OKAbAdaJpbshxk9/TU91SfWqeqG0PyImGbQkbXr2c
+cAoylImud3EMUwkWgWfzsBECAwEAAaMhMB8wHQYDVR0OBBYEFPuTo5I6pHRzUZJX
+6mbznuriQbpWMA0GCSqGSIb3DQEBCwUAA4IBAQCdWs1cZvttuUJ1GnWjuwY+cJJR
+vNNl7D5Ge4zsyQeTzd8x0V4Pl5JVIyVxEP9Hu4+bgOo/daJtBT1y9B4eRfgtPGYO
+/3XP1OXJ7zpIIqEUozaCgfLo6L5Qk9DUfb+ikxVNagsC5DvlB+nKblqH3zbrEAPo
+tq7hFeow+PTr799U1flRkr9w/+FVJVZIvgKrb0E0BauPyR1+MHOr/XY50Mi6tH+/
+3F8SYRufLvdMSLQ9ZrBBZoQjcrfO1qb7GS364hbpu7TQIsXOHnfsAQsoGBBUrVla
+IATAHbog6SxpKT729dcrMVrJ/1inJ8NZuOZpIgJozi1tiuJ56QdOyuji59iU
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.key b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.key
new file mode 100644
index 0000000000000..8419d4052068e
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing2.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCvJqP0qiPkQG6t
+UO6g03XZ43aiDHJpwqR3EiA9n4YKDa4pCE/L6FkrYYKQqUe2aQm1Y9Syv0R7Re8f
+fx562a+FLNOyBErivPTTSmWAOleaoitZ6HLSBmtiiCA6dLlXS1L+P1GiurFHtLXs
+dXWrxYr6keNLh3pjtDmW/PdGLXQX7dHNKgaEu3lOH6nME+xm74MTG+wqzagLMU+w
+MeIGBo1htAWKfhYnJgegPWgCViI2nPWHwAX9Aw3rGs1AGKaAFhUQlSshlWvlOx06
+uMCExrbOjigGwHWiaW7IcZPf01PdUn1qnqhtD8iJhm0JG169nHAKMpSJrndxDFMJ
+FoFn87ARAgMBAAECggEAQjfAmrpMUXoOndXQMdtgXd0CN+euxc9BsIaYy9NosJpV
+csvipzuTorP90cU+CHeTpRE2E9H+CQD6k5TSqC1uj1HQQygeS7SWdt9UmnL0NdED
+xLz90+t7Y8gFLzR78cXXfangqj4c0fg/I0ORFjzb+zl2jlKM8e7sKuo+bji3AGhX
+1Hl6F2yZmdsp+k3qLAiELc0ExBXq4M7SOCOzKE3BnMRtRE5QXieF39+l7g/YtSru
+OZI+gdA101ONNxiQa/w0m39wqaZgWex0dHjSztlg3Ja9ZralH1op/Zhhh2txLMf1
+CTlKZOEa/rFMw4YGqVfSX460/W+gXC3h+VBogHY9VQKBgQDZKpV0fVisV8pEbZaj
+kLtk1TGXX4gucgt9EOPqjuS4xF8RGGbS+Mk3DQgXWGRU7OE3Pd+SrQ+mqoesHtJK
+4Md+cqKx1F0E8Gg2Dr+jWsmbmAWsOxk6cLZJPNTpIRxtBcjzUgawJoVnimz7L38p
+4++VqpXZcG5CVvOKQAie7QiIwwKBgQDOeK4loplQF+UWSs7HmkJ0/MeELR7ICvhP
+LUo70qppiXZzjgCbfDoT5TLriR8fA+mP+YIjPMmdC9iGY4mK6CxaDYtbcNW64aOW
+j50VG7p3grS52yvzFgRX9V4/xJARO4UuYCab+gFLBaWd0LIZL/GbLZVD77Llg+qT
+yifkhFV2mwKBgEEAHfvgCMqM/F2+73fFulFT/qPwVDKSsB2fEJkEbDFhlZLoWt6K
+2K9Bl0pliqJgxIhkXKbqZeTc5f7+urLu1MdkSI4xoiX4Tzje8T1Tz0RjHE+Q4Nw9
+IFKLgq7E2l1Hff5WbuoKnaOAK16DuOww7WU+ydtra5UvsXWP9QScrhFBAoGAbjs1
+1nf53C31q7IitaQg6O5v4pxoO6rHqRXn9Qpi2td9+vGMV3a6Vxbd2awqhleDYc92
+8DMq/IJvhkgsDZIo46cpiJc0NIL04JQp7DQSmG2a6UMfMrfFd+RS/j6GH/0j/H9u
+/YU7HX+VTS03eIUFO7DZaRzW6PxFx8N9Ct6v9hMCgYABNFC7oPj24NwmJHZPXWuN
+V+ZRG2dC2E9K+MVg2Az1r+EyXCReBrzj0hYoZ2zS3SBGDD52ktIl6ekWCzEoC0VH
+uyg/237P1/Vs1IfuneI2/zq4R6sjeNmSEFsjhMqu1K1nCM4ZiWl6SzZrwo12bFCL
+93Lv9ufTBtMUgg154OM2xg==
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.crt b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.crt
new file mode 100644
index 0000000000000..3dbc9ae0486c6
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.crt
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyTCCAbGgAwIBAgIEC9b3+jANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwpz
+YW1sMy10ZXN0MB4XDTIwMDEzMDEzMDY0MloXDTIwMDQyOTEzMDY0MlowFTETMBEG
+A1UEAxMKc2FtbDMtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AKHUOCpcF2LYVujyV1OQEOteDFs+vysWfbPu+gdmmQtW4wbK2ZMx7dppbWz9uC3f
+asyOqsxmhgGy9fkP/NfmKPPzN9gx+PzaGnu2oqvpAQMndBdIs4pmHn8ZlhNaHJvs
+1cTJfEMacQPNwFxjxkavl4dfnYDKRdxD0RFpfdHlFyM48P7emJqEUASx419/cRSh
+UoP6WUL1SInGkUDP0Zu8NR46Zk8XboeCKvTxPyOzMsDBZP726upAWQmmtYGpifra
+ScyDp/YkeDn6gWuSkThfGZI7x4E/H2/qlwJXeU9vIonauZYvIFGpjWD0Xw8iojSO
+2nkPutfvOOeb2Dvbxz1ANJsCAwEAAaMhMB8wHQYDVR0OBBYEFLlybUTvk1Yh5L55
+xhRWVN+i0D+/MA0GCSqGSIb3DQEBCwUAA4IBAQAv9l5udI0iSQmLPnRf876lczZG
+dylwrMv1zduThOTQ2hok1vf+D2Bi1U8mFl/T2WIM+2JlELi9yJDcFKh85AKvkRxG
+LY4CfKiOXEBpOrVETpNA2oEdh4HHsokKSt6mMe5GbUN+YdDYHNkmgTg01O7SIL4C
+9Et0GJlZ59uXTC9Z6RK27qdGm3zvCWQezOdE88Q1hWr0vjlPEDk9bQFiLJvCkAEo
+1vKjT3/NWJBK1mG7Z14YxFe1h28bHOCGruW84+sL4WLzJsSRXzEIJBJXpi0COdSg
+LrTkx482oen6U/IaZ6IDuQDhex/AhLhjsDsr/cJ2zkfD2d+P+ZQm2XIa96RE
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.key b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.key
new file mode 100644
index 0000000000000..74cdc8cf9dd30
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing3.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCh1DgqXBdi2Fbo
+8ldTkBDrXgxbPr8rFn2z7voHZpkLVuMGytmTMe3aaW1s/bgt32rMjqrMZoYBsvX5
+D/zX5ijz8zfYMfj82hp7tqKr6QEDJ3QXSLOKZh5/GZYTWhyb7NXEyXxDGnEDzcBc
+Y8ZGr5eHX52AykXcQ9ERaX3R5RcjOPD+3piahFAEseNff3EUoVKD+llC9UiJxpFA
+z9GbvDUeOmZPF26Hgir08T8jszLAwWT+9urqQFkJprWBqYn62knMg6f2JHg5+oFr
+kpE4XxmSO8eBPx9v6pcCV3lPbyKJ2rmWLyBRqY1g9F8PIqI0jtp5D7rX7zjnm9g7
+28c9QDSbAgMBAAECggEAG43+DkBl/v+lNL7m3aSfLORzqCmz9Us8FiIv0ojGsGaX
++3UYvPZ4cIMZDcDZ1n2/OptkO6+NS8bEGSd+vuhCut8D+DzyD2O3tK7xXeIruPTD
+0J8hawReVvftVNg4HNijwYEpNmfPLH2hLloSzcBiRbujHuXSJWIqvBNBSDcAzI9E
+GjxNHNUT39OiRrwkWFWEP+GdytC1QN34/FVY3IHuqytIMAu49b7ejTDi5PWKjHdt
+7XjIwaFDQfxFihB3UxvifSZIosvc7nsQ6AZdIewmTHcmj3yRZoOWlh1UayeaCAu7
+6funIIpK0WqL6yaqDtB0X+Fc5nGiyxpNtO5ZytdPcQKBgQDy+QnUboC7LRQwiM1g
+Wk2pXzXMQ68Bt50Ta6a4klOnNfeTcklfmOgmuneCdXfEIQUER9Jy0e2jappZDVY/
+S7XEHfKZAld5b9j71xUr/J+RxgD20EVWALM07zBiHiinPOcAKpuWY0y0uVWumO5o
+blNtwlGAIwZLyjLcb+0RLDY/CQKBgQCqgW26zxM75tZWff7e0enXxhfEBeSdXVc3
+cTirnGWdI0ooGq3fLkpEL9gACg40GdoZtwLsiN8uN5JZJKaVAA0byO5jSDZwXPol
+XOW0guex/s37XsB1ps2Nq3XzL85+y7UG8dxUFXe5n0LNa51iDEhWEDJRAofD32vD
+iiLRHBgbgwKBgQDys8EV6au9TLI2KPC9pnadEPX0LEfMN1U36PiIteHJzSaYqjiQ
+vXRn+ysVrbEMwM76D6Qv/XKbIgk7mw7j+wR9+kwyexxVuVvNdHM5lQbv3hFoeXJJ
+Djn0TljOKcICC9vTVrxD5LziaZ8k4K6Esm+z2779d/76Bef4OfVcigVFwQKBgBMV
+T1m0KXvv1xdxHKCFr/11b12CJ1cxhjxvUJ9sbrcKpxcb/U/YV3mQPUWr2nEcX5u3
+Lv2YSBpdH+UYRAizK8RrVbsI7wbecSTTY+YzLIlwck9rCHFijOkJQrdIJ31rgURY
+t4c1q+O7nndtTv0VUNdJEsicVuOmUuJcyI7M7SnPAoGBAJhlAT+xRSolWcvNgdVH
+g20Wvu6o0uHpYNsp75Ogk2uwufK+lXQx5zPG7NBAd8s50nsAzvuONVLJvNwu9gg1
+xQ+ytfhazZgfUrUXhtLu2qOxN8bRGk8aKve8UQ9sRsMKKGvoPEuHVDm74Eh+x32D
+jdX6Jzav+5DL3/JvSGFGS6L/
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.crt b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.crt
new file mode 100644
index 0000000000000..53d1f356d2456
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.crt
@@ -0,0 +1,9 @@
+-----BEGIN CERTIFICATE-----
+MIIBPzCB5aADAgECAgRijs0GMAwGCCqGSM49BAMCBQAwFTETMBEGA1UEAxMKc2Ft
+bDQtdGVzdDAeFw0yMDAxMzAxMzU5MDFaFw0yMDA0MjkxMzU5MDFaMBUxEzARBgNV
+BAMTCnNhbWw0LXRlc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAT5gZs/zPxN
+8Ch2aEYMchapSxcMrGYVqtVBO4gIV2ZwYsFhq/mM3+DvenY/9xLwkMHdWx6E8eQG
+gWFcsjPJcQ1CoyEwHzAdBgNVHQ4EFgQUrrGtOyJFw4hhgcdZIke00/BU9OIwDAYI
+KoZIzj0EAwIFAANHADBEAiB+PMEby4C9HNPI0vhQetJhzMFBYg1CXNBCwy/uHFHf
+qAIgTNPpG8hjuOWx7qAy6CSg5CPmO0huRm2sP5gLx68HLOo=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.key b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.key
new file mode 100644
index 0000000000000..bf569cb418626
--- /dev/null
+++ b/x-pack/plugin/identity-provider/src/test/resources/org/elasticsearch/xpack/idp/saml/idp/signing4.key
@@ -0,0 +1,4 @@
+-----BEGIN PRIVATE KEY-----
+MEECAQAwEwYHKoZIzj0CAQYIKoZIzj0DAQcEJzAlAgEBBCDpKTGa+t0VcDav3WQ5
+ahJDxbmeCgjdZ/hhogVGE9GgyA==
+-----END PRIVATE KEY-----
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
index feb2854a0986a..697c7f8f97686 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
@@ -1556,6 +1556,7 @@ private void assertHistoryIsPresent(String policyName, String indexName, boolean
     }
 
     // This method should be called inside an assertBusy, it has no retry logic of its own
+    @SuppressWarnings("unchecked")
     private void assertHistoryIsPresent(String policyName, String indexName, boolean success,
                                         @Nullable String phase, @Nullable String action, String stepName) throws IOException {
         logger.info("--> checking for history item [{}], [{}], success: [{}], phase: [{}], action: [{}], step: [{}]",
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
index f2c9e9c609502..4efff6541b2d7 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
@@ -573,6 +573,7 @@ private Map<String, Object> getSLMStats() {
     }
 
     // This method should be called inside an assertBusy, it has no retry logic of its own
+    @SuppressWarnings("unchecked")
     private void assertHistoryIsPresent(String policyName, boolean success, String repository, String operation) throws IOException {
         final Request historySearchRequest = new Request("GET", ".slm-history*/_search");
         historySearchRequest.setJsonEntity("{\n" +
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
index 3d22402e89779..5b6378dab4e91 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
@@ -63,7 +62,7 @@ public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDat
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.singletonList(
             new SystemIndexDescriptor(LOGSTASH_CONCRETE_INDEX_NAME, "Contains data for Logstash Central Management")
         );
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
index a58418bbd35ef..52030df5b5af3 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.BoostedTreeParams;
@@ -48,6 +49,7 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.in;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -56,6 +58,7 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase {
     private static final String BOOLEAN_FIELD = "boolean-field";
     private static final String NUMERICAL_FIELD = "numerical-field";
     private static final String DISCRETE_NUMERICAL_FIELD = "discrete-numerical-field";
+    private static final String TEXT_FIELD = "text-field";
     private static final String KEYWORD_FIELD = "keyword-field";
     private static final String NESTED_FIELD = "outer-field.inner-field";
     private static final String ALIAS_TO_KEYWORD_FIELD = "alias-to-keyword-field";
@@ -75,7 +78,6 @@ public void cleanup() {
         cleanUp();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53236")
     public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws Exception {
         initialize("classification_single_numeric_feature_and_mixed_data_set");
         String predictedClassField = KEYWORD_FIELD + "_prediction";
@@ -107,7 +109,9 @@ public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws
             assertThat(getFieldValue(resultsObject, predictedClassField), is(in(KEYWORD_FIELD_VALUES)));
             assertThat(getFieldValue(resultsObject, "is_training"), is(destDoc.containsKey(KEYWORD_FIELD)));
             assertTopClasses(resultsObject, 2, KEYWORD_FIELD, KEYWORD_FIELD_VALUES);
-            assertThat(resultsObject.keySet().stream().filter(k -> k.startsWith("feature_importance.")).findAny().isPresent(), is(true));
+            @SuppressWarnings("unchecked")
+            List<Map<String, Object>> importanceArray = (List<Map<String, Object>>)resultsObject.get("feature_importance");
+            assertThat(importanceArray, hasSize(greaterThan(0)));
         }
 
         assertProgress(jobId, 100, 100, 100, 100);
@@ -155,6 +159,12 @@ public void testWithOnlyTrainingRowsAndTrainingPercentIsHundred() throws Excepti
             assertTopClasses(resultsObject, 2, KEYWORD_FIELD, KEYWORD_FIELD_VALUES);
         }
 
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(jobId);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(jobId));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), equalTo(300L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), equalTo(0L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(0L));
+
         assertProgress(jobId, 100, 100, 100, 100);
         assertThat(searchStoredProgress(jobId).getHits().getTotalHits().value, equalTo(1L));
         assertModelStatePersisted(stateDocId());
@@ -222,6 +232,14 @@ public <T> void testWithOnlyTrainingRowsAndTrainingPercentIsFifty(String jobId,
         assertThat(trainingRowsCount, greaterThan(0));
         assertThat(nonTrainingRowsCount, greaterThan(0));
 
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(jobId);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(jobId));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), greaterThan(0L));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), lessThan(300L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), greaterThan(0L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), lessThan(300L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(0L));
+
         assertProgress(jobId, 100, 100, 100, 100);
         assertThat(searchStoredProgress(jobId).getHits().getTotalHits().value, equalTo(1L));
         assertModelStatePersisted(stateDocId());
@@ -260,6 +278,14 @@ public void testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableI
         assertThat(e.getMessage(), startsWith("invalid types [double] for required field [numerical-field];"));
     }
 
+    public void testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableIsText() throws Exception {
+        ElasticsearchStatusException e = expectThrows(
+            ElasticsearchStatusException.class,
+            () -> testWithOnlyTrainingRowsAndTrainingPercentIsFifty(
+                "classification_training_percent_is_50_text", TEXT_FIELD, KEYWORD_FIELD_VALUES, null));
+        assertThat(e.getMessage(), startsWith("field [text-field] of type [text] is non-aggregatable"));
+    }
+
     public void testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableIsBoolean() throws Exception {
         testWithOnlyTrainingRowsAndTrainingPercentIsFifty(
             "classification_training_percent_is_50_boolean", BOOLEAN_FIELD, BOOLEAN_FIELD_VALUES, "boolean");
@@ -322,16 +348,20 @@ public void testStopAndRestart() throws Exception {
         assertEvaluation(KEYWORD_FIELD, KEYWORD_FIELD_VALUES, "ml." + predictedClassField);
     }
 
-    @AwaitsFix(bugUrl = "Muted until ml-cpp supports multiple classes")
     public void testDependentVariableCardinalityTooHighError() throws Exception {
         initialize("cardinality_too_high");
         indexData(sourceIndex, 6, 5, KEYWORD_FIELD);
 
-        // Index one more document with a class different than the two already used.
-        client().execute(IndexAction.INSTANCE, new IndexRequest(sourceIndex)
-            .source(KEYWORD_FIELD, "fox")
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE))
-            .actionGet();
+        // Index enough documents to have more classes than the allowed limit
+        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        for (int i = 0; i < Classification.MAX_DEPENDENT_VARIABLE_CARDINALITY - 1; i++) {
+            IndexRequest indexRequest = new IndexRequest(sourceIndex).source(KEYWORD_FIELD, "fox-" + i);
+            bulkRequestBuilder.add(indexRequest);
+        }
+        BulkResponse bulkResponse = bulkRequestBuilder.get();
+        if (bulkResponse.hasFailures()) {
+            fail("Failed to index data: " + bulkResponse.buildFailureMessage());
+        }
 
         DataFrameAnalyticsConfig config = buildAnalytics(jobId, sourceIndex, destIndex, null, new Classification(KEYWORD_FIELD));
         registerAnalytics(config);
@@ -339,7 +369,7 @@ public void testDependentVariableCardinalityTooHighError() throws Exception {
 
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> startAnalytics(jobId));
         assertThat(e.status().getStatus(), equalTo(400));
-        assertThat(e.getMessage(), equalTo("Field [keyword-field] must have at most [2] distinct values but there were at least [3]"));
+        assertThat(e.getMessage(), equalTo("Field [keyword-field] must have at most [30] distinct values but there were at least [31]"));
     }
 
     public void testDependentVariableCardinalityTooHighButWithQueryMakesItWithinRange() throws Exception {
@@ -517,6 +547,7 @@ private static void createIndex(String index) {
                 BOOLEAN_FIELD, "type=boolean",
                 NUMERICAL_FIELD, "type=double",
                 DISCRETE_NUMERICAL_FIELD, "type=integer",
+                TEXT_FIELD, "type=text",
                 KEYWORD_FIELD, "type=keyword",
                 NESTED_FIELD, "type=keyword",
                 ALIAS_TO_KEYWORD_FIELD, "type=alias,path=" + KEYWORD_FIELD,
@@ -532,6 +563,7 @@ private static void indexData(String sourceIndex, int numTrainingRows, int numNo
                 BOOLEAN_FIELD, BOOLEAN_FIELD_VALUES.get(i % BOOLEAN_FIELD_VALUES.size()),
                 NUMERICAL_FIELD, NUMERICAL_FIELD_VALUES.get(i % NUMERICAL_FIELD_VALUES.size()),
                 DISCRETE_NUMERICAL_FIELD, DISCRETE_NUMERICAL_FIELD_VALUES.get(i % DISCRETE_NUMERICAL_FIELD_VALUES.size()),
+                TEXT_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size()),
                 KEYWORD_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size()),
                 NESTED_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size()));
             IndexRequest indexRequest = new IndexRequest(sourceIndex).source(source.toArray());
@@ -549,6 +581,9 @@ private static void indexData(String sourceIndex, int numTrainingRows, int numNo
                 source.addAll(
                     List.of(DISCRETE_NUMERICAL_FIELD, DISCRETE_NUMERICAL_FIELD_VALUES.get(i % DISCRETE_NUMERICAL_FIELD_VALUES.size())));
             }
+            if (TEXT_FIELD.equals(dependentVariable) == false) {
+                source.addAll(List.of(TEXT_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size())));
+            }
             if (KEYWORD_FIELD.equals(dependentVariable) == false) {
                 source.addAll(List.of(KEYWORD_FIELD, KEYWORD_FIELD_VALUES.get(i % KEYWORD_FIELD_VALUES.size())));
             }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
index d6c88fbfdb994..4f729d1dec122 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
@@ -156,8 +156,10 @@ public void testSimulate() throws IOException {
         String responseString = EntityUtils.toString(response.getEntity());
         assertThat(responseString, containsString("\"predicted_value\":\"second\""));
         assertThat(responseString, containsString("\"predicted_value\":1.0"));
-        assertThat(responseString, containsString("\"col2\":0.944"));
-        assertThat(responseString, containsString("\"col1\":0.19999"));
+        assertThat(responseString, containsString("\"feature_name\":\"col1\""));
+        assertThat(responseString, containsString("\"feature_name\":\"col2\""));
+        assertThat(responseString, containsString("\"importance\":0.944"));
+        assertThat(responseString, containsString("\"importance\":0.19999"));
 
         String sourceWithMissingModel = "{\n" +
             "  \"pipeline\": {\n" +
@@ -221,8 +223,10 @@ public void testSimulateWithDefaultMappedField() throws IOException {
         Response response = client().performRequest(simulateRequest(source));
         String responseString = EntityUtils.toString(response.getEntity());
         assertThat(responseString, containsString("\"predicted_value\":\"second\""));
-        assertThat(responseString, containsString("\"col2\":0.944"));
-        assertThat(responseString, containsString("\"col1\":0.19999"));
+        assertThat(responseString, containsString("\"feature_name\":\"col1\""));
+        assertThat(responseString, containsString("\"feature_name\":\"col2\""));
+        assertThat(responseString, containsString("\"importance\":0.944"));
+        assertThat(responseString, containsString("\"importance\":0.19999"));
     }
 
     public void testSimulateLangIdent() throws IOException {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java
index ee55209134900..e871ab5237727 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java
@@ -314,7 +314,8 @@ protected static void assertMlResultsFieldMappings(String index, String predicte
      * - allows dots (".") in the path elements provided as arguments
      * - supports implicit casting to the appropriate type
      */
+    @SuppressWarnings("unchecked")
     protected static <T> T getFieldValue(Map<String, Object> doc, String... path) {
-        return (T)extractValue(String.join(".", path), doc);
+        return (T) extractValue(String.join(".", path), doc);
     }
 }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java
index e32d923852038..ced8c987fd685 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
 import org.junit.After;
@@ -79,6 +80,12 @@ public void testMissingFields() throws Exception {
         startAnalytics(id);
         waitUntilAnalyticsIsStopped(id);
 
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(id);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(id));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), equalTo(5L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), equalTo(0L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(2L));
+
         SearchResponse sourceData = client().prepareSearch(sourceIndex).get();
         for (SearchHit hit : sourceData.getHits()) {
             GetResponse destDocGetResponse = client().prepareGet().setIndex(config.getDest().getIndex()).setId(hit.getId()).get();
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
index 5fc6d618da9ab..07752468a743f 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.BoostedTreeParams;
@@ -25,10 +26,13 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.test.hamcrest.OptionalMatchers.isPresent;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
 
 public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase {
 
@@ -48,7 +52,6 @@ public void cleanup() {
         cleanUp();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53236")
     public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws Exception {
         initialize("regression_single_numeric_feature_and_mixed_data_set");
         String predictedClassField = DEPENDENT_VARIABLE_FIELD + "_prediction";
@@ -86,11 +89,13 @@ public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws
             assertThat(resultsObject.containsKey(predictedClassField), is(true));
             assertThat(resultsObject.containsKey("is_training"), is(true));
             assertThat(resultsObject.get("is_training"), is(destDoc.containsKey(DEPENDENT_VARIABLE_FIELD)));
+            @SuppressWarnings("unchecked")
+            List<Map<String, Object>> importanceArray = (List<Map<String, Object>>)resultsObject.get("feature_importance");
+            assertThat(importanceArray, hasSize(greaterThan(0)));
             assertThat(
-                resultsObject.toString(),
-                resultsObject.containsKey("feature_importance." + NUMERICAL_FEATURE_FIELD)
-                    || resultsObject.containsKey("feature_importance." + DISCRETE_NUMERICAL_FEATURE_FIELD),
-                is(true));
+                importanceArray.stream().filter(m -> NUMERICAL_FEATURE_FIELD.equals(m.get("feature_name"))
+                    || DISCRETE_NUMERICAL_FEATURE_FIELD.equals(m.get("feature_name"))).findAny(),
+                isPresent());
         }
 
         assertProgress(jobId, 100, 100, 100, 100);
@@ -138,6 +143,13 @@ public void testWithOnlyTrainingRowsAndTrainingPercentIsHundred() throws Excepti
 
         assertProgress(jobId, 100, 100, 100, 100);
         assertThat(searchStoredProgress(jobId).getHits().getTotalHits().value, equalTo(1L));
+
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(jobId);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(jobId));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), equalTo(350L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), equalTo(0L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(0L));
+
         assertModelStatePersisted(stateDocId());
         assertInferenceModelPersisted(jobId);
         assertMlResultsFieldMappings(destIndex, predictedClassField, "double");
@@ -194,6 +206,14 @@ public void testWithOnlyTrainingRowsAndTrainingPercentIsFifty() throws Exception
         assertThat(trainingRowsCount, greaterThan(0));
         assertThat(nonTrainingRowsCount, greaterThan(0));
 
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(jobId);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(jobId));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), greaterThan(0L));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), lessThan(350L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), greaterThan(0L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), lessThan(350L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(0L));
+
         assertProgress(jobId, 100, 100, 100, 100);
         assertThat(searchStoredProgress(jobId).getHits().getTotalHits().value, equalTo(1L));
         assertModelStatePersisted(stateDocId());
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
index 4ffe948c9f5b7..3f7a5a7156637 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
@@ -85,6 +85,11 @@ public void testOutlierDetectionWithFewDocuments() throws Exception {
 
         startAnalytics(id);
         waitUntilAnalyticsIsStopped(id);
+        GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(id);
+        assertThat(stats.getDataCounts().getJobId(), equalTo(id));
+        assertThat(stats.getDataCounts().getTrainingDocsCount(), equalTo(5L));
+        assertThat(stats.getDataCounts().getTestDocsCount(), equalTo(0L));
+        assertThat(stats.getDataCounts().getSkippedDocsCount(), equalTo(0L));
 
         SearchResponse sourceData = client().prepareSearch(sourceIndex).get();
         double scoreOfOutlier = 0.0;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index 2a22a37841cd1..e17d06dbf9390 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -982,7 +982,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return List.of(
             new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, "Contains scheduling and anomaly tracking metadata"),
             new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, "Contains ML configuration data"),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
index 47f2a216f2a81..422c690a5fc92 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
@@ -36,11 +37,13 @@
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ml.MlStatsIndex;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.StopDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -138,20 +141,38 @@ private void normalDelete(ParentTaskAssigningClient parentTaskClient, ClusterSta
         // We clean up the memory tracker on delete because there is no stop; the task stops by itself
         memoryTracker.removeDataFrameAnalyticsJob(id);
 
-        // Step 3. Delete the config
+        // Step 4. Delete the config
+        ActionListener<BulkByScrollResponse> deleteStatsHandler = ActionListener.wrap(
+            bulkByScrollResponse -> {
+                if (bulkByScrollResponse.isTimedOut()) {
+                    logger.warn("[{}] DeleteByQuery for stats timed out", id);
+                }
+                if (bulkByScrollResponse.getBulkFailures().isEmpty() == false) {
+                    logger.warn("[{}] {} failures and {} conflicts encountered while running DeleteByQuery for stats", id,
+                        bulkByScrollResponse.getBulkFailures().size(), bulkByScrollResponse.getVersionConflicts());
+                    for (BulkItemResponse.Failure failure : bulkByScrollResponse.getBulkFailures()) {
+                        logger.warn("[{}] DBQ failure: {}", id, failure);
+                    }
+                }
+                deleteConfig(parentTaskClient, id, listener);
+            },
+            listener::onFailure
+        );
+
+        // Step 3. Delete job docs from stats index
         ActionListener<BulkByScrollResponse> deleteStateHandler = ActionListener.wrap(
             bulkByScrollResponse -> {
                 if (bulkByScrollResponse.isTimedOut()) {
                     logger.warn("[{}] DeleteByQuery for state timed out", id);
                 }
                 if (bulkByScrollResponse.getBulkFailures().isEmpty() == false) {
-                    logger.warn("[{}] {} failures and {} conflicts encountered while runnint DeleteByQuery for state", id,
+                    logger.warn("[{}] {} failures and {} conflicts encountered while running DeleteByQuery for state", id,
                         bulkByScrollResponse.getBulkFailures().size(), bulkByScrollResponse.getVersionConflicts());
                     for (BulkItemResponse.Failure failure : bulkByScrollResponse.getBulkFailures()) {
                         logger.warn("[{}] DBQ failure: {}", id, failure);
                     }
                 }
-                deleteConfig(parentTaskClient, id, listener);
+                deleteStats(parentTaskClient, id, deleteStatsHandler);
             },
             listener::onFailure
         );
@@ -193,8 +214,29 @@ private void deleteState(ParentTaskAssigningClient parentTaskClient,
         if (config.getAnalysis().persistsState()) {
             ids.add(config.getAnalysis().getStateDocId(config.getId()));
         }
-        DeleteByQueryRequest request = new DeleteByQueryRequest(AnomalyDetectorsIndex.jobStateIndexPattern());
-        request.setQuery(QueryBuilders.idsQuery().addIds(ids.toArray(String[]::new)));
+        executeDeleteByQuery(
+            parentTaskClient,
+            AnomalyDetectorsIndex.jobStateIndexPattern(),
+            QueryBuilders.idsQuery().addIds(ids.toArray(String[]::new)),
+            listener
+        );
+    }
+
+    private void deleteStats(ParentTaskAssigningClient parentTaskClient,
+                             String jobId,
+                             ActionListener<BulkByScrollResponse> listener) {
+        executeDeleteByQuery(
+            parentTaskClient,
+            MlStatsIndex.indexPattern(),
+            QueryBuilders.termQuery(Fields.JOB_ID.getPreferredName(), jobId),
+            listener
+        );
+    }
+
+    private void executeDeleteByQuery(ParentTaskAssigningClient parentTaskClient, String index, QueryBuilder query,
+                                      ActionListener<BulkByScrollResponse> listener) {
+        DeleteByQueryRequest request = new DeleteByQueryRequest(index);
+        request.setQuery(query);
         request.setIndicesOptions(MlIndicesUtils.addIgnoreUnavailable(IndicesOptions.lenientExpandOpen()));
         request.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
         request.setAbortOnVersionConflict(false);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
index d415156450082..776c84031428b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
@@ -23,6 +23,17 @@
 import java.util.Map;
 import java.util.Set;
 
+/**
+ * Calculates the estimated model memory requirement of an anomaly detection job
+ * from its analysis config and estimates of the cardinality of the various fields
+ * referenced in it.
+ *
+ * Answers are capped at <code>Long.MAX_VALUE</code> bytes, to avoid returning
+ * values with bigger units that cannot trivially be converted back to bytes.
+ * (In reality if the memory estimate is greater than <code>Long.MAX_VALUE</code>
+ * bytes then the job will be impossible to run successfully, so this is not a
+ * major limitation.)
+ */
 public class TransportEstimateModelMemoryAction
     extends HandledTransportAction<EstimateModelMemoryAction.Request, EstimateModelMemoryAction.Response> {
 
@@ -47,23 +58,24 @@ protected void doExecute(Task task,
         Map<String, Long> overallCardinality = request.getOverallCardinality();
         Map<String, Long> maxBucketCardinality = request.getMaxBucketCardinality();
 
-        long answer = BASIC_REQUIREMENT.getBytes()
-            + calculateDetectorsRequirementBytes(analysisConfig, overallCardinality)
-            + calculateInfluencerRequirementBytes(analysisConfig, maxBucketCardinality)
-            + calculateCategorizationRequirementBytes(analysisConfig);
+        long answer = BASIC_REQUIREMENT.getBytes();
+        answer = addNonNegativeLongsWithMaxValueCap(answer, calculateDetectorsRequirementBytes(analysisConfig, overallCardinality));
+        answer = addNonNegativeLongsWithMaxValueCap(answer, calculateInfluencerRequirementBytes(analysisConfig, maxBucketCardinality));
+        answer = addNonNegativeLongsWithMaxValueCap(answer, calculateCategorizationRequirementBytes(analysisConfig));
 
         listener.onResponse(new EstimateModelMemoryAction.Response(roundUpToNextMb(answer)));
     }
 
     static long calculateDetectorsRequirementBytes(AnalysisConfig analysisConfig, Map<String, Long> overallCardinality) {
         return analysisConfig.getDetectors().stream().map(detector -> calculateDetectorRequirementBytes(detector, overallCardinality))
-            .reduce(0L, Long::sum);
+            .reduce(0L, TransportEstimateModelMemoryAction::addNonNegativeLongsWithMaxValueCap);
     }
 
     static long calculateDetectorRequirementBytes(Detector detector, Map<String, Long> overallCardinality) {
 
         long answer = 0;
 
+        // These values for detectors assume splitting is via a partition field
         switch (detector.getFunction()) {
             case COUNT:
             case LOW_COUNT:
@@ -71,7 +83,7 @@ static long calculateDetectorRequirementBytes(Detector detector, Map<String, Lon
             case NON_ZERO_COUNT:
             case LOW_NON_ZERO_COUNT:
             case HIGH_NON_ZERO_COUNT:
-                answer = 1; // TODO add realistic number
+                answer = new ByteSizeValue(32, ByteSizeUnit.KB).getBytes();
                 break;
             case DISTINCT_COUNT:
             case LOW_DISTINCT_COUNT:
@@ -88,7 +100,8 @@ static long calculateDetectorRequirementBytes(Detector detector, Map<String, Lon
                 answer = 1; // TODO add realistic number
                 break;
             case METRIC:
-                answer = 1; // TODO add realistic number
+                // metric analyses mean, min and max simultaneously, and uses about 2.5 times the memory of one of these
+                answer = new ByteSizeValue(160, ByteSizeUnit.KB).getBytes();
                 break;
             case MEAN:
             case LOW_MEAN:
@@ -104,18 +117,14 @@ static long calculateDetectorRequirementBytes(Detector detector, Map<String, Lon
             case NON_NULL_SUM:
             case LOW_NON_NULL_SUM:
             case HIGH_NON_NULL_SUM:
-                // 64 comes from https://github.com/elastic/kibana/issues/18722
-                answer = new ByteSizeValue(64, ByteSizeUnit.KB).getBytes();
-                break;
             case MEDIAN:
             case LOW_MEDIAN:
             case HIGH_MEDIAN:
-                answer = 1; // TODO add realistic number
-                break;
             case VARP:
             case LOW_VARP:
             case HIGH_VARP:
-                answer = 1; // TODO add realistic number
+                // 64 comes from https://github.com/elastic/kibana/issues/18722
+                answer = new ByteSizeValue(64, ByteSizeUnit.KB).getBytes();
                 break;
             case TIME_OF_DAY:
             case TIME_OF_WEEK:
@@ -130,19 +139,26 @@ static long calculateDetectorRequirementBytes(Detector detector, Map<String, Lon
 
         String byFieldName = detector.getByFieldName();
         if (byFieldName != null) {
-            answer *= cardinalityEstimate(Detector.BY_FIELD_NAME_FIELD.getPreferredName(), byFieldName, overallCardinality, true);
+            long cardinalityEstimate =
+                cardinalityEstimate(Detector.BY_FIELD_NAME_FIELD.getPreferredName(), byFieldName, overallCardinality, true);
+            // The memory cost of a by field is about 2/3rds that of a partition field
+            long multiplier = addNonNegativeLongsWithMaxValueCap(cardinalityEstimate, 2) / 3 * 2;
+            answer = multiplyNonNegativeLongsWithMaxValueCap(answer, multiplier);
         }
 
         String overFieldName = detector.getOverFieldName();
         if (overFieldName != null) {
-            cardinalityEstimate(Detector.OVER_FIELD_NAME_FIELD.getPreferredName(), overFieldName, overallCardinality, true);
-            // TODO - how should "over" field cardinality affect estimate?
+            long cardinalityEstimate =
+                cardinalityEstimate(Detector.OVER_FIELD_NAME_FIELD.getPreferredName(), overFieldName, overallCardinality, true);
+            // Over fields don't multiply the whole estimate, just add a small amount (estimate 512 bytes) per value
+            answer = addNonNegativeLongsWithMaxValueCap(answer, multiplyNonNegativeLongsWithMaxValueCap(cardinalityEstimate, 512));
         }
 
         String partitionFieldName = detector.getPartitionFieldName();
         if (partitionFieldName != null) {
-            answer *=
+            long multiplier =
                 cardinalityEstimate(Detector.PARTITION_FIELD_NAME_FIELD.getPreferredName(), partitionFieldName, overallCardinality, true);
+            answer = multiplyNonNegativeLongsWithMaxValueCap(answer, multiplier);
         }
 
         return answer;
@@ -156,10 +172,10 @@ static long calculateInfluencerRequirementBytes(AnalysisConfig analysisConfig, M
             pureInfluencers.removeAll(detector.extractAnalysisFields());
         }
 
-        return pureInfluencers.stream()
-            .map(influencer -> cardinalityEstimate(AnalysisConfig.INFLUENCERS.getPreferredName(), influencer, maxBucketCardinality, false)
-                * BYTES_PER_INFLUENCER_VALUE)
-            .reduce(0L, Long::sum);
+        long totalInfluencerCardinality = pureInfluencers.stream()
+            .map(influencer -> cardinalityEstimate(AnalysisConfig.INFLUENCERS.getPreferredName(), influencer, maxBucketCardinality, false))
+            .reduce(0L, TransportEstimateModelMemoryAction::addNonNegativeLongsWithMaxValueCap);
+        return multiplyNonNegativeLongsWithMaxValueCap(BYTES_PER_INFLUENCER_VALUE, totalInfluencerCardinality);
     }
 
     static long calculateCategorizationRequirementBytes(AnalysisConfig analysisConfig) {
@@ -187,7 +203,25 @@ static long cardinalityEstimate(String description, String fieldName, Map<String
     }
 
     static ByteSizeValue roundUpToNextMb(long bytes) {
-        assert bytes >= 0;
-        return new ByteSizeValue((BYTES_IN_MB - 1 + bytes) / BYTES_IN_MB, ByteSizeUnit.MB);
+        assert bytes >= 0 : "negative bytes " + bytes;
+        return new ByteSizeValue(addNonNegativeLongsWithMaxValueCap(bytes, BYTES_IN_MB - 1) / BYTES_IN_MB, ByteSizeUnit.MB);
+    }
+
+    private static long addNonNegativeLongsWithMaxValueCap(long a, long b) {
+        assert a >= 0;
+        assert b >= 0;
+        if (Long.MAX_VALUE - a - b < 0) {
+            return Long.MAX_VALUE;
+        }
+        return a + b;
+    }
+
+    private static long multiplyNonNegativeLongsWithMaxValueCap(long a, long b) {
+        assert a >= 0;
+        assert b >= 0;
+        if (Long.MAX_VALUE / a < b) {
+            return Long.MAX_VALUE;
+        }
+        return a * b;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
index 9ee28ba29cf3a..cb8d7bcb2be4a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.Fields;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.MemoryUsage;
 import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStats;
@@ -108,6 +109,7 @@ protected void taskOperation(GetDataFrameAnalyticsStatsAction.Request request, D
                 Stats stats = buildStats(
                     task.getParams().getId(),
                     statsHolder.getProgressTracker().report(),
+                    statsHolder.getDataCountsTracker().report(task.getParams().getId()),
                     statsHolder.getMemoryUsage(),
                     statsHolder.getAnalysisStats()
                 );
@@ -200,6 +202,7 @@ private void searchStats(String configId, ActionListener<Stats> listener) {
 
         MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
         multiSearchRequest.add(buildStoredProgressSearch(configId));
+        multiSearchRequest.add(buildStatsDocSearch(configId, DataCounts.TYPE_VALUE));
         multiSearchRequest.add(buildStatsDocSearch(configId, MemoryUsage.TYPE_VALUE));
         multiSearchRequest.add(buildStatsDocSearch(configId, OutlierDetectionStats.TYPE_VALUE));
         multiSearchRequest.add(buildStatsDocSearch(configId, ClassificationStats.TYPE_VALUE));
@@ -224,6 +227,7 @@ private void searchStats(String configId, ActionListener<Stats> listener) {
                 }
                 listener.onResponse(buildStats(configId,
                     retrievedStatsHolder.progress.get(),
+                    retrievedStatsHolder.dataCounts,
                     retrievedStatsHolder.memoryUsage,
                     retrievedStatsHolder.analysisStats
                 ));
@@ -258,6 +262,8 @@ private static void parseHit(SearchHit hit, String configId, RetrievedStatsHolde
         String hitId = hit.getId();
         if (StoredProgress.documentId(configId).equals(hitId)) {
             retrievedStatsHolder.progress = MlParserUtils.parse(hit, StoredProgress.PARSER);
+        } else if (DataCounts.documentId(configId).equals(hitId)) {
+            retrievedStatsHolder.dataCounts = MlParserUtils.parse(hit, DataCounts.LENIENT_PARSER);
         } else if (hitId.startsWith(MemoryUsage.documentIdPrefix(configId))) {
             retrievedStatsHolder.memoryUsage = MlParserUtils.parse(hit, MemoryUsage.LENIENT_PARSER);
         } else if (hitId.startsWith(OutlierDetectionStats.documentIdPrefix(configId))) {
@@ -273,6 +279,7 @@ private static void parseHit(SearchHit hit, String configId, RetrievedStatsHolde
 
     private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concreteAnalyticsId,
                                                                        List<PhaseProgress> progress,
+                                                                       DataCounts dataCounts,
                                                                        MemoryUsage memoryUsage,
                                                                        AnalysisStats analysisStats) {
         ClusterState clusterState = clusterService.state();
@@ -295,6 +302,7 @@ private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concre
             analyticsState,
             failureReason,
             progress,
+            dataCounts,
             memoryUsage,
             analysisStats,
             node,
@@ -305,6 +313,7 @@ private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concre
     private static class RetrievedStatsHolder {
 
         private volatile StoredProgress progress = new StoredProgress(new ProgressTracker().report());
+        private volatile DataCounts dataCounts;
         private volatile MemoryUsage memoryUsage;
         private volatile AnalysisStats analysisStats;
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
index 44bf8c483dfd9..6fc8000b0a559 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
@@ -289,7 +289,7 @@ public void onResponse(PersistentTasksCustomMetaData.PersistentTask<OpenJobActio
                         listener.onFailure(predicate.exception);
                     }
                 } else {
-                    listener.onResponse(new AcknowledgedResponse(predicate.opened));
+                    listener.onResponse(new AcknowledgedResponse(true));
                 }
             }
 
@@ -537,9 +537,8 @@ void closeJob(String reason) {
      * Important: the methods of this class must NOT throw exceptions.  If they did then the callers
      * of endpoints waiting for a condition tested by this predicate would never get a response.
      */
-    private class JobPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
+    private static class JobPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
 
-        private volatile boolean opened;
         private volatile Exception exception;
         private volatile boolean shouldCancel;
 
@@ -586,7 +585,6 @@ public boolean test(PersistentTasksCustomMetaData.PersistentTask<?> persistentTa
                 case CLOSED:
                     return false;
                 case OPENED:
-                    opened = true;
                     return true;
                 case CLOSING:
                     exception = ExceptionsHelper.conflictStatusException("The job has been " + JobState.CLOSED + " while waiting to be "
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
index 40c4cea806f6f..0dd24f8f85994 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
 import org.elasticsearch.persistent.PersistentTaskState;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
@@ -39,7 +40,6 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.ml.MlTasks;
@@ -91,7 +91,7 @@ public class TransportStartDatafeedAction extends TransportMasterNodeAction<Star
     private final AnomalyDetectionAuditor auditor;
     private final MlConfigMigrationEligibilityCheck migrationEligibilityCheck;
     private final NamedXContentRegistry xContentRegistry;
-    private final boolean remoteClusterSearchSupported;
+    private final boolean remoteClusterClient;
 
     @Inject
     public TransportStartDatafeedAction(Settings settings, TransportService transportService, ThreadPool threadPool,
@@ -110,7 +110,7 @@ public TransportStartDatafeedAction(Settings settings, TransportService transpor
         this.auditor = auditor;
         this.migrationEligibilityCheck = new MlConfigMigrationEligibilityCheck(settings, clusterService);
         this.xContentRegistry = xContentRegistry;
-        this.remoteClusterSearchSupported = RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings);
+        this.remoteClusterClient = Node.NODE_REMOTE_CLUSTER_CLIENT.get(settings);
     }
 
     static void validate(Job job,
@@ -201,7 +201,7 @@ public void onFailure(Exception e) {
                                     response -> {
                                         if (response.isSuccess() == false) {
                                             listener.onFailure(createUnlicensedError(params.getDatafeedId(), response));
-                                        } else if (remoteClusterSearchSupported == false) {
+                                        } else if (remoteClusterClient == false) {
                                             listener.onFailure(
                                                 ExceptionsHelper.badRequestException(Messages.getMessage(
                                                     Messages.DATAFEED_NEEDS_REMOTE_CLUSTER_SEARCH,
@@ -477,7 +477,7 @@ public void isolate() {
      * Important: the methods of this class must NOT throw exceptions.  If they did then the callers
      * of endpoints waiting for a condition tested by this predicate would never get a response.
      */
-    private class DatafeedPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
+    private static class DatafeedPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
 
         private volatile Exception exception;
 
@@ -487,12 +487,17 @@ public boolean test(PersistentTasksCustomMetaData.PersistentTask<?> persistentTa
                 return false;
             }
             PersistentTasksCustomMetaData.Assignment assignment = persistentTask.getAssignment();
-            if (assignment != null && assignment.equals(PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT) == false &&
-                    assignment.isAssigned() == false) {
-                // Assignment has failed despite passing our "fast fail" validation
-                exception = new ElasticsearchStatusException("Could not start datafeed, allocation explanation [" +
+            if (assignment != null) {
+                // This means we are awaiting the datafeed's job to be assigned to a node
+                if (assignment.equals(DatafeedNodeSelector.AWAITING_JOB_ASSIGNMENT)) {
+                    return true;
+                }
+                if (assignment.equals(PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT) == false && assignment.isAssigned() == false) {
+                    // Assignment has failed despite passing our "fast fail" validation
+                    exception = new ElasticsearchStatusException("Could not start datafeed, allocation explanation [" +
                         assignment.getExplanation() + "]", RestStatus.TOO_MANY_REQUESTS);
-                return true;
+                    return true;
+                }
             }
             DatafeedState datafeedState = (DatafeedState) persistentTask.getState();
             return datafeedState == DatafeedState.STARTED;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
index 903733e09970c..db94905a1bdb5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
@@ -93,14 +93,8 @@ private static void addDatafeedTaskIdAccordingToState(String datafeedId,
                                                           List<String> stoppingDatafeedIds,
                                                           List<String> notStoppedDatafeedIds) {
         switch (datafeedState) {
+            // Treat STARTING like STARTED for stop API behaviour.
             case STARTING:
-                // The STARTING state is not used anywhere at present, so this should never happen.
-                // At present datafeeds that have a persistent task that hasn't yet been assigned
-                // a state are reported as STOPPED (which is not great).  It could be considered a
-                // breaking change to introduce the STARTING state though, so let's aim to do it in
-                // version 8.  Also consider treating STARTING like STARTED for stop API behaviour.
-                notStoppedDatafeedIds.add(datafeedId);
-                break;
             case STARTED:
                 startedDatafeedIds.add(datafeedId);
                 notStoppedDatafeedIds.add(datafeedId);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java
index dc88dac0e7cce..499da77ecbb95 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
-import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedJobValidator;
@@ -51,7 +51,7 @@ public class DatafeedJobBuilder {
     private final JobResultsProvider jobResultsProvider;
     private final DatafeedConfigProvider datafeedConfigProvider;
     private final JobResultsPersister jobResultsPersister;
-    private final boolean remoteClusterSearchSupported;
+    private final boolean remoteClusterClient;
     private final String nodeName;
 
     public DatafeedJobBuilder(Client client, NamedXContentRegistry xContentRegistry, AnomalyDetectionAuditor auditor,
@@ -66,7 +66,7 @@ public DatafeedJobBuilder(Client client, NamedXContentRegistry xContentRegistry,
         this.jobResultsProvider = Objects.requireNonNull(jobResultsProvider);
         this.datafeedConfigProvider = Objects.requireNonNull(datafeedConfigProvider);
         this.jobResultsPersister = Objects.requireNonNull(jobResultsPersister);
-        this.remoteClusterSearchSupported = RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings);
+        this.remoteClusterClient = Node.NODE_REMOTE_CLUSTER_CLIENT.get(settings);
         this.nodeName = nodeName;
     }
 
@@ -181,7 +181,7 @@ void build(String datafeedId, ActionListener<DatafeedJob> listener) {
                 configBuilder -> {
                     try {
                         datafeedConfigHolder.set(configBuilder.build());
-                        if (remoteClusterSearchSupported == false) {
+                        if (remoteClusterClient == false) {
                             List<String> remoteIndices = RemoteClusterLicenseChecker.remoteIndices(datafeedConfigHolder.get().getIndices());
                             if (remoteIndices.isEmpty() == false) {
                                 listener.onFailure(
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
index 03c8272a979d8..a4c9748dd64c7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
@@ -32,6 +32,9 @@ public class DatafeedNodeSelector {
 
     private static final Logger LOGGER = LogManager.getLogger(DatafeedNodeSelector.class);
 
+    public static final PersistentTasksCustomMetaData.Assignment AWAITING_JOB_ASSIGNMENT =
+        new PersistentTasksCustomMetaData.Assignment(null, "datafeed awaiting job assignment.");
+
     private final String datafeedId;
     private final String jobId;
     private final List<String> datafeedIndices;
@@ -76,9 +79,14 @@ public PersistentTasksCustomMetaData.Assignment selectNode() {
 
         AssignmentFailure assignmentFailure = checkAssignment();
         if (assignmentFailure == null) {
-            return new PersistentTasksCustomMetaData.Assignment(jobTask.getExecutorNode(), "");
+            String jobNode = jobTask.getExecutorNode();
+            if (jobNode == null) {
+                return AWAITING_JOB_ASSIGNMENT;
+            }
+            return new PersistentTasksCustomMetaData.Assignment(jobNode, "");
         }
         LOGGER.debug(assignmentFailure.reason);
+        assert assignmentFailure.reason.isEmpty() == false;
         return new PersistentTasksCustomMetaData.Assignment(null, assignmentFailure.reason);
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
index ba1bdf9a6a5d0..aad06b71c075a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
@@ -19,6 +19,9 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.sort.SortOrder;
@@ -187,7 +190,7 @@ private Row createRow(SearchHit hit) {
             if (values.length == 1 && (values[0] instanceof Number || values[0] instanceof String)) {
                 extractedValues[i] = Objects.toString(values[0]);
             } else {
-                if (values.length == 0 && context.includeRowsWithMissingValues) {
+                if (values.length == 0 && context.supportsRowsWithMissingValues) {
                     // if values is empty then it means it's a missing value
                     extractedValues[i] = NULL_VALUE;
                 } else {
@@ -263,13 +266,29 @@ public void collectDataSummaryAsync(ActionListener<DataSummary> dataSummaryActio
     }
 
     private SearchRequestBuilder buildDataSummarySearchRequestBuilder() {
+
+        QueryBuilder summaryQuery = context.query;
+        if (context.supportsRowsWithMissingValues == false) {
+            summaryQuery = QueryBuilders.boolQuery()
+                .filter(summaryQuery)
+                .filter(allExtractedFieldsExistQuery());
+        }
+
         return new SearchRequestBuilder(client, SearchAction.INSTANCE)
             .setIndices(context.indices)
             .setSize(0)
-            .setQuery(context.query)
+            .setQuery(summaryQuery)
             .setTrackTotalHits(true);
     }
 
+    private QueryBuilder allExtractedFieldsExistQuery() {
+        BoolQueryBuilder query = QueryBuilders.boolQuery();
+        for (ExtractedField field : context.extractedFields.getAllFields()) {
+            query.filter(QueryBuilders.existsQuery(field.getName()));
+        }
+        return query;
+    }
+
     public Set<String> getCategoricalFields(DataFrameAnalysis analysis) {
         return ExtractedFieldsDetector.getCategoricalFields(context.extractedFields, analysis);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java
index 0cf391bc33b2e..64ad4bed452e7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java
@@ -21,10 +21,10 @@ public class DataFrameDataExtractorContext {
     final int scrollSize;
     final Map<String, String> headers;
     final boolean includeSource;
-    final boolean includeRowsWithMissingValues;
+    final boolean supportsRowsWithMissingValues;
 
     DataFrameDataExtractorContext(String jobId, ExtractedFields extractedFields, List<String> indices, QueryBuilder query, int scrollSize,
-                                  Map<String, String> headers, boolean includeSource, boolean includeRowsWithMissingValues) {
+                                  Map<String, String> headers, boolean includeSource, boolean supportsRowsWithMissingValues) {
         this.jobId = Objects.requireNonNull(jobId);
         this.extractedFields = Objects.requireNonNull(extractedFields);
         this.indices = indices.toArray(new String[indices.size()]);
@@ -32,6 +32,6 @@ public class DataFrameDataExtractorContext {
         this.scrollSize = scrollSize;
         this.headers = headers;
         this.includeSource = includeSource;
-        this.includeRowsWithMissingValues = includeRowsWithMissingValues;
+        this.supportsRowsWithMissingValues = supportsRowsWithMissingValues;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java
index 3243d92bf77b6..a699e16a7d602 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java
@@ -7,11 +7,9 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
-import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
 
 import java.util.Arrays;
@@ -28,18 +26,18 @@ public class DataFrameDataExtractorFactory {
     private final QueryBuilder sourceQuery;
     private final ExtractedFields extractedFields;
     private final Map<String, String> headers;
-    private final boolean includeRowsWithMissingValues;
+    private final boolean supportsRowsWithMissingValues;
 
     private DataFrameDataExtractorFactory(Client client, String analyticsId, List<String> indices, QueryBuilder sourceQuery,
                                           ExtractedFields extractedFields, Map<String, String> headers,
-                                          boolean includeRowsWithMissingValues) {
+                                          boolean supportsRowsWithMissingValues) {
         this.client = Objects.requireNonNull(client);
         this.analyticsId = Objects.requireNonNull(analyticsId);
         this.indices = Objects.requireNonNull(indices);
         this.sourceQuery = Objects.requireNonNull(sourceQuery);
         this.extractedFields = Objects.requireNonNull(extractedFields);
         this.headers = headers;
-        this.includeRowsWithMissingValues = includeRowsWithMissingValues;
+        this.supportsRowsWithMissingValues = supportsRowsWithMissingValues;
     }
 
     public DataFrameDataExtractor newExtractor(boolean includeSource) {
@@ -47,11 +45,11 @@ public DataFrameDataExtractor newExtractor(boolean includeSource) {
                 analyticsId,
                 extractedFields,
                 indices,
-                createQuery(),
+                QueryBuilders.boolQuery().filter(sourceQuery),
                 1000,
                 headers,
                 includeSource,
-                includeRowsWithMissingValues
+                supportsRowsWithMissingValues
             );
         return new DataFrameDataExtractor(client, context);
     }
@@ -60,23 +58,6 @@ public ExtractedFields getExtractedFields() {
         return extractedFields;
     }
 
-    private QueryBuilder createQuery() {
-        BoolQueryBuilder query = QueryBuilders.boolQuery();
-        query.filter(sourceQuery);
-        if (includeRowsWithMissingValues == false) {
-            query.filter(allExtractedFieldsExistQuery());
-        }
-        return query;
-    }
-
-    private QueryBuilder allExtractedFieldsExistQuery() {
-        BoolQueryBuilder query = QueryBuilders.boolQuery();
-        for (ExtractedField field : extractedFields.getAllFields()) {
-            query.filter(QueryBuilders.existsQuery(field.getName()));
-        }
-        return query;
-    }
-
     /**
      * Create a new extractor factory
      *
@@ -109,6 +90,7 @@ public static void createForDestinationIndex(Client client,
         extractedFieldsDetectorFactory.createFromDest(config, ActionListener.wrap(
             extractedFieldsDetector -> {
                 ExtractedFields extractedFields = extractedFieldsDetector.detect().v1();
+
                 DataFrameDataExtractorFactory extractorFactory = new DataFrameDataExtractorFactory(client, config.getId(),
                     Collections.singletonList(config.getDest().getIndex()), config.getSource().getParsedQuery(), extractedFields,
                     config.getHeaders(), config.getAnalysis().supportsMissingValues());
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java
index b73f64e35283e..918999014d06c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
@@ -78,7 +79,7 @@ private void create(String[] index, DataFrameAnalyticsConfig config, ActionListe
         ActionListener<FieldCapabilitiesResponse> fieldCapabilitiesHandler = ActionListener.wrap(
             fieldCapabilitiesResponse -> {
                 fieldCapsResponseHolder.set(fieldCapabilitiesResponse);
-                getCardinalitiesForFieldsWithConstraints(index, config, fieldCardinalitiesHandler);
+                getCardinalitiesForFieldsWithConstraints(index, config, fieldCapabilitiesResponse, fieldCardinalitiesHandler);
             },
             listener::onFailure
         );
@@ -96,7 +97,9 @@ private void create(String[] index, DataFrameAnalyticsConfig config, ActionListe
         getDocValueFieldsLimit(index, docValueFieldsLimitListener);
     }
 
-    private void getCardinalitiesForFieldsWithConstraints(String[] index, DataFrameAnalyticsConfig config,
+    private void getCardinalitiesForFieldsWithConstraints(String[] index,
+                                                          DataFrameAnalyticsConfig config,
+                                                          FieldCapabilitiesResponse fieldCapabilitiesResponse,
                                                           ActionListener<Map<String, Long>> listener) {
         List<FieldCardinalityConstraint> fieldCardinalityConstraints = config.getAnalysis().getFieldCardinalityConstraints();
         if (fieldCardinalityConstraints.isEmpty()) {
@@ -111,6 +114,12 @@ private void getCardinalitiesForFieldsWithConstraints(String[] index, DataFrameA
 
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0).query(config.getSource().getParsedQuery());
         for (FieldCardinalityConstraint constraint : fieldCardinalityConstraints) {
+            for (FieldCapabilities fieldCaps : fieldCapabilitiesResponse.getField(constraint.getField()).values()) {
+                if (fieldCaps.isAggregatable() == false) {
+                    throw ExceptionsHelper.badRequestException("field [{}] of type [{}] is non-aggregatable",
+                        fieldCaps.getName(), fieldCaps.getType());
+                }
+            }
             searchSourceBuilder.aggregation(
                 AggregationBuilders.cardinality(constraint.getField())
                     .field(constraint.getField())
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
index c7baad202d2f6..c0bb2457bfb9d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
@@ -22,8 +23,9 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.MlStatsIndex;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -34,13 +36,16 @@
 import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitter;
 import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitterFactory;
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
+import org.elasticsearch.xpack.ml.dataframe.stats.DataCountsTracker;
 import org.elasticsearch.xpack.ml.dataframe.stats.ProgressTracker;
+import org.elasticsearch.xpack.ml.dataframe.stats.StatsPersister;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
 import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -156,7 +161,10 @@ private void processData(DataFrameAnalyticsTask task, ProcessContext processCont
         AnalyticsResultProcessor resultProcessor = processContext.resultProcessor.get();
         try {
             writeHeaderRecord(dataExtractor, process);
-            writeDataRows(dataExtractor, process, config.getAnalysis(), task.getStatsHolder().getProgressTracker());
+            writeDataRows(dataExtractor, process, config, task.getStatsHolder().getProgressTracker(),
+                task.getStatsHolder().getDataCountsTracker());
+            processContext.statsPersister.persistWithRetry(task.getStatsHolder().getDataCountsTracker().report(config.getId()),
+                DataCounts::documentId);
             process.writeEndOfDataMessage();
             process.flushStream();
 
@@ -170,7 +178,7 @@ private void processData(DataFrameAnalyticsTask task, ProcessContext processCont
             processContext.setFailureReason(resultProcessor.getFailure());
 
             refreshDest(config);
-            refreshStateIndex(config.getId());
+            refreshIndices(config.getId());
             LOGGER.info("[{}] Result processor has completed", config.getId());
         } catch (Exception e) {
             if (task.isStopping()) {
@@ -206,10 +214,11 @@ private void processData(DataFrameAnalyticsTask task, ProcessContext processCont
     }
 
     private void writeDataRows(DataFrameDataExtractor dataExtractor, AnalyticsProcess<AnalyticsResult> process,
-                               DataFrameAnalysis analysis, ProgressTracker progressTracker) throws IOException {
+                               DataFrameAnalyticsConfig config, ProgressTracker progressTracker, DataCountsTracker dataCountsTracker)
+        throws IOException {
 
-        CrossValidationSplitter crossValidationSplitter = new CrossValidationSplitterFactory(dataExtractor.getFieldNames())
-            .create(analysis);
+        CrossValidationSplitter crossValidationSplitter = new CrossValidationSplitterFactory(client, config, dataExtractor.getFieldNames())
+            .create();
 
         // The extra fields are for the doc hash and the control field (should be an empty string)
         String[] record = new String[dataExtractor.getFieldNames().size() + 2];
@@ -223,11 +232,14 @@ private void writeDataRows(DataFrameDataExtractor dataExtractor, AnalyticsProces
             Optional<List<DataFrameDataExtractor.Row>> rows = dataExtractor.next();
             if (rows.isPresent()) {
                 for (DataFrameDataExtractor.Row row : rows.get()) {
-                    if (row.shouldSkip() == false) {
+                    if (row.shouldSkip()) {
+                        dataCountsTracker.incrementSkippedDocsCount();
+                    } else {
                         String[] rowValues = row.getValues();
                         System.arraycopy(rowValues, 0, record, 0, rowValues.length);
                         record[record.length - 2] = String.valueOf(row.getChecksum());
-                        crossValidationSplitter.process(record);
+                        crossValidationSplitter.process(record, dataCountsTracker::incrementTrainingDocsCount,
+                            dataCountsTracker::incrementTestDocsCount);
                         process.writeRecord(record);
                     }
                 }
@@ -253,6 +265,10 @@ private void writeHeaderRecord(DataFrameDataExtractor dataExtractor, AnalyticsPr
         process.writeRecord(headerRecord);
     }
 
+    private void indexDataCounts(DataCounts dataCounts) {
+        IndexRequest indexRequest = new IndexRequest(MlStatsIndex.writeAlias());
+    }
+
     private void restoreState(DataFrameAnalyticsTask task, DataFrameAnalyticsConfig config, @Nullable BytesReference state,
                               AnalyticsProcess<AnalyticsResult> process) {
         if (config.getAnalysis().persistsState() == false) {
@@ -301,12 +317,16 @@ private void refreshDest(DataFrameAnalyticsConfig config) {
             () -> client.execute(RefreshAction.INSTANCE, new RefreshRequest(config.getDest().getIndex())).actionGet());
     }
 
-    private void refreshStateIndex(String jobId) {
-        String indexName = AnomalyDetectorsIndex.jobStateIndexPattern();
-        LOGGER.debug("[{}] Refresh index {}", jobId, indexName);
-
-        RefreshRequest refreshRequest = new RefreshRequest(indexName);
+    private void refreshIndices(String jobId) {
+        RefreshRequest refreshRequest = new RefreshRequest(
+            AnomalyDetectorsIndex.jobStateIndexPattern(),
+            MlStatsIndex.indexPattern()
+        );
         refreshRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
+
+        LOGGER.debug(() -> new ParameterizedMessage("[{}] Refreshing indices {}",
+            jobId, Arrays.toString(refreshRequest.indices())));
+
         try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) {
             client.admin().indices().refresh(refreshRequest).actionGet();
         }
@@ -353,9 +373,11 @@ class ProcessContext {
         private final SetOnce<DataFrameDataExtractor> dataExtractor = new SetOnce<>();
         private final SetOnce<AnalyticsResultProcessor> resultProcessor = new SetOnce<>();
         private final SetOnce<String> failureReason = new SetOnce<>();
+        private final StatsPersister statsPersister;
 
         ProcessContext(DataFrameAnalyticsConfig config) {
             this.config = Objects.requireNonNull(config);
+            this.statsPersister = new StatsPersister(config.getId(), resultsPersisterService, auditor);
         }
 
         String getFailureReason() {
@@ -378,6 +400,7 @@ synchronized void stop() {
             if (resultProcessor.get() != null) {
                 resultProcessor.get().cancel();
             }
+            statsPersister.cancel();
             if (process.get() != null) {
                 try {
                     process.get().kill();
@@ -434,7 +457,7 @@ private AnalyticsResultProcessor createResultProcessor(DataFrameAnalyticsTask ta
             DataFrameRowsJoiner dataFrameRowsJoiner =
                 new DataFrameRowsJoiner(config.getId(), dataExtractorFactory.newExtractor(true), resultsPersisterService);
             return new AnalyticsResultProcessor(
-                config, dataFrameRowsJoiner, task.getStatsHolder(), trainedModelProvider, auditor, resultsPersisterService,
+                config, dataFrameRowsJoiner, task.getStatsHolder(), trainedModelProvider, auditor, statsPersister,
                 dataExtractor.get().getAllExtractedFields());
         }
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
index eaa202df5fc4a..d27fa31b41193 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
@@ -11,14 +11,10 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
-import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.license.License;
-import org.elasticsearch.xpack.core.ml.MlStatsIndex;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
@@ -31,18 +27,16 @@
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
 import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
 import org.elasticsearch.xpack.ml.dataframe.stats.StatsHolder;
+import org.elasticsearch.xpack.ml.dataframe.stats.StatsPersister;
 import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 import org.elasticsearch.xpack.ml.extractor.MultiField;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
-import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
 
-import java.io.IOException;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.Iterator;
@@ -51,7 +45,6 @@
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.toList;
@@ -77,7 +70,7 @@ public class AnalyticsResultProcessor {
     private final StatsHolder statsHolder;
     private final TrainedModelProvider trainedModelProvider;
     private final DataFrameAnalyticsAuditor auditor;
-    private final ResultsPersisterService resultsPersisterService;
+    private final StatsPersister statsPersister;
     private final List<ExtractedField> fieldNames;
     private final CountDownLatch completionLatch = new CountDownLatch(1);
     private volatile String failure;
@@ -85,14 +78,13 @@ public class AnalyticsResultProcessor {
 
     public AnalyticsResultProcessor(DataFrameAnalyticsConfig analytics, DataFrameRowsJoiner dataFrameRowsJoiner,
                                     StatsHolder statsHolder, TrainedModelProvider trainedModelProvider,
-                                    DataFrameAnalyticsAuditor auditor, ResultsPersisterService resultsPersisterService,
-                                    List<ExtractedField> fieldNames) {
+                                    DataFrameAnalyticsAuditor auditor, StatsPersister statsPersister, List<ExtractedField> fieldNames) {
         this.analytics = Objects.requireNonNull(analytics);
         this.dataFrameRowsJoiner = Objects.requireNonNull(dataFrameRowsJoiner);
         this.statsHolder = Objects.requireNonNull(statsHolder);
         this.trainedModelProvider = Objects.requireNonNull(trainedModelProvider);
         this.auditor = Objects.requireNonNull(auditor);
-        this.resultsPersisterService = Objects.requireNonNull(resultsPersisterService);
+        this.statsPersister = Objects.requireNonNull(statsPersister);
         this.fieldNames = Collections.unmodifiableList(Objects.requireNonNull(fieldNames));
     }
 
@@ -112,6 +104,7 @@ public void awaitForCompletion() {
 
     public void cancel() {
         dataFrameRowsJoiner.cancel();
+        statsPersister.cancel();
         isCancelled = true;
     }
 
@@ -177,22 +170,22 @@ private void processResult(AnalyticsResult result, DataFrameRowsJoiner resultsJo
         MemoryUsage memoryUsage = result.getMemoryUsage();
         if (memoryUsage != null) {
             statsHolder.setMemoryUsage(memoryUsage);
-            indexStatsResult(memoryUsage, memoryUsage::documentId);
+            statsPersister.persistWithRetry(memoryUsage, memoryUsage::documentId);
         }
         OutlierDetectionStats outlierDetectionStats = result.getOutlierDetectionStats();
         if (outlierDetectionStats != null) {
             statsHolder.setAnalysisStats(outlierDetectionStats);
-            indexStatsResult(outlierDetectionStats, outlierDetectionStats::documentId);
+            statsPersister.persistWithRetry(outlierDetectionStats, outlierDetectionStats::documentId);
         }
         ClassificationStats classificationStats = result.getClassificationStats();
         if (classificationStats != null) {
             statsHolder.setAnalysisStats(classificationStats);
-            indexStatsResult(classificationStats, classificationStats::documentId);
+            statsPersister.persistWithRetry(classificationStats, classificationStats::documentId);
         }
         RegressionStats regressionStats = result.getRegressionStats();
         if (regressionStats != null) {
             statsHolder.setAnalysisStats(regressionStats);
-            indexStatsResult(regressionStats, regressionStats::documentId);
+            statsPersister.persistWithRetry(regressionStats, regressionStats::documentId);
         }
     }
 
@@ -275,23 +268,4 @@ private void setAndReportFailure(Exception e) {
         failure = "error processing results; " + e.getMessage();
         auditor.error(analytics.getId(), "Error processing results; " + e.getMessage());
     }
-
-    private void indexStatsResult(ToXContentObject result, Function<String, String> docIdSupplier) {
-        try {
-            resultsPersisterService.indexWithRetry(analytics.getId(),
-                MlStatsIndex.writeAlias(),
-                result,
-                new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")),
-                WriteRequest.RefreshPolicy.IMMEDIATE,
-                docIdSupplier.apply(analytics.getId()),
-                () -> isCancelled == false,
-                errorMsg -> auditor.error(analytics.getId(),
-                    "failed to persist result with id [" + docIdSupplier.apply(analytics.getId()) + "]; " + errorMsg)
-            );
-        } catch (IOException ioe) {
-            LOGGER.error(() -> new ParameterizedMessage("[{}] Failed serializing stats result", analytics.getId()), ioe);
-        } catch (Exception e) {
-            LOGGER.error(() -> new ParameterizedMessage("[{}] Failed indexing stats result", analytics.getId()), e);
-        }
-    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java
index 5d12a2a81a607..fce602b28e2c5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java
@@ -10,5 +10,5 @@
  */
 public interface CrossValidationSplitter {
 
-    void process(String[] row);
+    void process(String[] row, Runnable incrementTrainingDocs, Runnable incrementTestDocs);
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java
index 47c052dd0bf84..7632916c0d4df 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java
@@ -5,32 +5,81 @@
  */
 package org.elasticsearch.xpack.ml.dataframe.process.crossvalidation;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 public class CrossValidationSplitterFactory {
 
+    private static final Logger LOGGER = LogManager.getLogger(CrossValidationSplitterFactory.class);
+
+    private final Client client;
+    private final DataFrameAnalyticsConfig config;
     private final List<String> fieldNames;
 
-    public CrossValidationSplitterFactory(List<String> fieldNames) {
+    public CrossValidationSplitterFactory(Client client, DataFrameAnalyticsConfig config, List<String> fieldNames) {
+        this.client = Objects.requireNonNull(client);
+        this.config = Objects.requireNonNull(config);
         this.fieldNames = Objects.requireNonNull(fieldNames);
     }
 
-    public CrossValidationSplitter create(DataFrameAnalysis analysis) {
-        if (analysis instanceof Regression) {
-            Regression regression = (Regression) analysis;
-            return new RandomCrossValidationSplitter(
-                fieldNames, regression.getDependentVariable(), regression.getTrainingPercent(), regression.getRandomizeSeed());
+    public CrossValidationSplitter create() {
+        if (config.getAnalysis() instanceof Regression) {
+            return createRandomSplitter();
+        }
+        if (config.getAnalysis() instanceof Classification) {
+            return createStratifiedSplitter((Classification) config.getAnalysis());
         }
-        if (analysis instanceof Classification) {
-            Classification classification = (Classification) analysis;
-            return new RandomCrossValidationSplitter(
-                fieldNames, classification.getDependentVariable(), classification.getTrainingPercent(), classification.getRandomizeSeed());
+        return (row, incrementTrainingDocs, incrementTestDocs) -> incrementTrainingDocs.run();
+    }
+
+    private CrossValidationSplitter createRandomSplitter() {
+        Regression regression = (Regression) config.getAnalysis();
+        return new RandomCrossValidationSplitter(
+            fieldNames, regression.getDependentVariable(), regression.getTrainingPercent(), regression.getRandomizeSeed());
+    }
+
+    private CrossValidationSplitter createStratifiedSplitter(Classification classification) {
+        String aggName = "dependent_variable_terms";
+        SearchRequestBuilder searchRequestBuilder = client.prepareSearch(config.getDest().getIndex())
+            .setSize(0)
+            .setAllowPartialSearchResults(false)
+            .addAggregation(AggregationBuilders.terms(aggName)
+                .field(classification.getDependentVariable())
+                .size(Classification.MAX_DEPENDENT_VARIABLE_CARDINALITY));
+
+        try {
+            SearchResponse searchResponse = ClientHelper.executeWithHeaders(config.getHeaders(), ClientHelper.ML_ORIGIN, client,
+                searchRequestBuilder::get);
+            Aggregations aggs = searchResponse.getAggregations();
+            Terms terms = aggs.get(aggName);
+            Map<String, Long> classCardinalities = new HashMap<>();
+            for (Terms.Bucket bucket : terms.getBuckets()) {
+                classCardinalities.put(String.valueOf(bucket.getKey()), bucket.getDocCount());
+            }
+
+            return new StratifiedCrossValidationSplitter(fieldNames, classification.getDependentVariable(), classCardinalities,
+                classification.getTrainingPercent(), classification.getRandomizeSeed());
+        } catch (Exception e) {
+            ParameterizedMessage msg = new ParameterizedMessage("[{}] Dependent variable terms search failed", config.getId());
+            LOGGER.error(msg, e);
+            throw new ElasticsearchException(msg.getFormattedMessage(), e);
         }
-        return row -> {};
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java
index 0afc59628e7de..2bd0a209848fb 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitter.java
@@ -25,37 +25,40 @@ class RandomCrossValidationSplitter implements CrossValidationSplitter {
     private boolean isFirstRow = true;
 
     RandomCrossValidationSplitter(List<String> fieldNames, String dependentVariable, double trainingPercent, long randomizeSeed) {
+        assert trainingPercent >= 1.0 && trainingPercent <= 100.0;
         this.dependentVariableIndex = findDependentVariableIndex(fieldNames, dependentVariable);
         this.trainingPercent = trainingPercent;
         this.random = new Random(randomizeSeed);
     }
 
     private static int findDependentVariableIndex(List<String> fieldNames, String dependentVariable) {
-        for (int i = 0; i < fieldNames.size(); i++) {
-            if (fieldNames.get(i).equals(dependentVariable)) {
-                return i;
-            }
+        int dependentVariableIndex = fieldNames.indexOf(dependentVariable);
+        if (dependentVariableIndex < 0) {
+            throw ExceptionsHelper.serverError("Could not find dependent variable [" + dependentVariable + "] in fields " + fieldNames);
         }
-        throw ExceptionsHelper.serverError("Could not find dependent variable [" + dependentVariable + "] in fields " + fieldNames);
+        return dependentVariableIndex;
     }
 
     @Override
-    public void process(String[] row) {
-        if (canBeUsedForTraining(row)) {
-            if (isFirstRow) {
-                // Let's make sure we have at least one training row
-                isFirstRow = false;
-            } else if (isRandomlyExcludedFromTraining()) {
-                row[dependentVariableIndex] = DataFrameDataExtractor.NULL_VALUE;
-            }
+    public void process(String[] row, Runnable incrementTrainingDocs, Runnable incrementTestDocs) {
+        if (canBeUsedForTraining(row) && isPickedForTraining()) {
+            incrementTrainingDocs.run();
+        } else {
+            row[dependentVariableIndex] = DataFrameDataExtractor.NULL_VALUE;
+            incrementTestDocs.run();
         }
     }
 
     private boolean canBeUsedForTraining(String[] row) {
-        return row[dependentVariableIndex].length() > 0;
+        return row[dependentVariableIndex] != DataFrameDataExtractor.NULL_VALUE;
     }
 
-    private boolean isRandomlyExcludedFromTraining() {
-        return random.nextDouble() * 100 > trainingPercent;
+    private boolean isPickedForTraining() {
+        if (isFirstRow) {
+            // Let's make sure we have at least one training row
+            isFirstRow = false;
+            return true;
+        }
+        return random.nextDouble() * 100 <= trainingPercent;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitter.java
new file mode 100644
index 0000000000000..a46b50d933ff7
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitter.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ml.dataframe.process.crossvalidation;
+
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+/**
+ * Given a dependent variable, randomly splits the dataset trying
+ * to preserve the proportion of each class in the training sample.
+ */
+public class StratifiedCrossValidationSplitter implements CrossValidationSplitter {
+
+    private final int dependentVariableIndex;
+    private final double samplingRatio;
+    private final Random random;
+    private final Map<String, ClassSample> classSamples;
+
+    public StratifiedCrossValidationSplitter(List<String> fieldNames, String dependentVariable, Map<String, Long> classCardinalities,
+                                             double trainingPercent, long randomizeSeed) {
+        assert trainingPercent >= 1.0 && trainingPercent <= 100.0;
+        this.dependentVariableIndex = findDependentVariableIndex(fieldNames, dependentVariable);
+        this.samplingRatio = trainingPercent / 100.0;
+        this.random = new Random(randomizeSeed);
+        this.classSamples = new HashMap<>();
+        classCardinalities.entrySet().forEach(entry -> classSamples.put(entry.getKey(), new ClassSample(entry.getValue())));
+    }
+
+    private static int findDependentVariableIndex(List<String> fieldNames, String dependentVariable) {
+        int dependentVariableIndex = fieldNames.indexOf(dependentVariable);
+        if (dependentVariableIndex < 0) {
+            throw ExceptionsHelper.serverError("Could not find dependent variable [" + dependentVariable + "] in fields " + fieldNames);
+        }
+        return dependentVariableIndex;
+    }
+
+    @Override
+    public void process(String[] row, Runnable incrementTrainingDocs, Runnable incrementTestDocs) {
+
+        if (canBeUsedForTraining(row) == false) {
+            incrementTestDocs.run();
+            return;
+        }
+
+        String classValue = row[dependentVariableIndex];
+        ClassSample sample = classSamples.get(classValue);
+        if (sample == null) {
+            throw new IllegalStateException("Unknown class [" + classValue + "]; expected one of " + classSamples.keySet());
+        }
+
+        // The idea here is that the probability increases as the chances we have to get the target proportion
+        // for a class decreases.
+        double p = (samplingRatio * sample.cardinality - sample.training) / (sample.cardinality - sample.observed);
+
+        boolean isTraining = random.nextDouble() <= p;
+
+        sample.observed++;
+        if (isTraining) {
+            sample.training++;
+            incrementTrainingDocs.run();
+        } else {
+            row[dependentVariableIndex] = DataFrameDataExtractor.NULL_VALUE;
+            incrementTestDocs.run();
+        }
+    }
+
+    private boolean canBeUsedForTraining(String[] row) {
+        return row[dependentVariableIndex] != DataFrameDataExtractor.NULL_VALUE;
+    }
+
+    private static class ClassSample {
+
+        private final long cardinality;
+        private long training;
+        private long observed;
+
+        private ClassSample(long cardinality) {
+            this.cardinality = cardinality;
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/DataCountsTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/DataCountsTracker.java
new file mode 100644
index 0000000000000..bed9f52b448cf
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/DataCountsTracker.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ml.dataframe.stats;
+
+import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
+
+public class DataCountsTracker {
+
+    private volatile long trainingDocsCount;
+    private volatile long testDocsCount;
+    private volatile long skippedDocsCount;
+
+    public void incrementTrainingDocsCount() {
+        trainingDocsCount++;
+    }
+
+    public void incrementTestDocsCount() {
+        testDocsCount++;
+    }
+
+    public void incrementSkippedDocsCount() {
+        skippedDocsCount++;
+    }
+
+    public DataCounts report(String jobId) {
+        return new DataCounts(
+            jobId,
+            trainingDocsCount,
+            testDocsCount,
+            skippedDocsCount
+        );
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java
index ff6b9ec7bcfef..d01eee6a3a3d8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java
@@ -19,11 +19,13 @@ public class StatsHolder {
     private final ProgressTracker progressTracker;
     private final AtomicReference<MemoryUsage> memoryUsageHolder;
     private final AtomicReference<AnalysisStats> analysisStatsHolder;
+    private final DataCountsTracker dataCountsTracker;
 
     public StatsHolder() {
         progressTracker = new ProgressTracker();
         memoryUsageHolder = new AtomicReference<>();
         analysisStatsHolder = new AtomicReference<>();
+        dataCountsTracker = new DataCountsTracker();
     }
 
     public ProgressTracker getProgressTracker() {
@@ -45,4 +47,8 @@ public void setAnalysisStats(AnalysisStats analysisStats) {
     public AnalysisStats getAnalysisStats() {
         return analysisStatsHolder.get();
     }
+
+    public DataCountsTracker getDataCountsTracker() {
+        return dataCountsTracker;
+    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java
new file mode 100644
index 0000000000000..e553ac1810729
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsPersister.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ml.dataframe.stats;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.xpack.core.ml.MlStatsIndex;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
+import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.function.Function;
+
+public class StatsPersister {
+
+    private static final Logger LOGGER = LogManager.getLogger(StatsPersister.class);
+
+    private final String jobId;
+    private final ResultsPersisterService resultsPersisterService;
+    private final DataFrameAnalyticsAuditor auditor;
+    private volatile boolean isCancelled;
+
+    public StatsPersister(String jobId, ResultsPersisterService resultsPersisterService, DataFrameAnalyticsAuditor auditor) {
+        this.jobId = Objects.requireNonNull(jobId);
+        this.resultsPersisterService = Objects.requireNonNull(resultsPersisterService);
+        this.auditor = Objects.requireNonNull(auditor);
+    }
+
+    public void persistWithRetry(ToXContentObject result, Function<String, String> docIdSupplier) {
+        if (isCancelled) {
+            return;
+        }
+
+        try {
+            resultsPersisterService.indexWithRetry(jobId,
+                MlStatsIndex.writeAlias(),
+                result,
+                new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")),
+                WriteRequest.RefreshPolicy.NONE,
+                docIdSupplier.apply(jobId),
+                () -> isCancelled == false,
+                errorMsg -> auditor.error(jobId,
+                    "failed to persist result with id [" + docIdSupplier.apply(jobId) + "]; " + errorMsg)
+            );
+        } catch (IOException ioe) {
+            LOGGER.error(() -> new ParameterizedMessage("[{}] Failed serializing stats result", jobId), ioe);
+        } catch (Exception e) {
+            LOGGER.error(() -> new ParameterizedMessage("[{}] Failed indexing stats result", jobId), e);
+        }
+    }
+
+    public void cancel() {
+        isCancelled = true;
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java
index ea10c9fb5f64c..30118104e6510 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryActionTests.java
@@ -36,7 +36,7 @@ public void testCalculateDetectorRequirementBytes() {
 
         Detector withByField = createDetector(function, "field", "buy", null, null);
         assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(withByField,
-            overallCardinality), is(200 * 65536L));
+            overallCardinality), is(134 * 65536L));
 
         Detector withPartitionField = createDetector(function, "field", null, null, "part");
         assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(withPartitionField,
@@ -44,7 +44,7 @@ public void testCalculateDetectorRequirementBytes() {
 
         Detector withByAndPartitionFields = createDetector(function, "field", "buy", null, "part");
         assertThat(TransportEstimateModelMemoryAction.calculateDetectorRequirementBytes(withByAndPartitionFields,
-            overallCardinality), is(200 * 100 * 65536L));
+            overallCardinality), is(134 * 100 * 65536L));
     }
 
     public void testCalculateInfluencerRequirementBytes() {
@@ -98,6 +98,10 @@ public void testRoundUpToNextMb() {
             equalTo(new ByteSizeValue(2, ByteSizeUnit.MB)));
         assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(2 * 1024 * 1024),
             equalTo(new ByteSizeValue(2, ByteSizeUnit.MB)));
+        // We don't round up at the extremes, to ensure that the resulting value can be represented as bytes in a long
+        // (At such extreme scale it won't be possible to actually run the analysis, so ease of use trumps precision)
+        assertThat(TransportEstimateModelMemoryAction.roundUpToNextMb(Long.MAX_VALUE - randomIntBetween(0, 1000000)),
+            equalTo(new ByteSizeValue(Long.MAX_VALUE / new ByteSizeValue(1, ByteSizeUnit.MB).getBytes() , ByteSizeUnit.MB)));
     }
 
     public static Detector createDetector(String function, String fieldName, String byFieldName,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java
index ac8657575e2dd..9646fda946aac 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java
@@ -11,9 +11,9 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.mock.orig.Mockito;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
@@ -207,7 +207,7 @@ public void testBuild_GivenBucketsRequestFails() {
     }
 
     public void testBuildGivenRemoteIndicesButNoRemoteSearching() throws Exception {
-        Settings settings = Settings.builder().put(RemoteClusterService.ENABLE_REMOTE_CLUSTERS.getKey(), false).build();
+        Settings settings = Settings.builder().put(Node.NODE_REMOTE_CLUSTER_CLIENT.getKey(), false).build();
         datafeedJobBuilder =
             new DatafeedJobBuilder(
                 client,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
index b75392e03c2bb..01661f6ec82a6 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
@@ -324,9 +324,46 @@ public void testIncludeSourceIsFalseAndAtLeastOneSourceField() throws IOExceptio
         assertThat(searchRequest, containsString("\"_source\":{\"includes\":[\"field_2\"],\"excludes\":[]}"));
     }
 
-    public void testMissingValues_GivenShouldNotInclude() throws IOException {
+    public void testCollectDataSummary_GivenAnalysisSupportsMissingFields() {
+        TestExtractor dataExtractor = createExtractor(true, true);
+
+        // First and only batch
+        SearchResponse response = createSearchResponse(Arrays.asList(1_1, 1_2), Arrays.asList(2_1, 2_2));
+        dataExtractor.setNextResponse(response);
+
+        DataFrameDataExtractor.DataSummary dataSummary = dataExtractor.collectDataSummary();
+
+        assertThat(dataSummary.rows, equalTo(2L));
+        assertThat(dataSummary.cols, equalTo(2));
+
+        assertThat(dataExtractor.capturedSearchRequests.size(), equalTo(1));
+        String searchRequest = dataExtractor.capturedSearchRequests.get(0).request().toString().replaceAll("\\s", "");
+        assertThat(searchRequest, containsString("\"query\":{\"match_all\":{\"boost\":1.0}}"));
+    }
+
+    public void testCollectDataSummary_GivenAnalysisDoesNotSupportMissingFields() {
         TestExtractor dataExtractor = createExtractor(true, false);
 
+        // First and only batch
+        SearchResponse response = createSearchResponse(Arrays.asList(1_1, 1_2), Arrays.asList(2_1, 2_2));
+        dataExtractor.setNextResponse(response);
+
+        DataFrameDataExtractor.DataSummary dataSummary = dataExtractor.collectDataSummary();
+
+        assertThat(dataSummary.rows, equalTo(2L));
+        assertThat(dataSummary.cols, equalTo(2));
+
+        assertThat(dataExtractor.capturedSearchRequests.size(), equalTo(1));
+        String searchRequest = dataExtractor.capturedSearchRequests.get(0).request().toString().replaceAll("\\s", "");
+        assertThat(searchRequest, containsString(
+            "\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}},{\"bool\":{\"filter\":" +
+                "[{\"exists\":{\"field\":\"field_1\",\"boost\":1.0}},{\"exists\":{\"field\":\"field_2\",\"boost\":1.0}}]," +
+                "\"boost\":1.0}}],\"boost\":1.0}"));
+    }
+
+    public void testMissingValues_GivenSupported() throws IOException {
+        TestExtractor dataExtractor = createExtractor(true, true);
+
         // First and only batch
         SearchResponse response1 = createSearchResponse(Arrays.asList(1_1, null, 1_3), Arrays.asList(2_1, 2_2, 2_3));
         dataExtractor.setNextResponse(response1);
@@ -343,11 +380,12 @@ public void testMissingValues_GivenShouldNotInclude() throws IOException {
         assertThat(rows.get().size(), equalTo(3));
 
         assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
-        assertThat(rows.get().get(1).getValues(), is(nullValue()));
+        assertThat(rows.get().get(1).getValues()[0], equalTo(DataFrameDataExtractor.NULL_VALUE));
+        assertThat(rows.get().get(1).getValues()[1], equalTo("22"));
         assertThat(rows.get().get(2).getValues(), equalTo(new String[] {"13", "23"}));
 
         assertThat(rows.get().get(0).shouldSkip(), is(false));
-        assertThat(rows.get().get(1).shouldSkip(), is(true));
+        assertThat(rows.get().get(1).shouldSkip(), is(false));
         assertThat(rows.get().get(2).shouldSkip(), is(false));
 
         assertThat(dataExtractor.hasNext(), is(true));
@@ -358,8 +396,8 @@ public void testMissingValues_GivenShouldNotInclude() throws IOException {
         assertThat(dataExtractor.hasNext(), is(false));
     }
 
-    public void testMissingValues_GivenShouldInclude() throws IOException {
-        TestExtractor dataExtractor = createExtractor(true, true);
+    public void testMissingValues_GivenNotSupported() throws IOException {
+        TestExtractor dataExtractor = createExtractor(true, false);
 
         // First and only batch
         SearchResponse response1 = createSearchResponse(Arrays.asList(1_1, null, 1_3), Arrays.asList(2_1, 2_2, 2_3));
@@ -377,12 +415,11 @@ public void testMissingValues_GivenShouldInclude() throws IOException {
         assertThat(rows.get().size(), equalTo(3));
 
         assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
-        assertThat(rows.get().get(1).getValues()[0], equalTo(DataFrameDataExtractor.NULL_VALUE));
-        assertThat(rows.get().get(1).getValues()[1], equalTo("22"));
+        assertThat(rows.get().get(1).getValues(), is(nullValue()));
         assertThat(rows.get().get(2).getValues(), equalTo(new String[] {"13", "23"}));
 
         assertThat(rows.get().get(0).shouldSkip(), is(false));
-        assertThat(rows.get().get(1).shouldSkip(), is(false));
+        assertThat(rows.get().get(1).shouldSkip(), is(true));
         assertThat(rows.get().get(2).shouldSkip(), is(false));
 
         assertThat(dataExtractor.hasNext(), is(true));
@@ -424,9 +461,9 @@ public void testGetCategoricalFields() {
             containsInAnyOrder("field_keyword", "field_text", "field_boolean"));
     }
 
-    private TestExtractor createExtractor(boolean includeSource, boolean includeRowsWithMissingValues) {
+    private TestExtractor createExtractor(boolean includeSource, boolean supportsRowsWithMissingValues) {
         DataFrameDataExtractorContext context = new DataFrameDataExtractorContext(
-            JOB_ID, extractedFields, indices, query, scrollSize, headers, includeSource, includeRowsWithMissingValues);
+            JOB_ID, extractedFields, indices, query, scrollSize, headers, includeSource, supportsRowsWithMissingValues);
         return new TestExtractor(client, context);
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
index da7bbff71d5a7..ee2b399ef2d48 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
@@ -24,13 +24,13 @@
 import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
 import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
 import org.elasticsearch.xpack.ml.dataframe.stats.StatsHolder;
+import org.elasticsearch.xpack.ml.dataframe.stats.StatsPersister;
 import org.elasticsearch.xpack.ml.extractor.DocValueField;
 import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
 import org.elasticsearch.xpack.ml.extractor.MultiField;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
-import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
 import org.mockito.InOrder;
@@ -66,7 +66,7 @@ public class AnalyticsResultProcessorTests extends ESTestCase {
     private StatsHolder statsHolder = new StatsHolder();
     private TrainedModelProvider trainedModelProvider;
     private DataFrameAnalyticsAuditor auditor;
-    private ResultsPersisterService resultsPersisterService;
+    private StatsPersister statsPersister;
     private DataFrameAnalyticsConfig analyticsConfig;
 
     @Before
@@ -76,7 +76,7 @@ public void setUpMocks() {
         dataFrameRowsJoiner = mock(DataFrameRowsJoiner.class);
         trainedModelProvider = mock(TrainedModelProvider.class);
         auditor = mock(DataFrameAnalyticsAuditor.class);
-        resultsPersisterService = mock(ResultsPersisterService.class);
+        statsPersister = mock(StatsPersister.class);
         analyticsConfig = new DataFrameAnalyticsConfig.Builder()
             .setId(JOB_ID)
             .setDescription(JOB_DESCRIPTION)
@@ -251,7 +251,7 @@ private AnalyticsResultProcessor createResultProcessor(List<ExtractedField> fiel
             statsHolder,
             trainedModelProvider,
             auditor,
-            resultsPersisterService,
+            statsPersister,
             fieldNames);
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java
index eea102e673893..0bbc9d75d8be5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/RandomCrossValidationSplitterTests.java
@@ -26,6 +26,8 @@ public class RandomCrossValidationSplitterTests extends ESTestCase {
     private int dependentVariableIndex;
     private String dependentVariable;
     private long randomizeSeed;
+    private long trainingDocsCount;
+    private long testDocsCount;
 
     @Before
     public void setUpTests() {
@@ -40,47 +42,48 @@ public void setUpTests() {
     }
 
     public void testProcess_GivenRowsWithoutDependentVariableValue() {
-        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(fields, dependentVariable, 50.0, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = createSplitter(50.0);
 
         for (int i = 0; i < 100; i++) {
             String[] row = new String[fields.size()];
             for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
-                String value = fieldIndex == dependentVariableIndex ? "" : randomAlphaOfLength(10);
+                String value = fieldIndex == dependentVariableIndex ? DataFrameDataExtractor.NULL_VALUE : randomAlphaOfLength(10);
                 row[fieldIndex] = value;
             }
 
             String[] processedRow = Arrays.copyOf(row, row.length);
-            crossValidationSplitter.process(processedRow);
+            crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
 
             // As all these rows have no dependent variable value, they're not for training and should be unaffected
             assertThat(Arrays.equals(processedRow, row), is(true));
         }
+        assertThat(trainingDocsCount, equalTo(0L));
+        assertThat(testDocsCount, equalTo(100L));
     }
 
     public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsHundred() {
-        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(
-            fields, dependentVariable, 100.0, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = createSplitter(100.0);
 
         for (int i = 0; i < 100; i++) {
             String[] row = new String[fields.size()];
             for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
-                String value = fieldIndex == dependentVariableIndex ? "" : randomAlphaOfLength(10);
-                row[fieldIndex] = value;
+                row[fieldIndex] = randomAlphaOfLength(10);
             }
 
             String[] processedRow = Arrays.copyOf(row, row.length);
-            crossValidationSplitter.process(processedRow);
+            crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
 
             // We should pick them all as training percent is 100
             assertThat(Arrays.equals(processedRow, row), is(true));
         }
+        assertThat(trainingDocsCount, equalTo(100L));
+        assertThat(testDocsCount, equalTo(0L));
     }
 
     public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsRandom() {
         double trainingPercent = randomDoubleBetween(1.0, 100.0, true);
         double trainingFraction = trainingPercent / 100;
-        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(
-            fields, dependentVariable, trainingPercent, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = createSplitter(trainingPercent);
 
         int runCount = 20;
         int rowsCount = 1000;
@@ -94,7 +97,7 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs
                 }
 
                 String[] processedRow = Arrays.copyOf(row, row.length);
-                crossValidationSplitter.process(processedRow);
+                crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
 
                 for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
                     if (fieldIndex != dependentVariableIndex) {
@@ -126,8 +129,7 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs
     }
 
     public void testProcess_ShouldHaveAtLeastOneTrainingRow() {
-        CrossValidationSplitter crossValidationSplitter = new RandomCrossValidationSplitter(
-            fields, dependentVariable, 1.0, randomizeSeed);
+        CrossValidationSplitter crossValidationSplitter = createSplitter(1.0);
 
         // We have some non-training rows and then a training row to check
         // we maintain the first training row and not just the first row
@@ -135,16 +137,30 @@ public void testProcess_ShouldHaveAtLeastOneTrainingRow() {
             String[] row = new String[fields.size()];
             for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
                 if (i < 9 && fieldIndex == dependentVariableIndex) {
-                    row[fieldIndex] = "";
+                    row[fieldIndex] = DataFrameDataExtractor.NULL_VALUE;
                 } else {
                     row[fieldIndex] = randomAlphaOfLength(10);
                 }
             }
 
             String[] processedRow = Arrays.copyOf(row, row.length);
-            crossValidationSplitter.process(processedRow);
+            crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
 
             assertThat(Arrays.equals(processedRow, row), is(true));
         }
+        assertThat(trainingDocsCount, equalTo(1L));
+        assertThat(testDocsCount, equalTo(9L));
+    }
+
+    private RandomCrossValidationSplitter createSplitter(double trainingPercent) {
+        return new RandomCrossValidationSplitter(fields, dependentVariable, trainingPercent, randomizeSeed);
+    }
+
+    private void incrementTrainingDocsCount() {
+        trainingDocsCount++;
+    }
+
+    private void incrementTestDocsCount() {
+        testDocsCount++;
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitterTests.java
new file mode 100644
index 0000000000000..9ef0f773ee852
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitterTests.java
@@ -0,0 +1,239 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ml.dataframe.process.crossvalidation;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class StratifiedCrossValidationSplitterTests extends ESTestCase {
+
+    private static final int ROWS_COUNT = 500;
+
+    private List<String> fields;
+    private int dependentVariableIndex;
+    private String dependentVariable;
+    private long randomizeSeed;
+    private Map<String, Long> classCardinalities;
+    private String[] classValuesPerRow;
+    private long trainingDocsCount;
+    private long testDocsCount;
+
+    @Before
+    public void setUpTests() {
+        int fieldCount = randomIntBetween(1, 5);
+        fields = new ArrayList<>(fieldCount);
+        for (int i = 0; i < fieldCount; i++) {
+            fields.add(randomAlphaOfLength(10));
+        }
+        dependentVariableIndex = randomIntBetween(0, fieldCount - 1);
+        dependentVariable = fields.get(dependentVariableIndex);
+        randomizeSeed = randomLong();
+
+        long classA = 0;
+        long classB = 0;
+        long classC = 0;
+
+
+        classValuesPerRow = new String[ROWS_COUNT];
+        for (int i = 0; i < classValuesPerRow.length; i++) {
+            double randomDouble = randomDoubleBetween(0.0, 1.0, true);
+            if (randomDouble < 0.2) {
+                classValuesPerRow[i] = "a";
+                classA++;
+            } else if (randomDouble < 0.5) {
+                classValuesPerRow[i] = "b";
+                classB++;
+            } else {
+                classValuesPerRow[i] = "c";
+                classC++;
+            }
+        }
+
+        classCardinalities = new HashMap<>();
+        classCardinalities.put("a", classA);
+        classCardinalities.put("b", classB);
+        classCardinalities.put("c", classC);
+    }
+
+    public void testConstructor_GivenMissingDependentVariable() {
+        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> new StratifiedCrossValidationSplitter(
+            Collections.emptyList(), "foo", Collections.emptyMap(), 100.0, 0));
+        assertThat(e.getMessage(), equalTo("Could not find dependent variable [foo] in fields []"));
+    }
+
+    public void testProcess_GivenUnknownClass() {
+        CrossValidationSplitter splitter = createSplitter(100.0);
+        String[] row = new String[fields.size()];
+        for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
+            row[fieldIndex] = randomAlphaOfLength(5);
+        }
+        row[dependentVariableIndex] = "unknown_class";
+
+        IllegalStateException e = expectThrows(IllegalStateException.class,
+            () -> splitter.process(row, this::incrementTrainingDocsCount, this::incrementTestDocsCount));
+
+        assertThat(e.getMessage(), equalTo("Unknown class [unknown_class]; expected one of [a, b, c]"));
+    }
+
+    public void testProcess_GivenRowsWithoutDependentVariableValue() {
+        CrossValidationSplitter splitter = createSplitter(50.0);
+
+        for (int i = 0; i < classValuesPerRow.length; i++) {
+            String[] row = new String[fields.size()];
+            for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
+                String value = fieldIndex == dependentVariableIndex ? DataFrameDataExtractor.NULL_VALUE : randomAlphaOfLength(10);
+                row[fieldIndex] = value;
+            }
+
+            String[] processedRow = Arrays.copyOf(row, row.length);
+            splitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
+
+            // As all these rows have no dependent variable value, they're not for training and should be unaffected
+            assertThat(Arrays.equals(processedRow, row), is(true));
+        }
+        assertThat(trainingDocsCount, equalTo(0L));
+        assertThat(testDocsCount, equalTo(500L));
+    }
+
+    public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsHundred() {
+        CrossValidationSplitter splitter = createSplitter(100.0);
+
+        for (int i = 0; i < classValuesPerRow.length; i++) {
+            String[] row = new String[fields.size()];
+            for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
+                String value = fieldIndex == dependentVariableIndex ? classValuesPerRow[i] : randomAlphaOfLength(10);
+                row[fieldIndex] = value;
+            }
+
+            String[] processedRow = Arrays.copyOf(row, row.length);
+            splitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
+
+            // As training percent is 100 all rows should be unaffected
+            assertThat(Arrays.equals(processedRow, row), is(true));
+        }
+        assertThat(trainingDocsCount, equalTo(500L));
+        assertThat(testDocsCount, equalTo(0L));
+    }
+
+    public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsRandom() {
+        // We don't go too low here to avoid flakiness
+        double trainingPercent = randomDoubleBetween(50.0, 100.0, true);
+
+        CrossValidationSplitter splitter = createSplitter(trainingPercent);
+
+        Map<String, Integer> totalRowsPerClass = new HashMap<>();
+        Map<String, Integer> trainingRowsPerClass = new HashMap<>();
+
+        for (String classValue : classCardinalities.keySet()) {
+            totalRowsPerClass.put(classValue, 0);
+            trainingRowsPerClass.put(classValue, 0);
+        }
+
+        for (int i = 0; i < classValuesPerRow.length; i++) {
+            String[] row = new String[fields.size()];
+            for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
+                String value = fieldIndex == dependentVariableIndex ? classValuesPerRow[i] : randomAlphaOfLength(10);
+                row[fieldIndex] = value;
+            }
+
+            String[] processedRow = Arrays.copyOf(row, row.length);
+            splitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
+
+            for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
+                if (fieldIndex != dependentVariableIndex) {
+                    assertThat(processedRow[fieldIndex], equalTo(row[fieldIndex]));
+                }
+            }
+
+            String classValue = row[dependentVariableIndex];
+            totalRowsPerClass.compute(classValue, (k, v) -> v + 1);
+
+            if (DataFrameDataExtractor.NULL_VALUE.equals(processedRow[dependentVariableIndex]) == false) {
+                assertThat(processedRow[dependentVariableIndex], equalTo(row[dependentVariableIndex]));
+                trainingRowsPerClass.compute(classValue, (k, v) -> v + 1);
+            }
+        }
+
+        double trainingFraction = trainingPercent / 100;
+
+        // We can assert we're plus/minus 1 from rounding error
+
+        double expectedTotalTrainingCount = ROWS_COUNT * trainingFraction;
+        assertThat(trainingDocsCount + testDocsCount, equalTo((long) ROWS_COUNT));
+        assertThat(trainingDocsCount, greaterThanOrEqualTo((long) (expectedTotalTrainingCount - 2)));
+        assertThat(trainingDocsCount, lessThanOrEqualTo((long) Math.ceil(expectedTotalTrainingCount) + 2));
+
+        for (String classValue : classCardinalities.keySet()) {
+            double expectedClassTrainingCount = totalRowsPerClass.get(classValue) * trainingFraction;
+            int classTrainingCount = trainingRowsPerClass.get(classValue);
+            assertThat((double) classTrainingCount, is(closeTo(expectedClassTrainingCount, 1.0)));
+        }
+    }
+
+    public void testProcess_SelectsTrainingRowsUniformly() {
+        double trainingPercent = 50.0;
+        int runCount = 500;
+
+        int[] trainingCountPerRow = new int[ROWS_COUNT];
+
+        for (int run = 0; run < runCount; run++) {
+
+            randomizeSeed = randomLong();
+            CrossValidationSplitter crossValidationSplitter = createSplitter(trainingPercent);
+
+            for (int i = 0; i < classValuesPerRow.length; i++) {
+                String[] row = new String[fields.size()];
+                for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) {
+                    String value = fieldIndex == dependentVariableIndex ? classValuesPerRow[i] : randomAlphaOfLength(10);
+                    row[fieldIndex] = value;
+                }
+
+                String[] processedRow = Arrays.copyOf(row, row.length);
+                crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount);
+
+                if (processedRow[dependentVariableIndex] != DataFrameDataExtractor.NULL_VALUE) {
+                    trainingCountPerRow[i]++;
+                }
+            }
+        }
+
+        // We expect each data row to be selected uniformly.
+        // Thus the fraction of the row count where it's selected for training against the number of runs
+        // should be close to the training percent, which is set to 0.5
+        for (int rowTrainingCount : trainingCountPerRow) {
+            double meanCount = rowTrainingCount / (double) runCount;
+            assertThat(meanCount, is(closeTo(0.5, 0.12)));
+        }
+    }
+
+    private CrossValidationSplitter createSplitter(double trainingPercent) {
+        return new StratifiedCrossValidationSplitter(fields, dependentVariable, classCardinalities, trainingPercent, randomizeSeed);
+    }
+
+    private void incrementTrainingDocsCount() {
+        trainingDocsCount++;
+    }
+
+    private void incrementTestDocsCount() {
+        testDocsCount++;
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java
index 478621b8de880..d4969468ef910 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ml.action.InternalInferModelAction;
 import org.elasticsearch.xpack.core.ml.inference.results.ClassificationInferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.results.FeatureImportance;
 import org.elasticsearch.xpack.core.ml.inference.results.RegressionInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.results.WarningInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfig;
@@ -120,9 +121,9 @@ public void testMutateDocumentClassificationFeatureInfluence() {
         classes.add(new ClassificationInferenceResults.TopClassEntry("foo", 0.6));
         classes.add(new ClassificationInferenceResults.TopClassEntry("bar", 0.4));
 
-        Map<String, Double> featureInfluence = new HashMap<>();
-        featureInfluence.put("feature_1", 1.13);
-        featureInfluence.put("feature_2", -42.0);
+        List<FeatureImportance> featureInfluence = new ArrayList<>();
+        featureInfluence.add(FeatureImportance.forRegression("feature_1", 1.13));
+        featureInfluence.add(FeatureImportance.forRegression("feature_2", -42.0));
 
         InternalInferModelAction.Response response = new InternalInferModelAction.Response(
             Collections.singletonList(new ClassificationInferenceResults(1.0,
@@ -135,8 +136,10 @@ public void testMutateDocumentClassificationFeatureInfluence() {
 
         assertThat(document.getFieldValue("ml.my_processor.model_id", String.class), equalTo("classification_model"));
         assertThat(document.getFieldValue("ml.my_processor.predicted_value", String.class), equalTo("foo"));
-        assertThat(document.getFieldValue("ml.my_processor.feature_importance.feature_1", Double.class), equalTo(1.13));
-        assertThat(document.getFieldValue("ml.my_processor.feature_importance.feature_2", Double.class), equalTo(-42.0));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.0.importance", Double.class), equalTo(-42.0));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.0.feature_name", String.class), equalTo("feature_2"));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.1.importance", Double.class), equalTo(1.13));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.1.feature_name", String.class), equalTo("feature_1"));
     }
 
     @SuppressWarnings("unchecked")
@@ -205,9 +208,9 @@ public void testMutateDocumentRegressionWithTopFetures() {
         Map<String, Object> ingestMetadata = new HashMap<>();
         IngestDocument document = new IngestDocument(source, ingestMetadata);
 
-        Map<String, Double> featureInfluence = new HashMap<>();
-        featureInfluence.put("feature_1", 1.13);
-        featureInfluence.put("feature_2", -42.0);
+        List<FeatureImportance> featureInfluence = new ArrayList<>();
+        featureInfluence.add(FeatureImportance.forRegression("feature_1", 1.13));
+        featureInfluence.add(FeatureImportance.forRegression("feature_2", -42.0));
 
         InternalInferModelAction.Response response = new InternalInferModelAction.Response(
             Collections.singletonList(new RegressionInferenceResults(0.7, regressionConfig, featureInfluence)), true);
@@ -215,8 +218,10 @@ public void testMutateDocumentRegressionWithTopFetures() {
 
         assertThat(document.getFieldValue("ml.my_processor.foo", Double.class), equalTo(0.7));
         assertThat(document.getFieldValue("ml.my_processor.model_id", String.class), equalTo("regression_model"));
-        assertThat(document.getFieldValue("ml.my_processor.feature_importance.feature_1", Double.class), equalTo(1.13));
-        assertThat(document.getFieldValue("ml.my_processor.feature_importance.feature_2", Double.class), equalTo(-42.0));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.0.importance", Double.class), equalTo(-42.0));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.0.feature_name", String.class), equalTo("feature_2"));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.1.importance", Double.class), equalTo(1.13));
+        assertThat(document.getFieldValue("ml.my_processor.feature_importance.1.feature_name", String.class), equalTo("feature_1"));
     }
 
     public void testGenerateRequestWithEmptyMapping() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
index 23275958b3c10..e0370066c047c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
+import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
@@ -424,6 +425,60 @@ public void testMlStateAndResultsIndicesNotAvailable() throws Exception {
         assertBusy(() -> assertJobTask(jobId, JobState.OPENED, true));
     }
 
+    public void testCloseUnassignedLazyJobAndDatafeed() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(3);
+        ensureStableCluster(3);
+
+        String jobId = "test-lazy-stop";
+        String datafeedId = jobId + "-datafeed";
+        // Assume the test machine won't have space to assign a 2TB job
+        Job.Builder job = createJob(jobId, new ByteSizeValue(2, ByteSizeUnit.TB), true);
+        PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
+        client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet();
+
+        client().admin().indices().prepareCreate("data").setMapping("time", "type=date").get();
+
+        DatafeedConfig config = createDatafeed(datafeedId, jobId, Collections.singletonList("data"));
+        PutDatafeedAction.Request putDatafeedRequest = new PutDatafeedAction.Request(config);
+        client().execute(PutDatafeedAction.INSTANCE, putDatafeedRequest).actionGet();
+
+        ensureYellow(); // at least the primary shards of the indices a job uses should be started
+        OpenJobAction.Request openJobRequest = new OpenJobAction.Request(jobId);
+        client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet();
+
+        // Job state should be opening because it won't fit anyway, but is allowed to open lazily
+        GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(jobId);
+        GetJobsStatsAction.Response jobStatsResponse = client().execute(GetJobsStatsAction.INSTANCE, jobStatsRequest).actionGet();
+        assertEquals(JobState.OPENING, jobStatsResponse.getResponse().results().get(0).getState());
+
+        StartDatafeedAction.Request startDataFeedRequest = new StartDatafeedAction.Request(config.getId(), 0L);
+        client().execute(StartDatafeedAction.INSTANCE, startDataFeedRequest).actionGet();
+
+        // Datafeed state should be starting while it waits for job assignment
+        GetDatafeedsStatsAction.Request datafeedStatsRequest = new GetDatafeedsStatsAction.Request(datafeedId);
+        GetDatafeedsStatsAction.Response datafeedStatsResponse =
+            client().execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest).actionGet();
+        assertEquals(DatafeedState.STARTING, datafeedStatsResponse.getResponse().results().get(0).getDatafeedState());
+
+        // A starting datafeed can be stopped normally or by force
+        StopDatafeedAction.Request stopDatafeedRequest = new StopDatafeedAction.Request(datafeedId);
+        stopDatafeedRequest.setForce(randomBoolean());
+        StopDatafeedAction.Response stopDatafeedResponse = client().execute(StopDatafeedAction.INSTANCE, stopDatafeedRequest).actionGet();
+        assertTrue(stopDatafeedResponse.isStopped());
+
+        datafeedStatsResponse = client().execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest).actionGet();
+        assertEquals(DatafeedState.STOPPED, datafeedStatsResponse.getResponse().results().get(0).getDatafeedState());
+
+        // An opening job can also be stopped normally or by force
+        CloseJobAction.Request closeJobRequest = new CloseJobAction.Request(jobId);
+        closeJobRequest.setForce(randomBoolean());
+        CloseJobAction.Response closeJobResponse = client().execute(CloseJobAction.INSTANCE, closeJobRequest).actionGet();
+        assertTrue(closeJobResponse.isClosed());
+
+        jobStatsResponse = client().execute(GetJobsStatsAction.INSTANCE, jobStatsRequest).actionGet();
+        assertEquals(JobState.CLOSED, jobStatsResponse.getResponse().results().get(0).getState());
+    }
+
     private void assertJobTask(String jobId, JobState expectedState, boolean hasExecutorNode) {
         ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
         PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
index 887cd17eb6b50..6ae0d3fbd57be 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
@@ -127,6 +127,10 @@ protected Job.Builder createJob(String id) {
     }
 
     protected Job.Builder createJob(String id, ByteSizeValue modelMemoryLimit) {
+        return createJob(id, modelMemoryLimit, false);
+    }
+
+    protected Job.Builder createJob(String id, ByteSizeValue modelMemoryLimit, boolean allowLazyOpen) {
         DataDescription.Builder dataDescription = new DataDescription.Builder();
         dataDescription.setFormat(DataDescription.DataFormat.XCONTENT);
         dataDescription.setTimeFormat(DataDescription.EPOCH_MS);
@@ -141,6 +145,7 @@ protected Job.Builder createJob(String id, ByteSizeValue modelMemoryLimit) {
         }
         builder.setAnalysisConfig(analysisConfig);
         builder.setDataDescription(dataDescription);
+        builder.setAllowLazyOpen(allowLazyOpen);
         return builder;
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java
index af47efeef7d2b..c1392604e43c2 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java
@@ -73,9 +73,9 @@ public class ResultsPersisterServiceTests extends ESTestCase {
     // Constants for searchWithRetry tests
     private static final SearchRequest SEARCH_REQUEST = new SearchRequest("my-index");
     private static final SearchResponse SEARCH_RESPONSE_SUCCESS =
-        new SearchResponse(null, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null);
+        new SearchResponse(null, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null, null);
     private static final SearchResponse SEARCH_RESPONSE_FAILURE =
-        new SearchResponse(null, null, 1, 0, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null);
+        new SearchResponse(null, null, 1, 0, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null, null);
 
     // Constants for bulkIndexWithRetry tests
     private static final IndexRequest INDEX_REQUEST_SUCCESS =
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
index 3fecf1bb08b89..813e99763f5a9 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
@@ -465,7 +465,8 @@ public void testToXContent() throws IOException {
                 + "        \"coordinating_only\": 0,"
                 + "        \"data\": 0,"
                 + "        \"ingest\": 0,"
-                + "        \"master\": 1"
+                + "        \"master\": 1,"
+                + "        \"remote_cluster_client\": 0"
                 + "      },"
                 + "      \"versions\": ["
                 + "        \"%s\""
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/BinaryScalarFunction.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/BinaryScalarFunction.java
index c06a5060d829b..9091f67b19c57 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/BinaryScalarFunction.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/BinaryScalarFunction.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
 import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
 import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.util.Check;
 
 import java.util.Arrays;
 import java.util.List;
@@ -16,6 +17,7 @@
 
 public abstract class BinaryScalarFunction extends ScalarFunction {
 
+    private static final int PKG_LENGTH = "org.elasticsearch.xpack.".length();
     private final Expression left, right;
 
     protected BinaryScalarFunction(Source source, Expression left, Expression right) {
@@ -56,7 +58,10 @@ public ScriptTemplate asScript() {
     }
 
     protected ScriptTemplate asScriptFrom(ScriptTemplate leftScript, ScriptTemplate rightScript) {
-        return Scripts.binaryMethod(scriptMethodName(), leftScript, rightScript, dataType());
+        String prefix = getClass().getPackageName().substring(PKG_LENGTH);
+        int index = prefix.indexOf('.');
+        Check.isTrue(index > 0, "invalid package {}", prefix);
+        return Scripts.binaryMethod("{" + prefix.substring(0, index) + "}", scriptMethodName(), leftScript, rightScript, dataType());
     }
     
     protected String scriptMethodName() {
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java
index 823465b28b271..4e82d0ac187c0 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java
@@ -123,7 +123,7 @@ protected ScriptTemplate scriptWithGrouping(GroupingFunction grouping) {
     }
 
     protected ScriptTemplate scriptWithField(FieldAttribute field) {
-        return new ScriptTemplate(processScript("doc[{}].value"),
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
                 paramsBuilder().variable(field.name()).build(),
                 dataType());
     }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java
index 1f63bd7ac34c4..f7956bc71b45d 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.NotProcessor;
 import org.elasticsearch.xpack.ql.expression.predicate.nulls.CheckNullProcessor.CheckNullOperation;
+import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.UnaryArithmeticProcessor.UnaryArithmeticOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.InProcessor;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation;
@@ -114,4 +115,11 @@ public static Boolean regex(String value, String pattern) {
         // TODO: this needs to be improved to avoid creating the pattern on every call
         return RegexOperation.match(value, pattern);
     }
+
+    //
+    // Math
+    //
+    public static Number neg(Number value) {
+        return UnaryArithmeticOperation.NEGATE.apply(value);
+    }
 }
\ No newline at end of file
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java
index 337957089e6dd..5a66bba66d374 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java
@@ -29,16 +29,17 @@ public final class Scripts {
     public static final String EQL_SCRIPTS = "{eql}";
     public static final String SQL_SCRIPTS = "{sql}";
     public static final String PARAM = "{}";
-    // FIXME: this needs to be either renamed (drop Sql) or find a pluggable approach (through ScriptWeaver)
-    public static final String INTERNAL_SCRIPT_UTILS = "InternalSqlScriptUtils";
+    public static final String INTERNAL_QL_SCRIPT_UTILS = "InternalQlScriptUtils";
+    public static final String INTERNAL_EQL_SCRIPT_UTILS = "InternalEqlScriptUtils";
+    public static final String INTERNAL_SQL_SCRIPT_UTILS = "InternalSqlScriptUtils";
 
     private Scripts() {}
 
     static final Map<Pattern, String> FORMATTING_PATTERNS = unmodifiableMap(Stream.of(
             new SimpleEntry<>(DOC_VALUE, QL_SCRIPTS + ".docValue(doc,{})"),
-            new SimpleEntry<>(QL_SCRIPTS, INTERNAL_SCRIPT_UTILS),
-            new SimpleEntry<>(EQL_SCRIPTS, INTERNAL_SCRIPT_UTILS),
-            new SimpleEntry<>(SQL_SCRIPTS, INTERNAL_SCRIPT_UTILS),
+            new SimpleEntry<>(QL_SCRIPTS, INTERNAL_QL_SCRIPT_UTILS),
+            new SimpleEntry<>(EQL_SCRIPTS, INTERNAL_EQL_SCRIPT_UTILS),
+            new SimpleEntry<>(SQL_SCRIPTS, INTERNAL_SQL_SCRIPT_UTILS),
             new SimpleEntry<>(PARAM, "params.%s"))
             .collect(toMap(e -> Pattern.compile(e.getKey(), Pattern.LITERAL), Map.Entry::getValue, (a, b) -> a, LinkedHashMap::new)));
 
@@ -74,16 +75,17 @@ public static ScriptTemplate nullSafeSort(ScriptTemplate script) {
     }
 
     public static ScriptTemplate and(ScriptTemplate left, ScriptTemplate right) {
-        return binaryMethod("and", left, right, DataTypes.BOOLEAN);
+        return binaryMethod("{ql}", "and", left, right, DataTypes.BOOLEAN);
     }
 
     public static ScriptTemplate or(ScriptTemplate left, ScriptTemplate right) {
-        return binaryMethod("or", left, right, DataTypes.BOOLEAN);
+        return binaryMethod("{ql}", "or", left, right, DataTypes.BOOLEAN);
     }
     
-    public static ScriptTemplate binaryMethod(String methodName, ScriptTemplate leftScript, ScriptTemplate rightScript,
+    public static ScriptTemplate binaryMethod(String prefix, String methodName, ScriptTemplate leftScript, ScriptTemplate rightScript,
             DataType dataType) {
-        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{sql}.%s(%s,%s)"),
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("%s.%s(%s,%s)"),
+                formatTemplate(prefix),
                 methodName,
                 leftScript.template(),
                 rightScript.template()),
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/Range.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/Range.java
index 35ccd1d06b334..b5319b4098842 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/Range.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/Range.java
@@ -129,7 +129,7 @@ public ScriptTemplate asScript() {
         ScriptTemplate upperScript = asScript(upper);
         
 
-        String template = formatTemplate(format(Locale.ROOT, "{sql}.and({sql}.%s(%s, %s), {sql}.%s(%s, %s))",
+        String template = formatTemplate(format(Locale.ROOT, "{ql}.and({ql}.%s(%s, %s), {ql}.%s(%s, %s))",
                         includeLower() ? "gte" : "gt",
                         valueScript.template(),
                         lowerScript.template(),
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/logical/Not.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/logical/Not.java
index 9beec2870a2b8..bd5a683aa2d03 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/logical/Not.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/logical/Not.java
@@ -54,7 +54,7 @@ protected Processor makeProcessor() {
 
     @Override
     public String processScript(String script) {
-        return Scripts.formatTemplate(Scripts.SQL_SCRIPTS + ".not(" + script + ")");
+        return Scripts.formatTemplate(Scripts.QL_SCRIPTS + ".not(" + script + ")");
     }
 
     @Override
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/IsNotNull.java
index ee14259608ddf..ade44b0ca3aac 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/IsNotNull.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/IsNotNull.java
@@ -45,7 +45,7 @@ protected Processor makeProcessor() {
 
     @Override
     public String processScript(String script) {
-        return Scripts.formatTemplate(Scripts.SQL_SCRIPTS + ".isNotNull(" + script + ")");
+        return Scripts.formatTemplate(Scripts.QL_SCRIPTS + ".isNotNull(" + script + ")");
     }
 
     @Override
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/IsNull.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/IsNull.java
index 08bd1b80d0b0e..7b04fe084a9a8 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/IsNull.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/IsNull.java
@@ -45,7 +45,7 @@ protected Processor makeProcessor() {
 
     @Override
     public String processScript(String script) {
-        return Scripts.formatTemplate(Scripts.SQL_SCRIPTS + ".isNull(" + script + ")");
+        return Scripts.formatTemplate(Scripts.QL_SCRIPTS + ".isNull(" + script + ")");
     }
 
     @Override
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Neg.java
index dc62f62122ab2..cc97dc019d9d6 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Neg.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Neg.java
@@ -53,7 +53,7 @@ public DataType dataType() {
 
     @Override
     public String processScript(String script) {
-        return Scripts.formatTemplate(Scripts.SQL_SCRIPTS + ".neg(" + script + ")");
+        return Scripts.formatTemplate(Scripts.QL_SCRIPTS + ".neg(" + script + ")");
     }
 
     @Override
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/In.java
index 8f084167f1529..bb786b1ef9039 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/In.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/In.java
@@ -95,7 +95,7 @@ public ScriptTemplate asScript() {
         List<Object> values = new ArrayList<>(new LinkedHashSet<>(foldAndConvertListOfValues(list, value.dataType())));
 
         return new ScriptTemplate(
-            formatTemplate(format("{sql}.","in({}, {})", leftScript.template())),
+            formatTemplate(format("{ql}.","in({}, {})", leftScript.template())),
             paramsBuilder()
                 .script(leftScript.params())
                 .variable(values)
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
index e93442af52cec..92a265ef82883 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
@@ -334,7 +334,7 @@ private static SearchResponse mergeFinalResponse(SearchResponse liveResponse, Li
 
         // Shard failures are ignored atm, so returning an empty array is fine
         return new SearchResponse(combinedInternal, null, totalShards, sucessfulShards, skippedShards,
-            took, ShardSearchFailure.EMPTY_ARRAY, rolledResponses.get(0).getClusters());
+            took, ShardSearchFailure.EMPTY_ARRAY, rolledResponses.get(0).getClusters(), null);
     }
 
     /**
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
index 60fbdc314c96c..fd61980753d40 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
@@ -47,6 +47,7 @@
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -105,7 +106,8 @@ protected void doExecute(Task task, SearchRequest request, ActionListener<Search
         MultiSearchRequest msearch = createMSearchRequest(request, registry, rollupSearchContext);
 
         client.multiSearch(msearch, ActionListener.wrap(msearchResponse -> {
-            InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction(bigArrays, scriptService);
+            InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction(
+                    bigArrays, scriptService, () -> PipelineAggregator.PipelineTree.EMPTY);
             listener.onResponse(processResponses(rollupSearchContext, msearchResponse, context));
         }, listener::onFailure));
     }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
index 6c369c3c94f24..3643fc5595575 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
@@ -500,11 +500,11 @@ public void testMismatch() throws IOException {
         MultiSearchResponse.Item unrolledItem = new MultiSearchResponse.Item(new SearchResponse(
                 new InternalSearchResponse(null,
                         new InternalAggregations(Collections.singletonList(responses.get(0))), null, null, false, false, 1),
-                        null, 1, 1, 0, 10, null, null), null);
+                        null, 1, 1, 0, 10, null, null, null), null);
         MultiSearchResponse.Item rolledItem = new MultiSearchResponse.Item(new SearchResponse(
                 new InternalSearchResponse(null,
                         new InternalAggregations(Collections.singletonList(responses.get(1))), null, null, false, false, 1),
-                        null, 1, 1, 0, 10, null, null), null);
+                        null, 1, 1, 0, 10, null, null, null), null);
 
         MultiSearchResponse.Item[] msearch = new MultiSearchResponse.Item[]{unrolledItem, rolledItem};
 
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
index 9f8e26d425f5d..556bc3b088065 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
@@ -67,7 +67,7 @@
 public class IndexerUtilsTests extends AggregatorTestCase {
     public void testMissingFields() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String timestampField = "the_histo";
         String valueField = "the_avg";
@@ -131,7 +131,7 @@ public void testMissingFields() throws IOException {
 
     public void testCorrectFields() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String timestampField = "the_histo";
         String valueField = "the_avg";
@@ -199,7 +199,7 @@ public void testCorrectFields() throws IOException {
 
     public void testNumericTerms() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats= new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats= new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String valueField = "the_avg";
 
@@ -255,7 +255,7 @@ public void testNumericTerms() throws IOException {
 
     public void testEmptyCounts() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String timestampField = "ts";
         String valueField = "the_avg";
@@ -444,7 +444,7 @@ public void testNullKeys() {
 
     public void testMissingBuckets() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String metricField = "metric_field";
         String valueField = "value_field";
@@ -517,7 +517,7 @@ public void testMissingBuckets() throws IOException {
 
     public void testTimezone() throws IOException {
         String indexName = randomAlphaOfLengthBetween(1, 10);
-        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        RollupIndexerJobStats stats = new RollupIndexerJobStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
         String timestampField = "the_histo";
         String valueField = "the_avg";
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
index baabbfd8d3e94..87b7bff27c3c4 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
@@ -663,7 +663,7 @@ protected void doNextSearch(SearchRequest request, ActionListener<SearchResponse
             SearchResponseSections sections = new SearchResponseSections(null, new Aggregations(Collections.singletonList(result)),
                     null, false, null, null, 1);
             SearchResponse response = new SearchResponse(sections, null, 1, 1,
-                    0, 0, ShardSearchFailure.EMPTY_ARRAY, null);
+                    0, 0, ShardSearchFailure.EMPTY_ARRAY, null, null);
             listener.onResponse(response);
         }
 
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java
index a7b3c82b36808..d46e62b609db1 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java
@@ -100,7 +100,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0),
                 aggs, null, false, null, null, 1);
             final SearchResponse response = new SearchResponse(sections, null, 1, 1, 0, 0,
-                new ShardSearchFailure[0], null);
+                new ShardSearchFailure[0], null, null);
             nextPhase.onResponse(response);
         }
 
@@ -456,7 +456,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                         new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0),
                         aggs, null, false, null, null, 1);
                     final SearchResponse response = new SearchResponse(sections, null, 1, 1, 0, 0,
-                        ShardSearchFailure.EMPTY_ARRAY, null);
+                        ShardSearchFailure.EMPTY_ARRAY, null, null);
                     nextPhase.onResponse(response);
                 }
 
@@ -661,8 +661,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             final SearchResponseSections sections = new SearchResponseSections(
                 new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0),
                 aggs, null, false, null, null, 1);
-            return new SearchResponse(sections, null, 1, 1, 0, 0,
-                ShardSearchFailure.EMPTY_ARRAY, null);
+            return new SearchResponse(sections, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null, null);
         };
 
         Function<BulkRequest, BulkResponse> bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100);
@@ -774,8 +773,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             final SearchResponseSections sections = new SearchResponseSections(
                 new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0),
                 aggs, null, false, null, null, 1);
-            return new SearchResponse(sections, null, 1, 1, 0, 0,
-                ShardSearchFailure.EMPTY_ARRAY, null);
+            return new SearchResponse(sections, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null, null);
         };
 
         Function<BulkRequest, BulkResponse> bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100);
@@ -931,8 +929,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             final SearchResponseSections sections = new SearchResponseSections(
                 new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0),
                 aggs, null, false, null, null, 1);
-            return new SearchResponse(sections, null, 1, 1, 0, 0,
-                ShardSearchFailure.EMPTY_ARRAY, null);
+            return new SearchResponse(sections, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null, null);
         };
 
         Function<BulkRequest, BulkResponse> bulkFunction = bulkRequest -> {
diff --git a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
index 2fe99bc8ad48d..3fdeb69e66adf 100644
--- a/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
+++ b/x-pack/plugin/security/qa/security-trial/src/test/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java
@@ -72,7 +72,7 @@ protected Tuple<String, String> createOAuthToken(String username, SecureString p
         final RestHighLevelClient client = getHighLevelAdminClient();
         final CreateTokenRequest request = CreateTokenRequest.passwordGrant(username, password.getChars());
         final CreateTokenResponse response = client.security().createToken(request, RequestOptions.DEFAULT);
-        return new Tuple(response.getAccessToken(), response.getRefreshToken());
+        return Tuple.tuple(response.getAccessToken(), response.getRefreshToken());
     }
 
     protected void deleteUser(String username) throws IOException {
diff --git a/x-pack/plugin/security/qa/tls-basic/src/test/java/org/elasticsearch/xpack/security/TlsWithBasicLicenseIT.java b/x-pack/plugin/security/qa/tls-basic/src/test/java/org/elasticsearch/xpack/security/TlsWithBasicLicenseIT.java
index 3b2aea4a08d68..5477dd638f9bf 100644
--- a/x-pack/plugin/security/qa/tls-basic/src/test/java/org/elasticsearch/xpack/security/TlsWithBasicLicenseIT.java
+++ b/x-pack/plugin/security/qa/tls-basic/src/test/java/org/elasticsearch/xpack/security/TlsWithBasicLicenseIT.java
@@ -98,6 +98,7 @@ private void checkSSLEnabled() throws IOException {
         assertThat(ObjectPath.evaluate(usage, "security.ssl.transport.enabled"), equalTo(true));
     }
 
+    @SuppressWarnings("unchecked")
     private void checkCertificateAPI() throws IOException {
         Response response = client().performRequest(new Request("GET", "/_ssl/certificates"));
         ObjectPath path = ObjectPath.createFromResponse(response);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 1f62f6c341f87..2809aa0e6ec99 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -1088,7 +1088,7 @@ private synchronized NioGroupFactory getNioGroupFactory(Settings settings) {
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return List.of(
             new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, "Contains Security configuration"),
             new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, "Contains Security configuration"),
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java
index b554309def497..2fa87e4607100 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java
@@ -7,11 +7,9 @@
 
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
 
@@ -20,14 +18,12 @@
  */
 public class SecurityInfoTransportAction extends XPackInfoFeatureTransportAction {
 
-    private final boolean enabled;
     private final XPackLicenseState licenseState;
 
     @Inject
     public SecurityInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
-                                       Settings settings, XPackLicenseState licenseState) {
+                                       XPackLicenseState licenseState) {
         super(XPackInfoFeatureAction.SECURITY.name(), transportService, actionFilters);
-        this.enabled = XPackSettings.SECURITY_ENABLED.get(settings);
         this.licenseState = licenseState;
     }
 
@@ -43,6 +39,6 @@ public boolean available() {
 
     @Override
     public boolean enabled() {
-        return enabled && licenseState.isSecurityDisabledByLicenseDefaults() == false;
+        return licenseState.isSecurityEnabled();
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
index 4d8114ef11e42..ce732276409c9 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
@@ -46,7 +46,6 @@
 
 public class SecurityUsageTransportAction extends XPackUsageFeatureTransportAction {
 
-    private final boolean enabledInSettings;
     private final Settings settings;
     private final XPackLicenseState licenseState;
     private final Realms realms;
@@ -60,7 +59,6 @@ public SecurityUsageTransportAction(TransportService transportService, ClusterSe
                                         Settings settings, XPackLicenseState licenseState, SecurityUsageServices securityServices) {
         super(XPackUsageFeatureAction.SECURITY.name(), transportService, clusterService, threadPool,
               actionFilters, indexNameExpressionResolver);
-        this.enabledInSettings = XPackSettings.SECURITY_ENABLED.get(settings);
         this.settings = settings;
         this.licenseState = licenseState;
         this.realms = securityServices.realms;
@@ -83,10 +81,11 @@ protected void masterOperation(Task task, XPackUsageRequest request, ClusterStat
         final AtomicReference<Map<String, Object>> rolesUsageRef = new AtomicReference<>();
         final AtomicReference<Map<String, Object>> roleMappingUsageRef = new AtomicReference<>();
         final AtomicReference<Map<String, Object>> realmsUsageRef = new AtomicReference<>();
+
+        final boolean enabled = licenseState.isSecurityEnabled();
         final CountDown countDown = new CountDown(3);
         final Runnable doCountDown = () -> {
             if (countDown.countDown()) {
-                boolean enabled = enabledInSettings && licenseState.isSecurityDisabledByLicenseDefaults() == false;
                 var usage = new SecurityFeatureSetUsage(licenseState.isSecurityAvailable(), enabled,
                         realmsUsageRef.get(), rolesUsageRef.get(), roleMappingUsageRef.get(), sslUsage, auditUsage,
                         ipFilterUsage, anonymousUsage, tokenServiceUsage, apiKeyServiceUsage, fips140Usage);
@@ -113,17 +112,17 @@ protected void masterOperation(Task task, XPackUsageRequest request, ClusterStat
                 doCountDown.run();
             }, listener::onFailure);
 
-        if (rolesStore == null) {
+        if (rolesStore == null || enabled == false) {
             rolesStoreUsageListener.onResponse(Collections.emptyMap());
         } else {
             rolesStore.usageStats(rolesStoreUsageListener);
         }
-        if (roleMappingStore == null) {
+        if (roleMappingStore == null || enabled == false) {
             roleMappingStoreUsageListener.onResponse(Collections.emptyMap());
         } else {
             roleMappingStore.usageStats(roleMappingStoreUsageListener);
         }
-        if (realms == null) {
+        if (realms == null || enabled == false) {
             realmsUsageListener.onResponse(Collections.emptyMap());
         } else {
             realms.usageStats(realmsUsageListener);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java
index b9c298023e04b..3a484c28dd6de 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java
@@ -111,7 +111,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void app
                 listener.onFailure(e);
             }
         } else if (SECURITY_ACTION_MATCHER.test(action)) {
-            if (licenseState.isSecurityDisabledByLicenseDefaults()) {
+            if (licenseState.isSecurityEnabled() == false) {
                 listener.onFailure(new ElasticsearchException("Security must be explicitly enabled when using a [" +
                         licenseState.getOperationMode().description() + "] license. " +
                         "Enable security by setting [xpack.security.enabled] to [true] in the elasticsearch.yml file " +
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index c363ad1253509..6ac7f03595ec7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -102,6 +102,7 @@
 import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING;
 import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
+import static org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
 
 public class ApiKeyService {
@@ -343,7 +344,7 @@ private void loadApiKeyAndValidateCredentials(ThreadContext ctx, ApiKeyCredentia
      * retrieval of role descriptors that are associated with the api key
      */
     public void getRoleForApiKey(Authentication authentication, ActionListener<ApiKeyRoleDescriptors> listener) {
-        if (authentication.getAuthenticationType() != Authentication.AuthenticationType.API_KEY) {
+        if (authentication.getAuthenticationType() != AuthenticationType.API_KEY) {
             throw new IllegalStateException("authentication type must be api key but is " + authentication.getAuthenticationType());
         }
 
@@ -894,7 +895,7 @@ public void getApiKeys(String realmName, String username, String apiKeyName, Str
      * @return realm name
      */
     public static String getCreatorRealmName(final Authentication authentication) {
-        if (authentication.getAuthenticatedBy().getType().equals(API_KEY_REALM_TYPE)) {
+        if (AuthenticationType.API_KEY == authentication.getAuthenticationType()) {
             return (String) authentication.getMetadata().get(API_KEY_CREATOR_REALM_NAME);
         } else {
             return authentication.getSourceRealm().getName();
@@ -909,7 +910,7 @@ public static String getCreatorRealmName(final Authentication authentication) {
      * @return realm type
      */
     public static String getCreatorRealmType(final Authentication authentication) {
-        if (authentication.getAuthenticatedBy().getType().equals(API_KEY_REALM_TYPE)) {
+        if (AuthenticationType.API_KEY == authentication.getAuthenticationType()) {
             return (String) authentication.getMetadata().get(API_KEY_CREATOR_REALM_TYPE);
         } else {
             return authentication.getSourceRealm().getType();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
index f32a57c1d7533..77a79cfc7a78b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
@@ -786,10 +786,8 @@ private void indexInvalidation(Collection<String> tokenIds, SecurityIndexManager
                                     retryTokenDocIds.size(), tokenIds.size());
                             final TokensInvalidationResult incompleteResult = new TokensInvalidationResult(invalidated,
                                         previouslyInvalidated, failedRequestResponses);
-                            final Runnable retryWithContextRunnable = client.threadPool().getThreadContext()
-                                    .preserveContext(() -> indexInvalidation(retryTokenDocIds, tokensIndexManager, backoff,
-                                            srcPrefix, incompleteResult, listener));
-                            client.threadPool().schedule(retryWithContextRunnable, backoff.next(), GENERIC);
+                            client.threadPool().schedule(() -> indexInvalidation(retryTokenDocIds, tokensIndexManager, backoff,
+                                srcPrefix, incompleteResult, listener), backoff.next(), GENERIC);
                         } else {
                             if (retryTokenDocIds.isEmpty() == false) {
                                 logger.warn("failed to invalidate [{}] tokens out of [{}] after all retries", retryTokenDocIds.size(),
@@ -809,10 +807,8 @@ private void indexInvalidation(Collection<String> tokenIds, SecurityIndexManager
                         traceLog("invalidate tokens", cause);
                         if (isShardNotAvailableException(cause) && backoff.hasNext()) {
                             logger.debug("failed to invalidate tokens, retrying ");
-                            final Runnable retryWithContextRunnable = client.threadPool().getThreadContext()
-                                    .preserveContext(() -> indexInvalidation(tokenIds, tokensIndexManager, backoff, srcPrefix,
-                                            previousResult, listener));
-                            client.threadPool().schedule(retryWithContextRunnable, backoff.next(), GENERIC);
+                            client.threadPool().schedule(() -> indexInvalidation(tokenIds, tokensIndexManager, backoff, srcPrefix,
+                                previousResult, listener), backoff.next(), GENERIC);
                         } else {
                             listener.onFailure(e);
                         }
@@ -894,9 +890,8 @@ private void findTokenFromRefreshToken(String refreshToken, SecurityIndexManager
             if (backoff.hasNext()) {
                 final TimeValue backofTimeValue = backoff.next();
                 logger.debug("retrying after [{}] back off", backofTimeValue);
-                final Runnable retryWithContextRunnable = client.threadPool().getThreadContext()
-                        .preserveContext(() -> findTokenFromRefreshToken(refreshToken, tokensIndexManager, backoff, listener));
-                client.threadPool().schedule(retryWithContextRunnable, backofTimeValue, GENERIC);
+                client.threadPool().schedule(() -> findTokenFromRefreshToken(refreshToken, tokensIndexManager, backoff, listener),
+                    backofTimeValue, GENERIC);
             } else {
                 logger.warn("failed to find token from refresh token after all retries");
                 onFailure.accept(ex);
@@ -1018,10 +1013,8 @@ private void innerRefresh(String refreshToken, String tokenDocId, Map<String, Ob
                         } else if (backoff.hasNext()) {
                             logger.info("failed to update the original token document [{}], the update result was [{}]. Retrying",
                                     tokenDocId, updateResponse.getResult());
-                            final Runnable retryWithContextRunnable = client.threadPool().getThreadContext()
-                                .preserveContext(() -> innerRefresh(refreshToken, tokenDocId, source, seqNo, primaryTerm, clientAuth,
-                                    backoff, refreshRequested, listener));
-                            client.threadPool().schedule(retryWithContextRunnable, backoff.next(), GENERIC);
+                            client.threadPool().schedule(() -> innerRefresh(refreshToken, tokenDocId, source, seqNo, primaryTerm,
+                                clientAuth, backoff, refreshRequested, listener), backoff.next(), GENERIC);
                         } else {
                             logger.info("failed to update the original token document [{}] after all retries, the update result was [{}]. ",
                                     tokenDocId, updateResponse.getResult());
@@ -1049,9 +1042,8 @@ public void onFailure(Exception e) {
                                     if (isShardNotAvailableException(e)) {
                                         if (backoff.hasNext()) {
                                             logger.info("could not get token document [{}] for refresh, retrying", tokenDocId);
-                                            final Runnable retryWithContextRunnable = client.threadPool().getThreadContext()
-                                                    .preserveContext(() -> getTokenDocAsync(tokenDocId, refreshedTokenIndex, this));
-                                            client.threadPool().schedule(retryWithContextRunnable, backoff.next(), GENERIC);
+                                            client.threadPool().schedule(() -> getTokenDocAsync(tokenDocId, refreshedTokenIndex, this),
+                                                backoff.next(), GENERIC);
                                         } else {
                                             logger.warn("could not get token document [{}] for refresh after all retries", tokenDocId);
                                             onFailure.accept(invalidGrantException("could not refresh the requested token"));
@@ -1064,10 +1056,8 @@ public void onFailure(Exception e) {
                         } else if (isShardNotAvailableException(e)) {
                             if (backoff.hasNext()) {
                                 logger.debug("failed to update the original token document [{}], retrying", tokenDocId);
-                                final Runnable retryWithContextRunnable = client.threadPool().getThreadContext()
-                                    .preserveContext(() -> innerRefresh(refreshToken, tokenDocId, source, seqNo, primaryTerm,
-                                        clientAuth, backoff, refreshRequested, listener));
-                                client.threadPool().schedule(retryWithContextRunnable, backoff.next(), GENERIC);
+                                client.threadPool().schedule(() -> innerRefresh(refreshToken, tokenDocId, source, seqNo, primaryTerm,
+                                    clientAuth, backoff, refreshRequested, listener), backoff.next(), GENERIC);
                             } else {
                                 logger.warn("failed to update the original token document [{}], after all retries", tokenDocId);
                                 onFailure.accept(invalidGrantException("could not refresh the requested token"));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java
index 77fe28f870145..7f074c8282c4f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java
@@ -230,7 +230,7 @@ private Assertion decrypt(EncryptedAssertion encrypted) {
 
     private List<Attribute> processAssertion(Assertion assertion, boolean requireSignature, Collection<String> allowedSamlRequestIds) {
         if (logger.isTraceEnabled()) {
-            logger.trace("(Possibly decrypted) Assertion: {}", SamlUtils.samlObjectToString(assertion));
+            logger.trace("(Possibly decrypted) Assertion: {}", SamlUtils.getXmlContent(assertion));
             logger.trace(SamlUtils.describeSamlObject(assertion));
         }
         // Do not further process unsigned Assertions
@@ -253,7 +253,7 @@ private List<Attribute> processAssertion(Assertion assertion, boolean requireSig
             for (EncryptedAttribute enc : statement.getEncryptedAttributes()) {
                 final Attribute attribute = decrypt(enc);
                 if (attribute != null) {
-                    logger.trace("Successfully decrypted attribute: {}" + SamlUtils.samlObjectToString(attribute));
+                    logger.trace("Successfully decrypted attribute: {}" + SamlUtils.getXmlContent(attribute));
                     attributes.add(attribute);
                 }
             }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
index 4f8677364dcf7..ca16bd4a9a095 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
@@ -656,7 +656,7 @@ public AuthnRequest buildAuthenticationRequest() {
                 .forceAuthn(forceAuthn)
                 .build();
         if (logger.isTraceEnabled()) {
-            logger.trace("Constructed SAML Authentication Request: {}", SamlUtils.samlObjectToString(authnRequest));
+            logger.trace("Constructed SAML Authentication Request: {}", SamlUtils.getXmlContent(authnRequest));
         }
         return authnRequest;
     }
@@ -672,7 +672,7 @@ public LogoutRequest buildLogoutRequest(NameID nameId, String session) {
             final LogoutRequest logoutRequest = new SamlLogoutRequestMessageBuilder(
                     Clock.systemUTC(), serviceProvider, idpDescriptor.get(), nameId, session).build();
             if (logoutRequest != null && logger.isTraceEnabled()) {
-                logger.trace("Constructed SAML Logout Request: {}", SamlUtils.samlObjectToString(logoutRequest));
+                logger.trace("Constructed SAML Logout Request: {}", SamlUtils.getXmlContent(logoutRequest));
             }
             return logoutRequest;
         } else {
@@ -688,7 +688,7 @@ public LogoutResponse buildLogoutResponse(String inResponseTo) {
         final LogoutResponse logoutResponse = new SamlLogoutResponseBuilder(
                 Clock.systemUTC(), serviceProvider, idpDescriptor.get(), inResponseTo, StatusCode.SUCCESS).build();
         if (logoutResponse != null && logger.isTraceEnabled()) {
-            logger.trace("Constructed SAML Logout Response: {}", SamlUtils.samlObjectToString(logoutResponse));
+            logger.trace("Constructed SAML Logout Response: {}", SamlUtils.getXmlContent(logoutResponse));
         }
         return logoutResponse;
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRequestHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRequestHandler.java
index 9781783673afd..ccd0ec22f20ab 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRequestHandler.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRequestHandler.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.xpack.security.support.RestorableContextClassLoader;
+import org.elasticsearch.xpack.core.security.support.RestorableContextClassLoader;
 import org.joda.time.DateTime;
 import org.opensaml.core.xml.XMLObject;
 import org.opensaml.core.xml.io.Unmarshaller;
@@ -47,7 +47,9 @@
 import javax.xml.parsers.DocumentBuilder;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.security.AccessController;
 import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.security.cert.CertificateEncodingException;
 import java.security.cert.X509Certificate;
 import java.time.Clock;
@@ -141,14 +143,20 @@ void validateSignature(Signature signature) {
         }
 
         checkIdpSignature(credential -> {
-            try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(SignatureValidator.class)) {
-                SignatureValidator.validate(signature, credential);
-                logger.debug(() -> new ParameterizedMessage("SAML Signature [{}] matches credentials [{}] [{}]",
-                        signatureText, credential.getEntityId(), credential.getPublicKey()));
-                return true;
+            try {
+                return AccessController.doPrivileged((PrivilegedExceptionAction<Boolean>) () -> {
+                    try (RestorableContextClassLoader ignore = new RestorableContextClassLoader(SignatureValidator.class)) {
+                        SignatureValidator.validate(signature, credential);
+                        logger.debug(() -> new ParameterizedMessage("SAML Signature [{}] matches credentials [{}] [{}]",
+                            signatureText, credential.getEntityId(), credential.getPublicKey()));
+                        return true;
+                    } catch (PrivilegedActionException e) {
+                        logger.warn("SecurityException while attempting to validate SAML signature", e);
+                        return false;
+                    }
+                });
             } catch (PrivilegedActionException e) {
-                logger.warn("SecurityException while attempting to validate SAML signature", e);
-                return false;
+                throw new SecurityException("SecurityException while attempting to validate SAML signature", e);
             }
         }, signatureText);
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlUtils.java
index 8451f3553dfcb..7777229dd0fcd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlUtils.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlUtils.java
@@ -43,7 +43,7 @@
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.xpack.security.support.RestorableContextClassLoader;
+import org.elasticsearch.xpack.core.security.support.RestorableContextClassLoader;
 import org.opensaml.core.config.InitializationService;
 import org.opensaml.core.xml.XMLObject;
 import org.opensaml.core.xml.XMLObjectBuilderFactory;
@@ -162,9 +162,9 @@ static void print(Element element, Writer writer, boolean pretty) throws Transfo
         serializer.transform(new DOMSource(element), new StreamResult(writer));
     }
 
-    static String samlObjectToString(SAMLObject object) {
+    static String getXmlContent(SAMLObject object) {
         try {
-            return toString(XMLObjectSupport.marshall(object), true);
+            return toString(XMLObjectSupport.marshall(object), false);
         } catch (MarshallingException e) {
             LOGGER.info("Error marshalling SAMLObject ", e);
             return SAML_MARSHALLING_ERROR_STRING;
@@ -202,7 +202,7 @@ static String describeSamlObject(SAMLObject object) {
             sb.append("]");
             return sb.toString();
         }
-        return samlObjectToString(object);
+        return getXmlContent(object);
     }
 
     @SuppressForbidden(reason = "This is the only allowed way to construct a Transformer")
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
index 2bd02e6decb1f..9586ddb47ab20 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.TemplateRoleName;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel;
 import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
@@ -165,6 +166,10 @@ protected ExpressionRoleMapping buildMapping(String id, BytesReference source) {
      * Stores (create or update) a single mapping in the index
      */
     public void putRoleMapping(PutRoleMappingRequest request, ActionListener<Boolean> listener) {
+        // Validate all templates before storing the role mapping
+        for (TemplateRoleName templateRoleName : request.getRoleTemplates()) {
+            templateRoleName.validate(scriptService);
+        }
         modifyMapping(request.getName(), this::innerPutMapping, request, listener);
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
index 67d88efc0bc1d..783f31f8ce855 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationFailureHandler;
 import org.elasticsearch.xpack.core.security.authc.esnative.ClientReservedRealm;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
@@ -585,7 +586,7 @@ private ElasticsearchSecurityException denialException(Authentication authentica
                     authentication.getUser().principal());
         }
         // check for authentication by API key
-        if (authentication.getAuthenticatedBy().getType().equals(ApiKeyService.API_KEY_REALM_TYPE)) {
+        if (AuthenticationType.API_KEY == authentication.getAuthenticationType()) {
             final String apiKeyId = (String) authentication.getMetadata().get(ApiKeyService.API_KEY_ID_KEY);
             assert apiKeyId != null : "api key id must be present in the metadata";
             logger.debug("action [{}] is unauthorized for API key id [{}] of user [{}]", action, apiKeyId, authUser.principal());
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
index 11c57fa3f5d4a..77d8c9d9737bb 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
@@ -22,6 +22,7 @@
 import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.ENRICH_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.IDP_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN;
@@ -115,6 +116,7 @@ public static void switchUserBasedOnActionOriginAndExecute(ThreadContext threadC
             case ROLLUP_ORIGIN:
             case INDEX_LIFECYCLE_ORIGIN:
             case ENRICH_ORIGIN:
+            case IDP_ORIGIN:
             case TASKS_ORIGIN:   // TODO use a more limited user for tasks
                 securityContext.executeAsUser(XPackUser.INSTANCE, consumer, Version.CURRENT);
                 break;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
index 6113cf4dfd305..e3f6fbbd5df4f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.action.delete.DeleteAction;
 import org.elasticsearch.action.get.MultiGetAction;
 import org.elasticsearch.action.index.IndexAction;
+import org.elasticsearch.action.search.ClearReaderAction;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.MultiSearchAction;
 import org.elasticsearch.action.search.SearchScrollAction;
@@ -47,6 +48,7 @@
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
 import org.elasticsearch.xpack.core.security.action.user.UserRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
@@ -179,7 +181,7 @@ boolean checkSameUserPermissions(String action, TransportRequest request, Authen
                 return sameUsername;
             } else if (request instanceof GetApiKeyRequest) {
                 GetApiKeyRequest getApiKeyRequest = (GetApiKeyRequest) request;
-                if (authentication.getAuthenticatedBy().getType().equals(ApiKeyService.API_KEY_REALM_TYPE)) {
+                if (AuthenticationType.API_KEY == authentication.getAuthenticationType()) {
                     assert authentication.getLookedUpBy() == null : "runAs not supported for api key authentication";
                     // if authenticated by API key then the request must also contain same API key id
                     String authenticatedApiKeyId = (String) authentication.getMetadata().get(ApiKeyService.API_KEY_ID_KEY);
@@ -276,6 +278,9 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth
                     // the same as the user that submitted the original request so we can skip security here.
                     listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
                 }
+            } else if (action.equals(ClearReaderAction.NAME)) {
+                // NORELEASE: ensure security works correctly
+                listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
             } else {
                 assert false : "only scroll and async-search related requests are known indices api that don't " +
                     "support retrieving the indices they relate to";
@@ -542,8 +547,8 @@ private static boolean checkChangePasswordAction(Authentication authentication)
         // Ensure that the user is not authenticated with an access token or an API key.
         // Also ensure that the user was authenticated by a realm that we can change a password for. The native realm is an internal realm
         // and right now only one can exist in the realm configuration - if this changes we should update this check
-        final Authentication.AuthenticationType authType = authentication.getAuthenticationType();
-        return (authType.equals(Authentication.AuthenticationType.REALM)
+        final AuthenticationType authType = authentication.getAuthenticationType();
+        return (authType.equals(AuthenticationType.REALM)
             && (ReservedRealm.TYPE.equals(realmType) || NativeRealmSettings.TYPE.equals(realmType)));
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java
index 7136a110bdc04..fb2444e764735 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java
@@ -70,7 +70,7 @@ protected Exception checkFeatureAvailable(RestRequest request) {
             return new IllegalStateException("Security is not enabled but a security rest handler is registered");
         } else if (licenseState.isSecurityAvailable() == false) {
             return LicenseUtils.newComplianceException(XPackField.SECURITY);
-        } else if (licenseState.isSecurityDisabledByLicenseDefaults()) {
+        } else if (licenseState.isSecurityEnabled() == false) {
             return new ElasticsearchException("Security must be explicitly enabled when using a [" +
                     licenseState.getOperationMode().description() + "] license. " +
                     "Enable security by setting [xpack.security.enabled] to [true] in the elasticsearch.yml file " +
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListener.java
index ddc41561afabd..572f78d1a3a1a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListener.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListener.java
@@ -35,7 +35,7 @@ public SecurityStatusChangeListener(XPackLicenseState licenseState) {
      */
     @Override
     public synchronized void licenseStateChanged() {
-        final boolean newState = licenseState.isSecurityAvailable() && licenseState.isSecurityDisabledByLicenseDefaults() == false;
+        final boolean newState = licenseState.isSecurityAvailable() && licenseState.isSecurityEnabled();
         // old state might be null (undefined) so do Object comparison
         if (Objects.equals(newState, securityEnabled) == false) {
             logger.info("Active license is now [{}]; Security is {}", licenseState.getOperationMode(), newState ? "enabled" : "disabled");
diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
index 3ecd08cf3d48d..836653c56fbc5 100644
--- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
@@ -4,9 +4,10 @@ grant {
   // needed because of problems in unbound LDAP library
   permission java.util.PropertyPermission "*", "read,write";
 
-  // needed because of SAML (cf. o.e.x.s.s.RestorableContextClassLoader)
+  // needed because of SAML (cf. o.e.x.c.s.s.RestorableContextClassLoader)
   permission java.lang.RuntimePermission "getClassLoader";
   permission java.lang.RuntimePermission "setContextClassLoader";
+
   // needed during initialization of OpenSAML library where xml security algorithms are registered
   // see https://github.com/apache/santuario-java/blob/e79f1fe4192de73a975bc7246aee58ed0703343d/src/main/java/org/apache/xml/security/utils/JavaUtils.java#L205-L220
   // and https://git.shibboleth.net/view/?p=java-opensaml.git;a=blob;f=opensaml-xmlsec-impl/src/main/java/org/opensaml/xmlsec/signature/impl/SignatureMarshaller.java;hb=db0eaa64210f0e32d359cd6c57bedd57902bf811#l52
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ScrollHelperIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ScrollHelperIntegTests.java
index 4db62a93be5de..5bd64c47137f8 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ScrollHelperIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ScrollHelperIntegTests.java
@@ -92,7 +92,7 @@ public void testFetchAllByEntityWithBrokenScroll() {
             false,
             1);
         SearchResponse response = new SearchResponse(internalResponse, scrollId, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY,
-                SearchResponse.Clusters.EMPTY);
+                SearchResponse.Clusters.EMPTY, null);
 
         Answer<?> returnResponse = invocation -> {
             @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java
index 618a1538a1561..bc31a50965a54 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java
@@ -70,7 +70,7 @@ public void init() throws Exception {
 
     public void testAvailable() {
         SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(
-            mock(TransportService.class), mock(ActionFilters.class), settings, licenseState);
+            mock(TransportService.class), mock(ActionFilters.class), licenseState);
         when(licenseState.isSecurityAvailable()).thenReturn(true);
         assertThat(featureSet.available(), is(true));
 
@@ -80,24 +80,28 @@ public void testAvailable() {
 
     public void testEnabled() {
         SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(
-            mock(TransportService.class), mock(ActionFilters.class), settings, licenseState);
+            mock(TransportService.class), mock(ActionFilters.class), licenseState);
+        when(licenseState.isSecurityEnabled()).thenReturn(true);
         assertThat(featureSet.enabled(), is(true));
 
-        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(true);
+        when(licenseState.isSecurityEnabled()).thenReturn(false);
         featureSet = new SecurityInfoTransportAction(
-            mock(TransportService.class), mock(ActionFilters.class), settings, licenseState);
+            mock(TransportService.class), mock(ActionFilters.class), licenseState);
         assertThat(featureSet.enabled(), is(false));
     }
 
     public void testUsage() throws Exception {
         final boolean authcAuthzAvailable = randomBoolean();
+        final boolean explicitlyDisabled = randomBoolean();
+        final boolean enabled = explicitlyDisabled == false && randomBoolean();
         when(licenseState.isSecurityAvailable()).thenReturn(authcAuthzAvailable);
+        when(licenseState.isSecurityEnabled()).thenReturn(enabled);
 
         Settings.Builder settings = Settings.builder().put(this.settings);
 
-        boolean enabled = randomBoolean();
-        settings.put(XPackSettings.SECURITY_ENABLED.getKey(), enabled);
-
+        if (explicitlyDisabled) {
+            settings.put("xpack.security.enabled", "false");
+        }
         final boolean httpSSLEnabled = randomBoolean();
         settings.put("xpack.security.http.ssl.enabled", httpSSLEnabled);
         final boolean transportSSLEnabled = randomBoolean();
@@ -226,8 +230,13 @@ public void testUsage() throws Exception {
                 // FIPS 140
                 assertThat(source.getValue("fips_140.enabled"), is(fips140Enabled));
             } else {
+                if (explicitlyDisabled) {
+                    assertThat(source.getValue("ssl"), is(nullValue()));
+                } else {
+                    assertThat(source.getValue("ssl.http.enabled"), is(httpSSLEnabled));
+                    assertThat(source.getValue("ssl.transport.enabled"), is(transportSSLEnabled));
+                }
                 assertThat(source.getValue("realms"), is(nullValue()));
-                assertThat(source.getValue("ssl"), is(nullValue()));
                 assertThat(source.getValue("token_service"), is(nullValue()));
                 assertThat(source.getValue("api_key_service"), is(nullValue()));
                 assertThat(source.getValue("audit"), is(nullValue()));
@@ -240,7 +249,7 @@ public void testUsage() throws Exception {
 
     public void testUsageOnTrialLicenseWithSecurityDisabledByDefault() throws Exception {
         when(licenseState.isSecurityAvailable()).thenReturn(true);
-        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(true);
+        when(licenseState.isSecurityEnabled()).thenReturn(false);
 
         Settings.Builder settings = Settings.builder().put(this.settings);
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java
index 4c8f626e952bb..a9e824506bd1c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java
@@ -168,7 +168,7 @@ void doExecute(ActionType<Response> action, Request request, ActionListener<Resp
                     final SearchHit[] hits = searchFunction.apply(searchRequest);
                     final SearchResponse response = new SearchResponse(
                         new SearchResponseSections(new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                            null, null, false, false, null, 1), "_scrollId1", 1, 1, 0, 1, null, null);
+                            null, null, false, false, null, 1), "_scrollId1", 1, 1, 0, 1, null, null, null);
                     listener.onResponse((Response) response);
                 } else if (SearchScrollAction.NAME.equals(action.name())){
                     assertThat(request, instanceOf(SearchScrollRequest.class));
@@ -176,7 +176,7 @@ void doExecute(ActionType<Response> action, Request request, ActionListener<Resp
                     final SearchHit[] hits = searchScrollFunction.apply(searchScrollRequest);
                     final SearchResponse response = new SearchResponse(
                         new SearchResponseSections(new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                            null, null, false, false, null, 1), "_scrollId1", 1, 1, 0, 1, null, null);
+                            null, null, false, false, null, 1), "_scrollId1", 1, 1, 0, 1, null, null, null);
                     listener.onResponse((Response) response);
                 } else if (ClearScrollAction.NAME.equals(action.name())) {
                     assertThat(request, instanceOf(ClearScrollRequest.class));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index e616b07c3f54f..bcf795f874030 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -725,7 +725,7 @@ public void testAuthenticateTransportContextAndHeader() throws Exception {
                 threadContext2.writeTo(output);
                 StreamInput input = output.bytes().streamInput();
                 threadContext2 = new ThreadContext(Settings.EMPTY);
-                threadContext2.readFrom(input);
+                threadContext2.readHeaders(input);
                 header = threadContext2.getHeader(AuthenticationField.AUTHENTICATION_KEY);
             }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
index 9d96f1e115869..37490d0ddec11 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheResponse;
+import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
@@ -210,6 +211,20 @@ public void testCacheIsNotClearedIfNoRealmsAreAttached() {
         assertEquals(0, numInvalidation.get());
     }
 
+    public void testPutRoleMappingWillValidateTemplateRoleNamesBeforeSave() {
+        final PutRoleMappingRequest putRoleMappingRequest = mock(PutRoleMappingRequest.class);
+        final TemplateRoleName templateRoleName = mock(TemplateRoleName.class);
+        final ScriptService scriptService = mock(ScriptService.class);
+        when(putRoleMappingRequest.getRoleTemplates()).thenReturn(Collections.singletonList(templateRoleName));
+        doAnswer(invocationOnMock -> {
+            throw new IllegalArgumentException();
+        }).when(templateRoleName).validate(scriptService);
+
+        final NativeRoleMappingStore nativeRoleMappingStore =
+            new NativeRoleMappingStore(Settings.EMPTY, mock(Client.class), mock(SecurityIndexManager.class), scriptService);
+        expectThrows(IllegalArgumentException.class, () -> nativeRoleMappingStore.putRoleMapping(putRoleMappingRequest, null));
+    }
+
     private NativeRoleMappingStore buildRoleMappingStoreForInvalidationTesting(AtomicInteger invalidationCounter, boolean attachRealm) {
         final Settings settings = Settings.builder().put("path.home", createTempDir()).build();
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java
index 3066ee2732a5a..dd160f2c8e6ec 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.xpack.core.security.action.user.PutUserAction;
 import org.elasticsearch.xpack.core.security.action.user.UserRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
@@ -291,7 +292,7 @@ public void testSameUserPermissionAllowsSelfApiKeyInfoRetrievalWhenAuthenticated
         final Authentication.RealmRef authenticatedBy = mock(Authentication.RealmRef.class);
         when(authentication.getUser()).thenReturn(user);
         when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
-        when(authenticatedBy.getType()).thenReturn(ApiKeyService.API_KEY_REALM_TYPE);
+        when(authentication.getAuthenticationType()).thenReturn(AuthenticationType.API_KEY);
         when(authentication.getMetadata()).thenReturn(Map.of(ApiKeyService.API_KEY_ID_KEY, apiKeyId));
 
         assertTrue(engine.checkSameUserPermissions(GetApiKeyAction.NAME, request, authentication));
@@ -321,7 +322,7 @@ public void testSameUserPermissionDeniesApiKeyInfoRetrievalWhenLookedupByIsPrese
         when(authentication.getUser()).thenReturn(user);
         when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
         when(authentication.getLookedUpBy()).thenReturn(lookedupBy);
-        when(authenticatedBy.getType()).thenReturn(ApiKeyService.API_KEY_REALM_TYPE);
+        when(authentication.getAuthenticationType()).thenReturn(AuthenticationType.API_KEY);
         when(authentication.getMetadata()).thenReturn(Map.of(ApiKeyService.API_KEY_ID_KEY, randomAlphaOfLengthBetween(4, 7)));
 
         final AssertionError assertionError = expectThrows(AssertionError.class, () -> engine.checkSameUserPermissions(GetApiKeyAction.NAME,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
index ea0fd1ae5c08d..53ad7bcbcba76 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
@@ -25,6 +25,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import static org.elasticsearch.test.SecurityTestsUtils.assertAuthorizationExceptionDefaultUsers;
 import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationExceptionDefaultUsers;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits;
@@ -33,6 +34,7 @@
 import static org.hamcrest.core.IsInstanceOf.instanceOf;
 import static org.hamcrest.number.OrderingComparison.greaterThan;
 
+@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53340")
 public class ReadActionsTests extends SecurityIntegTestCase {
 
     @Override
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
index f0cfe173a083a..afaeaa34de195 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
@@ -174,7 +174,7 @@ public void testGetPrivilegesByApplicationName() throws Exception {
         listener.get().onResponse(new SearchResponse(new SearchResponseSections(
             new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
             null, null, false, false, null, 1),
-        "_scrollId1", 1, 1, 0, 1, null, null));
+        "_scrollId1", 1, 1, 0, 1, null, null, null));
 
         assertResult(sourcePrivileges, future);
     }
@@ -196,7 +196,7 @@ public void testGetPrivilegesByWildcardApplicationName() throws Exception {
         listener.get().onResponse(new SearchResponse(new SearchResponseSections(
             new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
             null, null, false, false, null, 1),
-        "_scrollId1", 1, 1, 0, 1, null, null));
+        "_scrollId1", 1, 1, 0, 1, null, null, null));
     }
 
     public void testGetPrivilegesByStarApplicationName() throws Exception {
@@ -215,7 +215,7 @@ public void testGetPrivilegesByStarApplicationName() throws Exception {
         listener.get().onResponse(new SearchResponse(new SearchResponseSections(
             new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
             null, null, false, false, null, 1),
-        "_scrollId1", 1, 1, 0, 1, null, null));
+        "_scrollId1", 1, 1, 0, 1, null, null, null));
     }
 
     public void testGetAllPrivileges() throws Exception {
@@ -240,7 +240,7 @@ public void testGetAllPrivileges() throws Exception {
         listener.get().onResponse(new SearchResponse(new SearchResponseSections(
             new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
             null, null, false, false, null, 1),
-            "_scrollId1", 1, 1, 0, 1, null, null));
+            "_scrollId1", 1, 1, 0, 1, null, null, null));
 
         assertResult(sourcePrivileges, future);
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandlerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandlerTests.java
index 0000616002411..1c61bf8720872 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandlerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandlerTests.java
@@ -27,11 +27,11 @@
 public class SecurityBaseRestHandlerTests extends ESTestCase {
 
     public void testSecurityBaseRestHandlerChecksLicenseState() throws Exception {
-        final boolean securityDisabledByLicenseDefaults = randomBoolean();
+        final boolean securityDefaultEnabled = randomBoolean();
         final AtomicBoolean consumerCalled = new AtomicBoolean(false);
         final XPackLicenseState licenseState = mock(XPackLicenseState.class);
         when(licenseState.isSecurityAvailable()).thenReturn(true);
-        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(securityDisabledByLicenseDefaults);
+        when(licenseState.isSecurityEnabled()).thenReturn(securityDefaultEnabled);
         when(licenseState.getOperationMode()).thenReturn(
             randomFrom(License.OperationMode.BASIC, License.OperationMode.STANDARD, License.OperationMode.GOLD));
         SecurityBaseRestHandler handler = new SecurityBaseRestHandler(Settings.EMPTY, licenseState) {
@@ -56,7 +56,7 @@ protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClien
             }
         };
         FakeRestRequest fakeRestRequest = new FakeRestRequest();
-        FakeRestChannel fakeRestChannel = new FakeRestChannel(fakeRestRequest, randomBoolean(), securityDisabledByLicenseDefaults ? 1 : 0);
+        FakeRestChannel fakeRestChannel = new FakeRestChannel(fakeRestRequest, randomBoolean(), securityDefaultEnabled ? 0 : 1);
         NodeClient client = mock(NodeClient.class);
 
         assertFalse(consumerCalled.get());
@@ -64,7 +64,7 @@ protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClien
         handler.handleRequest(fakeRestRequest, fakeRestChannel, client);
 
         verify(licenseState).isSecurityAvailable();
-        if (securityDisabledByLicenseDefaults == false) {
+        if (securityDefaultEnabled) {
             assertTrue(consumerCalled.get());
             assertEquals(0, fakeRestChannel.responses().get());
             assertEquals(0, fakeRestChannel.errors().get());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java
index e51a4bfe09ebe..39201c8c602d2 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java
@@ -54,6 +54,7 @@ public void setUp() throws Exception {
                 .build();
         threadPool = new ThreadPool(settings);
         when(mockLicenseState.isSecurityAvailable()).thenReturn(true);
+        when(mockLicenseState.isSecurityEnabled()).thenReturn(true);
         when(mockLicenseState.isApiKeyServiceAllowed()).thenReturn(true);
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java
index 14d6555194c76..233b42d54e682 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java
@@ -55,6 +55,7 @@ public void setUp() throws Exception {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
         threadPool = new ThreadPool(settings);
         when(mockLicenseState.isSecurityAvailable()).thenReturn(true);
+        when(mockLicenseState.isSecurityEnabled()).thenReturn(true);
         when(mockLicenseState.isApiKeyServiceAllowed()).thenReturn(true);
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java
index 48ef745a5c4a6..640ae82a23f1a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java
@@ -54,6 +54,7 @@ public void setUp() throws Exception {
                 .build();
         threadPool = new ThreadPool(settings);
         when(mockLicenseState.isSecurityAvailable()).thenReturn(true);
+        when(mockLicenseState.isSecurityEnabled()).thenReturn(true);
         when(mockLicenseState.isApiKeyServiceAllowed()).thenReturn(true);
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListenerTests.java
index da18d5dc902d4..249c6b3d2a4e5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListenerTests.java
@@ -47,7 +47,7 @@ public void cleanup() {
     }
 
     public void testSecurityEnabledToDisabled() {
-        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(false);
+        when(licenseState.isSecurityEnabled()).thenReturn(true);
 
         when(licenseState.getOperationMode()).thenReturn(License.OperationMode.GOLD);
         logAppender.addExpectation(new MockLogAppender.SeenEventExpectation(
@@ -66,7 +66,7 @@ public void testSecurityEnabledToDisabled() {
             "Active license is now [PLATINUM]; Security is enabled"
         ));
 
-        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(true);
+        when(licenseState.isSecurityEnabled()).thenReturn(false);
         when(licenseState.getOperationMode()).thenReturn(License.OperationMode.BASIC);
         logAppender.addExpectation(new MockLogAppender.SeenEventExpectation(
             "change to basic",
@@ -80,7 +80,7 @@ public void testSecurityEnabledToDisabled() {
     }
 
     public void testSecurityDisabledToEnabled() {
-        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(true);
+        when(licenseState.isSecurityEnabled()).thenReturn(false);
 
         when(licenseState.getOperationMode()).thenReturn(License.OperationMode.TRIAL);
         logAppender.addExpectation(new MockLogAppender.SeenEventExpectation(
@@ -99,7 +99,7 @@ public void testSecurityDisabledToEnabled() {
             "Active license is now [BASIC]; Security is disabled"
         ));
 
-        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(false);
+        when(licenseState.isSecurityEnabled()).thenReturn(true);
         when(licenseState.getOperationMode()).thenReturn(License.OperationMode.PLATINUM);
         logAppender.addExpectation(new MockLogAppender.SeenEventExpectation(
             "change to platinum",
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java
index cb10395e3925c..7f4385fe01518 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/Debug.java
@@ -113,10 +113,9 @@ private static DebugLog logger(JdbcConfiguration info, PrintWriter managedPrinte
             synchronized (Debug.class) {
                 log = OUTPUT_MANAGED.get(managedPrinter);
                 if (log == null) {
-                    log = new DebugLog(managedPrinter);
+                    log = createLog(managedPrinter, info.flushAlways());
                     OUTPUT_MANAGED.put(managedPrinter, log);
                 }
-                return log;
             }
         }
 
@@ -135,7 +134,7 @@ private static DebugLog logger(JdbcConfiguration info, PrintWriter managedPrinte
                 ERR = null;
             }
             if (ERR == null) {
-                ERR = new DebugLog(new PrintWriter(new OutputStreamWriter(sys, StandardCharsets.UTF_8)));
+                ERR = createLog(new PrintWriter(new OutputStreamWriter(sys, StandardCharsets.UTF_8)), info.flushAlways());
             }
             return ERR;
         }
@@ -154,7 +153,7 @@ private static DebugLog logger(JdbcConfiguration info, PrintWriter managedPrinte
             }
 
             if (OUT == null) {
-                OUT = new DebugLog(new PrintWriter(new OutputStreamWriter(sys, StandardCharsets.UTF_8)));
+                OUT = createLog(new PrintWriter(new OutputStreamWriter(sys, StandardCharsets.UTF_8)), info.flushAlways());
             }
             return OUT;
         }
@@ -165,7 +164,7 @@ private static DebugLog logger(JdbcConfiguration info, PrintWriter managedPrinte
                 // must be local file
                 try {
                     PrintWriter print = new PrintWriter(Files.newBufferedWriter(Paths.get("").resolve(out), StandardCharsets.UTF_8));
-                    log = new DebugLog(print);
+                    log = createLog(print, info.flushAlways());
                     OUTPUT_CACHE.put(out, log);
                     OUTPUT_REFS.put(out, Integer.valueOf(0));
                 } catch (Exception ex) {
@@ -178,6 +177,12 @@ private static DebugLog logger(JdbcConfiguration info, PrintWriter managedPrinte
         return log;
     }
 
+    private static DebugLog createLog(PrintWriter print, boolean flushAlways) {
+        DebugLog log = new DebugLog(print, flushAlways);
+        log.logSystemInfo();
+        return log;
+    }
+
     static void release(JdbcConfiguration info) {
         if (!info.debug()) {
             return;
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/DebugLog.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/DebugLog.java
index e439502e971fa..c3f5598d16261 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/DebugLog.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/DebugLog.java
@@ -17,9 +17,11 @@ final class DebugLog {
     private static final String HEADER = "%tF/%tT.%tL - ";
 
     final PrintWriter print;
+    private boolean flushAlways;
 
-    DebugLog(PrintWriter print) {
+    DebugLog(PrintWriter print, boolean flushAlways) {
         this.print = print;
+        this.flushAlways = flushAlways;
     }
 
     void logMethod(Method m, Object[] args) {
@@ -31,9 +33,11 @@ void logMethod(Method m, Object[] args) {
                 m.getName(),
                 //array(m.getParameterTypes()),
                 array(args));
+        if (flushAlways) {
+            print.flush();
+        }
     }
 
-
     void logResult(Method m, Object[] args, Object r) {
         long time = System.currentTimeMillis();
         print.printf(Locale.ROOT, HEADER + "%s#%s(%s) returned %s%n",
@@ -44,6 +48,9 @@ void logResult(Method m, Object[] args, Object r) {
                 //array(m.getParameterTypes()),
                 array(args),
                 r);
+        if (flushAlways) {
+            print.flush();
+        }
     }
 
     void logException(Method m, Object[] args, Throwable t) {
@@ -57,6 +64,25 @@ void logException(Method m, Object[] args, Throwable t) {
         print.flush();
     }
 
+    void logSystemInfo() {
+        long time = System.currentTimeMillis();
+        print.printf(Locale.ROOT, HEADER + "OS[%s/%s/%s], JVM[%s/%s/%s/%s]",
+                time, time, time,
+                System.getProperty("os.name"),
+                System.getProperty("os.version"),
+                System.getProperty("os.arch"),
+                System.getProperty("java.vm.vendor"),
+                System.getProperty("java.vm.name"),
+                System.getProperty("java.version"),
+                System.getProperty("java.vm.version"));
+        print.println();
+        time = System.currentTimeMillis();
+        print.printf(Locale.ROOT, HEADER + "JVM default timezone: %s",
+                time, time, time,
+                java.util.TimeZone.getDefault().toString());
+        print.println();
+        print.flush();
+    }
 
     private static String array(Object[] a) {
         if (a == null || a.length == 0) {
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
index 8fd4c6119b434..04ba6046229b1 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
@@ -47,6 +47,10 @@ public class JdbcConfiguration extends ConnectionConfiguration {
     // can be out/err/url
     static final String DEBUG_OUTPUT_DEFAULT = "err";
 
+    static final String DEBUG_FLUSH_ALWAYS = "debug.flushAlways";
+    // can be buffered/immediate
+    static final String DEBUG_FLUSH_ALWAYS_DEFAULT = "false";
+
     public static final String TIME_ZONE = "timezone";
     // follow the JDBC spec and use the JVM default...
     // to avoid inconsistency, the default is picked up once at startup and reused across connections
@@ -63,7 +67,7 @@ public class JdbcConfiguration extends ConnectionConfiguration {
 
     // options that don't change at runtime
     private static final Set<String> OPTION_NAMES = new LinkedHashSet<>(
-            Arrays.asList(TIME_ZONE, FIELD_MULTI_VALUE_LENIENCY, INDEX_INCLUDE_FROZEN, DEBUG, DEBUG_OUTPUT));
+            Arrays.asList(TIME_ZONE, FIELD_MULTI_VALUE_LENIENCY, INDEX_INCLUDE_FROZEN, DEBUG, DEBUG_OUTPUT, DEBUG_FLUSH_ALWAYS));
 
     static {
         // trigger version initialization
@@ -76,6 +80,7 @@ public class JdbcConfiguration extends ConnectionConfiguration {
     // immutable properties
     private final boolean debug;
     private final String debugOut;
+    private final boolean flushAlways;
 
     // mutable ones
     private ZoneId zoneId;
@@ -158,6 +163,8 @@ private JdbcConfiguration(URI baseURI, String u, Properties props) throws JdbcSQ
 
         this.debug = parseValue(DEBUG, props.getProperty(DEBUG, DEBUG_DEFAULT), Boolean::parseBoolean);
         this.debugOut = props.getProperty(DEBUG_OUTPUT, DEBUG_OUTPUT_DEFAULT);
+        this.flushAlways = parseValue(DEBUG_FLUSH_ALWAYS, props.getProperty(DEBUG_FLUSH_ALWAYS, DEBUG_FLUSH_ALWAYS_DEFAULT),
+                Boolean::parseBoolean);
 
         this.zoneId = parseValue(TIME_ZONE, props.getProperty(TIME_ZONE, TIME_ZONE_DEFAULT),
                 s -> TimeZone.getTimeZone(s).toZoneId().normalized());
@@ -184,6 +191,10 @@ public String debugOut() {
         return debugOut;
     }
 
+    public boolean flushAlways() {
+        return flushAlways;
+    }
+
     public TimeZone timeZone() {
         return zoneId != null ? TimeZone.getTimeZone(zoneId) : null;
     }
diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java
index 9b7eb9262aac1..680b1eebe4883 100644
--- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java
+++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java
@@ -75,6 +75,23 @@ public void testDebugOut() throws Exception {
         assertThat(ci.debugOut(), is("jdbc.out"));
     }
 
+    public void testDebugFlushAlways() throws Exception {
+        JdbcConfiguration ci = ci("jdbc:es://a:1/?debug=true&debug.flushAlways=false");
+        assertThat(ci.baseUri().toString(), is("http://a:1/"));
+        assertThat(ci.debug(), is(true));
+        assertThat(ci.flushAlways(), is(false));
+
+        ci = ci("jdbc:es://a:1/?debug=true&debug.flushAlways=true");
+        assertThat(ci.baseUri().toString(), is("http://a:1/"));
+        assertThat(ci.debug(), is(true));
+        assertThat(ci.flushAlways(), is(true));
+
+        ci = ci("jdbc:es://a:1/?debug=true");
+        assertThat(ci.baseUri().toString(), is("http://a:1/"));
+        assertThat(ci.debug(), is(true));
+        assertThat(ci.flushAlways(), is(false));
+    }
+
     public void testTypeInParam() throws Exception {
         Exception e = expectThrows(JdbcSQLException.class, () -> ci("jdbc:es://a:1/foo/bar/tar?debug=true&debug.out=jdbc.out"));
         assertEquals("Unknown parameter [debug.out]; did you mean [debug.output]", e.getMessage());
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
index 0f656dcfcd6e9..7a30ead6c9764 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.qa.rest;
 
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
+
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
@@ -715,7 +716,7 @@ public void testTranslateQueryWithGroupByAndHaving() throws IOException {
         @SuppressWarnings("unchecked")
         Map<String, Object> termsScript = (Map<String, Object>) terms.get("script");
         assertEquals(3, termsScript.size());
-        assertEquals("InternalSqlScriptUtils.abs(InternalSqlScriptUtils.docValue(doc,params.v0))", termsScript.get("source"));
+        assertEquals("InternalSqlScriptUtils.abs(InternalQlScriptUtils.docValue(doc,params.v0))", termsScript.get("source"));
         assertEquals("painless", termsScript.get("lang"));
 
         @SuppressWarnings("unchecked")
@@ -764,7 +765,7 @@ public void testTranslateQueryWithGroupByAndHaving() throws IOException {
         @SuppressWarnings("unchecked")
         Map<String, Object> filterScript = (Map<String, Object>) bucketSelector.get("script");
         assertEquals(3, filterScript.size());
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(params.a0,params.v0))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(params.a0,params.v0))",
             filterScript.get("source"));
         assertEquals("painless", filterScript.get("lang"));
         @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
deleted file mode 100644
index fe30b9975cab4..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-7f057455901.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-39933692162e28c2719b60f499204b28236a2858
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0.jar.sha1
new file mode 100644
index 0000000000000..588b450154d09
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0.jar.sha1
@@ -0,0 +1 @@
+3f9ea85fff4fc3f7c83869dddb9b0ef7818c0cae
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/SqlFunctionRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/SqlFunctionRegistry.java
index ec1656f0dad54..7b651694b1c01 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/SqlFunctionRegistry.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/SqlFunctionRegistry.java
@@ -8,7 +8,6 @@
 import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.expression.function.aggregate.Count;
-import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mod;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.Avg;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.First;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.Kurtosis;
@@ -111,6 +110,7 @@
 import org.elasticsearch.xpack.sql.expression.predicate.conditional.Iif;
 import org.elasticsearch.xpack.sql.expression.predicate.conditional.Least;
 import org.elasticsearch.xpack.sql.expression.predicate.conditional.NullIf;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Mod;
 
 public class SqlFunctionRegistry extends FunctionRegistry {
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringFunction.java
index 57a1abe58d077..2901506930e24 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringFunction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringFunction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
 import org.elasticsearch.xpack.ql.tree.Source;
 
 import java.util.Locale;
@@ -66,7 +67,7 @@ protected String scriptMethodName() {
 
     @Override
     public ScriptTemplate scriptWithField(FieldAttribute field) {
-        return new ScriptTemplate(processScript("doc[{}].value"),
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
                 paramsBuilder().variable(field.exactAttribute().name()).build(),
                 dataType());
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java
index f811de23e0f7c..affa06af3c064 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.xpack.ql.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
@@ -79,7 +80,7 @@ protected NodeInfo<? extends Expression> info() {
 
     @Override
     public ScriptTemplate scriptWithField(FieldAttribute field) {
-        return new ScriptTemplate(processScript("doc[{}].value"),
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
                 paramsBuilder().variable(field.exactAttribute().name()).build(),
                 dataType());
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Insert.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Insert.java
index 7b0ef796c0f4b..524b513f04861 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Insert.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Insert.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
@@ -121,7 +122,7 @@ private ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate
     
     @Override
     public ScriptTemplate scriptWithField(FieldAttribute field) {
-        return new ScriptTemplate(processScript("doc[{}].value"),
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
                 paramsBuilder().variable(field.exactAttribute().name()).build(),
                 dataType());
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Locate.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Locate.java
index f45bd075a9499..f92b2eaaa78cb 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Locate.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Locate.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
@@ -123,7 +124,7 @@ private ScriptTemplate asScriptFrom(ScriptTemplate patternScript, ScriptTemplate
     
     @Override
     public ScriptTemplate scriptWithField(FieldAttribute field) {
-        return new ScriptTemplate(processScript("doc[{}].value"),
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
                 paramsBuilder().variable(field.exactAttribute().name()).build(),
                 dataType());
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Replace.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Replace.java
index 05203aa2816d7..814fa22c665e5 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Replace.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Replace.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
@@ -108,7 +109,7 @@ private ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate
 
     @Override
     public ScriptTemplate scriptWithField(FieldAttribute field) {
-        return new ScriptTemplate(processScript("doc[{}].value"),
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
                 paramsBuilder().variable(field.exactAttribute().name()).build(),
                 dataType());
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringFunction.java
index e52874b5715ad..b09a560ce91c3 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringFunction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringFunction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.xpack.ql.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.util.StringUtils;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation;
@@ -56,7 +57,7 @@ protected Processor makeProcessor() {
     @Override
     public ScriptTemplate scriptWithField(FieldAttribute field) {
         //TODO change this to use _source instead of the exact form (aka field.keyword for text fields)
-        return new ScriptTemplate(processScript("doc[{}].value"),
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
                 paramsBuilder().variable(field.exactAttribute().name()).build(),
                 dataType());
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringIntFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringIntFunction.java
index 16709bf2dc3d4..cb6af2cb9fffd 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringIntFunction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringIntFunction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.xpack.ql.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation;
 
@@ -57,7 +58,7 @@ protected Processor makeProcessor() {
     
     @Override
     public ScriptTemplate scriptWithField(FieldAttribute field) {
-        return new ScriptTemplate(processScript("doc[{}].value"),
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
                 paramsBuilder().variable(field.name()).build(),
                 dataType());
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
index 13aec510d2b6b..f372ca7a6be1a 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.script.JodaCompatibleZonedDateTime;
 import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;
-import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.UnaryArithmeticProcessor.UnaryArithmeticOperation;
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateAddProcessor;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateDiffProcessor;
@@ -105,10 +104,6 @@ public static Object mul(Object left, Object right) {
         return SqlBinaryArithmeticOperation.MUL.apply(left, right);
     }
 
-    public static Number neg(Number value) {
-        return UnaryArithmeticOperation.NEGATE.apply(value);
-    }
-
     public static Object sub(Object left, Object right) {
         return SqlBinaryArithmeticOperation.SUB.apply(left, right);
     }
diff --git a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
index 6e9490dda1a94..a5330b1a813e2 100644
--- a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
+++ b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
@@ -21,10 +21,6 @@ class java.time.OffsetTime {
 }
 
 class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils {
-}
-
-class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalSqlScriptUtils {
-
 #
 # Utilities
 # 
@@ -53,7 +49,20 @@ class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalS
   Boolean not(Boolean)
   Boolean isNull(Object)
   Boolean isNotNull(Object)
+  
+#
+# Regex
+#
+  Boolean regex(String, String)
 
+#
+# Math
+#
+  Number neg(Number)
+  
+}
+
+class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalSqlScriptUtils {
 #
 # Conditional
 #
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java
index 26c7ef2b3990b..4a77050f808d1 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java
@@ -219,7 +219,8 @@ public void testFoldingBooleanOrNull_HavingClause() {
         assertEquals(EsQueryExec.class, p.getClass());
         EsQueryExec ee = (EsQueryExec) p;
         assertTrue(ee.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", "").contains(
-            "\"script\":{\"source\":\"InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(params.a0,params.v0))\"," +
+                "\"script\":{\"source\":\"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(params.a0,params.v0))\","
+                        +
             "\"lang\":\"painless\",\"params\":{\"v0\":10}},"));
         assertEquals(2, ee.output().size());
         assertThat(ee.output().get(0).toString(), startsWith("test.keyword{f}#"));
@@ -292,7 +293,7 @@ public void testGroupKeyTypes_Boolean() {
         EsQueryExec ee = (EsQueryExec) p;
         assertThat(ee.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
             endsWith("{\"script\":{" +
-                "\"source\":\"InternalSqlScriptUtils.gt(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1)\"," +
+                "\"source\":\"InternalQlScriptUtils.gt(InternalQlScriptUtils.docValue(doc,params.v0),params.v1)\"," +
                 "\"lang\":\"painless\",\"params\":{\"v0\":\"int\",\"v1\":10}},\"missing_bucket\":true," +
                 "\"value_type\":\"boolean\",\"order\":\"asc\"}}}]}}}"));
         assertEquals(2, ee.output().size());
@@ -306,7 +307,7 @@ public void testGroupKeyTypes_Integer() {
         EsQueryExec ee = (EsQueryExec) p;
         assertThat(ee.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
             endsWith("{\"script\":{" +
-                "\"source\":\"InternalSqlScriptUtils.add(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1)\"," +
+                "\"source\":\"InternalSqlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),params.v1)\"," +
                 "\"lang\":\"painless\",\"params\":{\"v0\":\"int\",\"v1\":10}},\"missing_bucket\":true," +
                 "\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}"));
         assertEquals(2, ee.output().size());
@@ -320,7 +321,7 @@ public void testGroupKeyTypes_Rational() {
         EsQueryExec ee = (EsQueryExec) p;
         assertThat(ee.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
             endsWith("{\"script\":{" +
-                "\"source\":\"InternalSqlScriptUtils.sin(InternalSqlScriptUtils.docValue(doc,params.v0))\"," +
+                "\"source\":\"InternalSqlScriptUtils.sin(InternalQlScriptUtils.docValue(doc,params.v0))\"," +
                 "\"lang\":\"painless\",\"params\":{\"v0\":\"int\"}},\"missing_bucket\":true," +
                 "\"value_type\":\"double\",\"order\":\"asc\"}}}]}}}"));
         assertEquals(2, ee.output().size());
@@ -334,7 +335,7 @@ public void testGroupKeyTypes_String() {
         EsQueryExec ee = (EsQueryExec) p;
         assertThat(ee.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
             endsWith("{\"script\":{" +
-                "\"source\":\"InternalSqlScriptUtils.lcase(InternalSqlScriptUtils.docValue(doc,params.v0))\"," +
+                "\"source\":\"InternalSqlScriptUtils.lcase(InternalQlScriptUtils.docValue(doc,params.v0))\"," +
                 "\"lang\":\"painless\",\"params\":{\"v0\":\"keyword\"}},\"missing_bucket\":true," +
                 "\"value_type\":\"string\",\"order\":\"asc\"}}}]}}}"));
         assertEquals(2, ee.output().size());
@@ -348,7 +349,7 @@ public void testGroupKeyTypes_IP() {
         EsQueryExec ee = (EsQueryExec) p;
         assertThat(ee.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
             endsWith("{\"script\":{\"source\":\"InternalSqlScriptUtils.cast(" +
-                    "InternalSqlScriptUtils.docValue(doc,params.v0),params.v1)\"," +
+                    "InternalQlScriptUtils.docValue(doc,params.v0),params.v1)\"," +
                     "\"lang\":\"painless\",\"params\":{\"v0\":\"keyword\",\"v1\":\"IP\"}}," +
                     "\"missing_bucket\":true,\"value_type\":\"ip\",\"order\":\"asc\"}}}]}}}"));
         assertEquals(2, ee.output().size());
@@ -362,7 +363,7 @@ public void testGroupKeyTypes_DateTime() {
         EsQueryExec ee = (EsQueryExec) p;
         assertThat(ee.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
             endsWith("{\"script\":{" +
-                "\"source\":\"InternalSqlScriptUtils.add(InternalSqlScriptUtils.docValue(doc,params.v0)," +
+                "\"source\":\"InternalSqlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0)," +
                 "InternalSqlScriptUtils.intervalYearMonth(params.v1,params.v2))\",\"lang\":\"painless\",\"params\":{" +
                 "\"v0\":\"date\",\"v1\":\"P1Y2M\",\"v2\":\"INTERVAL_YEAR_TO_MONTH\"}},\"missing_bucket\":true," +
                 "\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}"));
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
index 2225ead0dfd31..fcb0d74c8242b 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
@@ -389,8 +389,8 @@ public void testTranslateDateAdd_WhereClause_Painless() {
         assertNull(translation.aggFilter);
         assertTrue(translation.query instanceof ScriptQuery);
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(InternalSqlScriptUtils.dateAdd(" +
-                "params.v0,InternalSqlScriptUtils.docValue(doc,params.v1),InternalSqlScriptUtils.docValue(doc,params.v2)," +
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalSqlScriptUtils.dateAdd(" +
+                "params.v0,InternalQlScriptUtils.docValue(doc,params.v1),InternalQlScriptUtils.docValue(doc,params.v2)," +
                 "params.v3),InternalSqlScriptUtils.asDateTime(params.v4)))",
             sc.script().toString());
         assertEquals("[{v=quarter}, {v=int}, {v=date}, {v=Z}, {v=2018-09-04T00:00:00.000Z}]", sc.script().params().toString());
@@ -406,8 +406,8 @@ public void testTranslateDateDiff_WhereClause_Painless() {
         assertNull(translation.aggFilter);
         assertTrue(translation.query instanceof ScriptQuery);
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(InternalSqlScriptUtils.dateDiff(" +
-                "params.v0,InternalSqlScriptUtils.docValue(doc,params.v1),InternalSqlScriptUtils.docValue(doc,params.v2)," +
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalSqlScriptUtils.dateDiff(" +
+                "params.v0,InternalQlScriptUtils.docValue(doc,params.v1),InternalQlScriptUtils.docValue(doc,params.v2)," +
                 "params.v3),InternalSqlScriptUtils.asDateTime(params.v4)))",
             sc.script().toString());
         assertEquals("[{v=week}, {v=date}, {v=date}, {v=Z}, {v=2018-09-04T00:00:00.000Z}]", sc.script().params().toString());
@@ -423,8 +423,8 @@ public void testTranslateDateTrunc_WhereClause_Painless() {
         assertNull(translation.aggFilter);
         assertTrue(translation.query instanceof ScriptQuery);
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(InternalSqlScriptUtils.dateTrunc(" +
-                "params.v0,InternalSqlScriptUtils.docValue(doc,params.v1),params.v2),InternalSqlScriptUtils.asDateTime(params.v3)))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalSqlScriptUtils.dateTrunc(" +
+                "params.v0,InternalQlScriptUtils.docValue(doc,params.v1),params.v2),InternalSqlScriptUtils.asDateTime(params.v3)))",
             sc.script().toString());
         assertEquals("[{v=month}, {v=date}, {v=Z}, {v=2018-09-04T00:00:00.000Z}]", sc.script().params().toString());
     }
@@ -439,8 +439,8 @@ public void testTranslateDatePart_WhereClause_Painless() {
         assertNull(translation.aggFilter);
         assertTrue(translation.query instanceof ScriptQuery);
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(InternalSqlScriptUtils.datePart(" +
-                "params.v0,InternalSqlScriptUtils.docValue(doc,params.v1),params.v2),InternalSqlScriptUtils.asDateTime(params.v3)))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalSqlScriptUtils.datePart(" +
+                "params.v0,InternalQlScriptUtils.docValue(doc,params.v1),params.v2),InternalSqlScriptUtils.asDateTime(params.v3)))",
             sc.script().toString());
         assertEquals("[{v=month}, {v=date}, {v=Z}, {v=2018-09-04T00:00:00.000Z}]", sc.script().params().toString());
     }
@@ -556,8 +556,8 @@ public void testLikeRLikeAsPainlessScripts() {
         GroupingContext groupingContext = QueryFolder.FoldAggregate.groupBy(((Aggregate) p).groupings());
         assertNotNull(groupingContext);
         ScriptTemplate scriptTemplate = groupingContext.tail.script();
-        assertEquals("InternalSqlScriptUtils.caseFunction([InternalSqlScriptUtils.regex(InternalSqlScriptUtils.docValue(" +
-                     "doc,params.v0),params.v1),params.v2,InternalSqlScriptUtils.regex(InternalSqlScriptUtils.docValue(" +
+        assertEquals("InternalSqlScriptUtils.caseFunction([InternalSqlScriptUtils.regex(InternalQlScriptUtils.docValue("
+                + "doc,params.v0),params.v1),params.v2,InternalSqlScriptUtils.regex(InternalQlScriptUtils.docValue(" +
                      "doc,params.v3),params.v4),params.v5,params.v6])",
                 scriptTemplate.toString());
         assertEquals("[{v=keyword}, {v=^.*foo.*$}, {v=1}, {v=keyword}, {v=.*bar.*}, {v=2}, {v=3}]",
@@ -573,9 +573,9 @@ public void testTranslateNotExpression_WhereClause_Painless() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, false);
         assertTrue(translation.query instanceof ScriptQuery);
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.not(" +
-            "InternalSqlScriptUtils.eq(InternalSqlScriptUtils.position(" +
-            "params.v0,InternalSqlScriptUtils.docValue(doc,params.v1)),params.v2)))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.not(" +
+                "InternalQlScriptUtils.eq(InternalSqlScriptUtils.position(" +
+                "params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2)))",
             sc.script().toString());
         assertEquals("[{v=x}, {v=keyword}, {v=0}]", sc.script().params().toString());
     }
@@ -604,8 +604,8 @@ public void testTranslateIsNullExpression_WhereClause_Painless() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, false);
         assertTrue(translation.query instanceof ScriptQuery);
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.isNull(" +
-            "InternalSqlScriptUtils.position(params.v0,InternalSqlScriptUtils.docValue(doc,params.v1))))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.isNull(" +
+                "InternalSqlScriptUtils.position(params.v0,InternalQlScriptUtils.docValue(doc,params.v1))))",
             sc.script().toString());
         assertEquals("[{v=x}, {v=keyword}]", sc.script().params().toString());
     }
@@ -632,8 +632,8 @@ public void testTranslateIsNotNullExpression_WhereClause_Painless() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, false);
         assertTrue(translation.query instanceof ScriptQuery);
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.isNotNull(" +
-                "InternalSqlScriptUtils.position(params.v0,InternalSqlScriptUtils.docValue(doc,params.v1))))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.isNotNull(" +
+                "InternalSqlScriptUtils.position(params.v0,InternalQlScriptUtils.docValue(doc,params.v1))))",
             sc.script().toString());
         assertEquals("[{v=x}, {v=keyword}]", sc.script().params().toString());
     }
@@ -646,7 +646,8 @@ public void testTranslateIsNullExpression_HavingClause_Painless() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, true);
         assertNull(translation.query);
         AggFilter aggFilter = translation.aggFilter;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.isNull(params.a0))",
+        assertEquals(
+                "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.isNull(params.a0))",
             aggFilter.scriptTemplate().toString());
         assertThat(aggFilter.scriptTemplate().params().toString(), startsWith("[{a=max(int)"));
     }
@@ -659,7 +660,8 @@ public void testTranslateIsNotNullExpression_HavingClause_Painless() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, true);
         assertNull(translation.query);
         AggFilter aggFilter = translation.aggFilter;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.isNotNull(params.a0))",
+        assertEquals(
+                "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.isNotNull(params.a0))",
             aggFilter.scriptTemplate().toString());
         assertThat(aggFilter.scriptTemplate().params().toString(), startsWith("[{a=max(int)"));
     }
@@ -676,17 +678,17 @@ public void testTranslateCoalesceExpression_WhereGroupByAndHaving_Painless() {
         String aggName = aggBuilder.getSubAggregations().iterator().next().getName();
         String havingName = aggBuilder.getPipelineAggregations().iterator().next().getName();
         assertThat(aggBuilder.toString(), containsString("{\"terms\":{\"script\":{\"source\":\"" +
-                "InternalSqlScriptUtils.coalesce([InternalSqlScriptUtils.docValue(doc,params.v0)])\",\"lang\":\"painless\"" +
+                "InternalSqlScriptUtils.coalesce([InternalQlScriptUtils.docValue(doc,params.v0)])\",\"lang\":\"painless\"" +
                 ",\"params\":{\"v0\":\"int\"}},\"missing_bucket\":true,\"value_type\":\"long\",\"order\":\"asc\"}}}]}," +
                 "\"aggregations\":{\"" + aggName + "\":{\"max\":{\"field\":\"date\"}},\"" + havingName + "\":" +
                 "{\"bucket_selector\":{\"buckets_path\":{\"a0\":\"" + aggName + "\"},\"script\":{\"source\":\"" +
-                "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(InternalSqlScriptUtils.coalesce(" +
+                "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalSqlScriptUtils.coalesce(" +
                 "[params.a0]),InternalSqlScriptUtils.asDateTime(params.v0)))\",\"lang\":\"painless\",\"params\":" +
                 "{\"v0\":\"2020-01-01T00:00:00.000Z\"}}"));
         assertTrue(esQExec.queryContainer().query() instanceof ScriptQuery);
         ScriptQuery sq = (ScriptQuery) esQExec.queryContainer().query();
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(" +
-                "InternalSqlScriptUtils.coalesce([InternalSqlScriptUtils.docValue(doc,params.v0)]),params.v1))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(" +
+                "InternalSqlScriptUtils.coalesce([InternalQlScriptUtils.docValue(doc,params.v0)]),params.v1))",
                 sq.script().toString());
         assertEquals("[{v=int}, {v=10}]", sq.script().params().toString());
     }
@@ -743,8 +745,8 @@ public void testTranslateInExpression_WhereClause_Painless() {
         assertNull(translation.aggFilter);
         assertTrue(translation.query instanceof ScriptQuery);
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.in(" +
-                "InternalSqlScriptUtils.power(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1), params.v2))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.in(" +
+                "InternalSqlScriptUtils.power(InternalQlScriptUtils.docValue(doc,params.v0),params.v1), params.v2))",
             sc.script().toString());
         assertEquals("[{v=int}, {v=2}, {v=[10.0, null, 20.0]}]", sc.script().params().toString());
     }
@@ -757,7 +759,7 @@ public void testTranslateInExpression_HavingClause_Painless() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, true);
         assertNull(translation.query);
         AggFilter aggFilter = translation.aggFilter;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.in(params.a0, params.v0))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.in(params.a0, params.v0))",
             aggFilter.scriptTemplate().toString());
         assertThat(aggFilter.scriptTemplate().params().toString(), startsWith("[{a=max(int)"));
         assertThat(aggFilter.scriptTemplate().params().toString(), endsWith(", {v=[10, 20]}]"));
@@ -771,7 +773,7 @@ public void testTranslateInExpression_HavingClause_PainlessOneArg() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, true);
         assertNull(translation.query);
         AggFilter aggFilter = translation.aggFilter;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.in(params.a0, params.v0))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.in(params.a0, params.v0))",
             aggFilter.scriptTemplate().toString());
         assertThat(aggFilter.scriptTemplate().params().toString(), startsWith("[{a=max(int)"));
         assertThat(aggFilter.scriptTemplate().params().toString(), endsWith(", {v=[10]}]"));
@@ -786,7 +788,7 @@ public void testTranslateInExpression_HavingClause_PainlessAndNullHandling() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, true);
         assertNull(translation.query);
         AggFilter aggFilter = translation.aggFilter;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.in(params.a0, params.v0))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.in(params.a0, params.v0))",
             aggFilter.scriptTemplate().toString());
         assertThat(aggFilter.scriptTemplate().params().toString(), startsWith("[{a=max(int)"));
         assertThat(aggFilter.scriptTemplate().params().toString(), endsWith(", {v=[10, null, 20, 30]}]"));
@@ -804,7 +806,7 @@ public void testTranslateMathFunction_HavingClause_Painless() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, true);
         assertNull(translation.query);
         AggFilter aggFilter = translation.aggFilter;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(InternalSqlScriptUtils." +
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalSqlScriptUtils." +
                 operation.name().toLowerCase(Locale.ROOT) + "(params.a0),params.v0))",
             aggFilter.scriptTemplate().toString());
         assertThat(aggFilter.scriptTemplate().params().toString(), startsWith("[{a=max(int)"));
@@ -826,7 +828,8 @@ public void testTranslateRoundWithOneParameter() {
         GroupingContext groupingContext = QueryFolder.FoldAggregate.groupBy(((Aggregate) p).groupings());
         assertNotNull(groupingContext);
         ScriptTemplate scriptTemplate = groupingContext.tail.script();
-        assertEquals("InternalSqlScriptUtils.round(InternalSqlScriptUtils.dateTimeChrono(InternalSqlScriptUtils.docValue(doc,params.v0), "
+        assertEquals(
+                "InternalSqlScriptUtils.round(InternalSqlScriptUtils.dateTimeChrono(InternalQlScriptUtils.docValue(doc,params.v0), "
                 + "params.v1, params.v2),params.v3)",
             scriptTemplate.toString());
         assertEquals("[{v=date}, {v=Z}, {v=YEAR}, {v=null}]", scriptTemplate.params().toString());
@@ -850,7 +853,8 @@ public void testTranslateRoundWithTwoParameters() {
         GroupingContext groupingContext = QueryFolder.FoldAggregate.groupBy(((Aggregate) p).groupings());
         assertNotNull(groupingContext);
         ScriptTemplate scriptTemplate = groupingContext.tail.script();
-        assertEquals("InternalSqlScriptUtils.round(InternalSqlScriptUtils.dateTimeChrono(InternalSqlScriptUtils.docValue(doc,params.v0), "
+        assertEquals(
+                "InternalSqlScriptUtils.round(InternalSqlScriptUtils.dateTimeChrono(InternalQlScriptUtils.docValue(doc,params.v0), "
                 + "params.v1, params.v2),params.v3)",
             scriptTemplate.toString());
         assertEquals("[{v=date}, {v=Z}, {v=YEAR}, {v=-2}]", scriptTemplate.params().toString());
@@ -864,7 +868,7 @@ public void testGroupByAndHavingWithFunctionOnTopOfAggregation() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, true);
         assertNull(translation.query);
         AggFilter aggFilter = translation.aggFilter;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(InternalSqlScriptUtils.abs" +
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalSqlScriptUtils.abs" +
                 "(params.a0),params.v0))",
             aggFilter.scriptTemplate().toString());
         assertThat(aggFilter.scriptTemplate().params().toString(), startsWith("[{a=MAX(int)"));
@@ -880,7 +884,7 @@ public void testTranslateStAsWktForPoints() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, true);
         assertNull(translation.query);
         AggFilter aggFilter = translation.aggFilter;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.eq(" +
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(" +
                 "InternalSqlScriptUtils.stAswkt(InternalSqlScriptUtils.geoDocValue(doc,params.v0))," +
                 "params.v1)" +
                 ")",
@@ -897,9 +901,9 @@ public void testTranslateStWktToSql() {
         QueryTranslation translation = QueryTranslator.toQuery(condition, true);
         assertNull(translation.query);
         AggFilter aggFilter = translation.aggFilter;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(" +
-                "InternalSqlScriptUtils.eq(InternalSqlScriptUtils.stWktToSql(" +
-                "InternalSqlScriptUtils.docValue(doc,params.v0)),InternalSqlScriptUtils.stWktToSql(params.v1)))",
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(" +
+                "InternalQlScriptUtils.eq(InternalSqlScriptUtils.stWktToSql(" +
+                "InternalQlScriptUtils.docValue(doc,params.v0)),InternalSqlScriptUtils.stWktToSql(params.v1)))",
             aggFilter.scriptTemplate().toString());
         assertEquals("[{v=keyword}, {v=POINT (10.0 20.0)}]", aggFilter.scriptTemplate().params().toString());
     }
@@ -916,8 +920,8 @@ public void testTranslateStDistanceToScript() {
         assertNull(translation.aggFilter);
         assertTrue(translation.query instanceof ScriptQuery);
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(" +
-                "InternalSqlScriptUtils." + operatorFunction + "(" +
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(" +
+                "InternalQlScriptUtils." + operatorFunction + "(" +
                 "InternalSqlScriptUtils.stDistance(" +
                 "InternalSqlScriptUtils.geoDocValue(doc,params.v0),InternalSqlScriptUtils.stWktToSql(params.v1)),params.v2))",
             sc.script().toString());
@@ -952,7 +956,7 @@ public void testTranslateStXY() {
         assertNull(translation.aggFilter);
         assertThat(translation.query, instanceOf(ScriptQuery.class));
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.eq(InternalSqlScriptUtils.st" + dim + "(" +
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalSqlScriptUtils.st" + dim + "(" +
                 "InternalSqlScriptUtils.geoDocValue(doc,params.v0)),params.v1))",
             sc.script().toString());
         assertEquals("[{v=point}, {v=10}]", sc.script().params().toString());
@@ -968,7 +972,7 @@ public void testTranslateStGeometryType() {
         assertNull(translation.aggFilter);
         assertThat(translation.query, instanceOf(ScriptQuery.class));
         ScriptQuery sc = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.eq(InternalSqlScriptUtils.stGeometryType(" +
+        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalSqlScriptUtils.stGeometryType(" +
                 "InternalSqlScriptUtils.geoDocValue(doc,params.v0)),params.v1))",
             sc.script().toString());
         assertEquals("[{v=point}, {v=POINT}]", sc.script().params().toString());
@@ -982,7 +986,8 @@ public void testTranslateCoalesce_GroupBy_Painless() {
         GroupingContext groupingContext = QueryFolder.FoldAggregate.groupBy(((Aggregate) p).groupings());
         assertNotNull(groupingContext);
         ScriptTemplate scriptTemplate = groupingContext.tail.script();
-        assertEquals("InternalSqlScriptUtils.coalesce([InternalSqlScriptUtils.docValue(doc,params.v0),params.v1])",
+        assertEquals(
+                "InternalSqlScriptUtils.coalesce([InternalQlScriptUtils.docValue(doc,params.v0),params.v1])",
             scriptTemplate.toString());
         assertEquals("[{v=int}, {v=10}]", scriptTemplate.params().toString());
     }
@@ -995,7 +1000,8 @@ public void testTranslateNullIf_GroupBy_Painless() {
         GroupingContext groupingContext = QueryFolder.FoldAggregate.groupBy(((Aggregate) p).groupings());
         assertNotNull(groupingContext);
         ScriptTemplate scriptTemplate = groupingContext.tail.script();
-        assertEquals("InternalSqlScriptUtils.nullif(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1)",
+        assertEquals(
+                "InternalSqlScriptUtils.nullif(InternalQlScriptUtils.docValue(doc,params.v0),params.v1)",
             scriptTemplate.toString());
         assertEquals("[{v=int}, {v=10}]", scriptTemplate.params().toString());
     }
@@ -1008,8 +1014,8 @@ public void testTranslateCase_GroupBy_Painless() {
         GroupingContext groupingContext = QueryFolder.FoldAggregate.groupBy(((Aggregate) p).groupings());
         assertNotNull(groupingContext);
         ScriptTemplate scriptTemplate = groupingContext.tail.script();
-        assertEquals("InternalSqlScriptUtils.caseFunction([InternalSqlScriptUtils.gt(InternalSqlScriptUtils.docValue(" + "" +
-                "doc,params.v0),params.v1),params.v2,InternalSqlScriptUtils.gt(InternalSqlScriptUtils.docValue(doc,params.v3)," +
+        assertEquals("InternalSqlScriptUtils.caseFunction([InternalQlScriptUtils.gt(InternalQlScriptUtils.docValue(" + ""
+                + "doc,params.v0),params.v1),params.v2,InternalQlScriptUtils.gt(InternalQlScriptUtils.docValue(doc,params.v3)," +
                 "params.v4),params.v5,params.v6])",
             scriptTemplate.toString());
         assertEquals("[{v=int}, {v=10}, {v=foo}, {v=int}, {v=20}, {v=bar}, {v=default}]", scriptTemplate.params().toString());
@@ -1023,8 +1029,8 @@ public void testTranslateIif_GroupBy_Painless() {
         GroupingContext groupingContext = QueryFolder.FoldAggregate.groupBy(((Aggregate) p).groupings());
         assertNotNull(groupingContext);
         ScriptTemplate scriptTemplate = groupingContext.tail.script();
-        assertEquals("InternalSqlScriptUtils.caseFunction([InternalSqlScriptUtils.gt("  +
-                "InternalSqlScriptUtils.docValue(doc,params.v0),params.v1),params.v2,params.v3])",
+        assertEquals("InternalSqlScriptUtils.caseFunction([InternalQlScriptUtils.gt("  +
+                "InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2,params.v3])",
             scriptTemplate.toString());
         assertEquals("[{v=int}, {v=20}, {v=foo}, {v=bar}]", scriptTemplate.params().toString());
     }
@@ -1155,7 +1161,7 @@ public void testGroupByMoreDaysHistogram_WithFunction_QueryTranslator() {
         assertEquals(DATETIME, eqe.output().get(0).dataType());
         assertThat(eqe.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
                 endsWith("\"date_histogram\":{\"script\":{\"source\":\"InternalSqlScriptUtils.add(" +
-                        "InternalSqlScriptUtils.docValue(doc,params.v0),InternalSqlScriptUtils.intervalDayTime(params.v1,params.v2))\"," +
+                        "InternalQlScriptUtils.docValue(doc,params.v0),InternalSqlScriptUtils.intervalDayTime(params.v1,params.v2))\"," +
                         "\"lang\":\"painless\",\"params\":{\"v0\":\"date\",\"v1\":\"PT120H\",\"v2\":\"INTERVAL_DAY\"}}," +
                         "\"missing_bucket\":true,\"value_type\":\"long\",\"order\":\"asc\"," +
                         "\"calendar_interval\":\"1d\",\"time_zone\":\"Z\"}}}]}}}"));
@@ -1170,7 +1176,7 @@ public void testGroupByYearAndScalarsQueryTranslator() {
         assertEquals(INTEGER, eqe.output().get(0).dataType());
         assertThat(eqe.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
                 endsWith("\"date_histogram\":{\"script\":{\"source\":\"InternalSqlScriptUtils.cast(" +
-                        "InternalSqlScriptUtils.add(InternalSqlScriptUtils.docValue(doc,params.v0)," +
+                        "InternalSqlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0)," +
                         "InternalSqlScriptUtils.intervalYearMonth(params.v1,params.v2)),params.v3)\"," +
                         "\"lang\":\"painless\",\"params\":{\"v0\":\"date\",\"v1\":\"P5M\",\"v2\":\"INTERVAL_MONTH\"," +
                         "\"v3\":\"DATE\"}},\"missing_bucket\":true,\"value_type\":\"long\",\"order\":\"asc\"," +
@@ -1185,8 +1191,9 @@ public void testOrderByYear() {
         assertEquals("YEAR(date)", eqe.output().get(0).qualifiedName());
         assertEquals(INTEGER, eqe.output().get(0).dataType());
         assertThat(eqe.queryContainer().toString().replaceAll("\\s+", ""),
-                endsWith("\"sort\":[{\"_script\":{\"script\":{\"source\":\"InternalSqlScriptUtils.nullSafeSortNumeric(" +
-                        "InternalSqlScriptUtils.dateTimeChrono(InternalSqlScriptUtils.docValue(doc,params.v0)," +
+                endsWith("\"sort\":[{\"_script\":{\"script\":{\"source\":\"InternalQlScriptUtils.nullSafeSortNumeric("
+                        +
+                        "InternalSqlScriptUtils.dateTimeChrono(InternalQlScriptUtils.docValue(doc,params.v0)," +
                         "params.v1,params.v2))\",\"lang\":\"painless\",\"params\":{\"v0\":\"date\",\"v1\":\"Z\"," +
                         "\"v2\":\"YEAR\"}},\"type\":\"number\",\"order\":\"asc\"}}]}"));
     }
@@ -1309,17 +1316,17 @@ public void testAllCountVariantsWithHavingGenerateCorrectAggregations() {
                         + "\"a4\":\"_count\","
                         + "\"a5\":\"" + avgInt.getName() + "\"},"
                         + "\"script\":{\"source\":\""
-                        + "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.and("
-                        +   "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.and("
-                        +     "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.and("
-                        +       "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.and("
-                        +         "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.and("
-                        +           "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(params.a0,params.v0)),"
-                        +           "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(params.a1,params.v1)))),"
-                        +         "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(params.a2,params.v2)))),"
-                        +       "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(params.a3,params.v3)))),"
-                        +     "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(params.a4,params.v4)))),"
-                        +   "InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(params.a5,params.v5))))\","
+                        + "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.and("
+                        +   "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.and("
+                        +     "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.and("
+                        +       "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.and("
+                        +         "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.and("
+                        +           "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(params.a0,params.v0)),"
+                        +           "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(params.a1,params.v1)))),"
+                        +         "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(params.a2,params.v2)))),"
+                        +       "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(params.a3,params.v3)))),"
+                        +     "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(params.a4,params.v4)))),"
+                        +   "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(params.a5,params.v5))))\","
                         + "\"lang\":\"painless\",\"params\":{\"v0\":3,\"v1\":32,\"v2\":1,\"v3\":2,\"v4\":5,\"v5\":50000}},"
                         + "\"gap_policy\":\"skip\"}}}}}"));
     }
@@ -1335,7 +1342,7 @@ public void testGroupByCastScalar() {
             ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                 .replaceAll("\\s+", ""),
             endsWith("{\"source\":\"InternalSqlScriptUtils.cast(InternalSqlScriptUtils.abs(InternalSqlScriptUtils.dateTimeChrono" +
-                "(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
+                        "(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
                 "\"params\":{\"v0\":\"date\",\"v1\":\"Z\",\"v2\":\"YEAR\",\"v3\":\"LONG\"}},\"missing_bucket\":true," +
                 "\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}")
         );
@@ -1352,7 +1359,7 @@ public void testGroupByCastScalarWithAlias() {
             ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                 .replaceAll("\\s+", ""),
             endsWith("{\"source\":\"InternalSqlScriptUtils.cast(InternalSqlScriptUtils.abs(InternalSqlScriptUtils.dateTimeChrono" +
-                "(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
+                        "(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
                 "\"params\":{\"v0\":\"date\",\"v1\":\"Z\",\"v2\":\"YEAR\",\"v3\":\"LONG\"}},\"missing_bucket\":true," +
                 "\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}")
         );
@@ -1369,7 +1376,7 @@ public void testGroupByCastScalarWithNumericRef() {
             ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                 .replaceAll("\\s+", ""),
             endsWith("{\"source\":\"InternalSqlScriptUtils.cast(InternalSqlScriptUtils.abs(InternalSqlScriptUtils.dateTimeChrono" +
-                "(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
+                        "(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
                 "\"params\":{\"v0\":\"date\",\"v1\":\"Z\",\"v2\":\"YEAR\",\"v3\":\"LONG\"}},\"missing_bucket\":true," +
                 "\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}")
         );
@@ -1388,7 +1395,7 @@ public void testGroupByConvertScalar() {
                 ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                     .replaceAll("\\s+", ""),
                 endsWith("{\"source\":\"InternalSqlScriptUtils.cast(InternalSqlScriptUtils.abs(InternalSqlScriptUtils.dateTimeChrono" +
-                    "(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
+                            "(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
                     "\"params\":{\"v0\":\"date\",\"v1\":\"Z\",\"v2\":\"YEAR\",\"v3\":\"LONG\"}},\"missing_bucket\":true," +
                     "\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}")
             );
@@ -1404,7 +1411,7 @@ public void testGroupByConvertScalar() {
                 ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                     .replaceAll("\\s+", ""),
                 endsWith("{\"source\":\"InternalSqlScriptUtils.dateTimeChrono(" +
-                    "InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)\",\"lang\":\"painless\"," +
+                            "InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)\",\"lang\":\"painless\"," +
                     "\"params\":{\"v0\":\"date\",\"v1\":\"Z\",\"v2\":\"HOUR_OF_DAY\"}},\"missing_bucket\":true," +
                     "\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}")
             );
@@ -1423,7 +1430,7 @@ public void testGroupByConvertScalarWithAlias() {
                 ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                     .replaceAll("\\s+", ""),
                 endsWith("{\"source\":\"InternalSqlScriptUtils.cast(InternalSqlScriptUtils.abs(InternalSqlScriptUtils.dateTimeChrono" +
-                    "(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
+                            "(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
                     "\"params\":{\"v0\":\"date\",\"v1\":\"Z\",\"v2\":\"YEAR\",\"v3\":\"LONG\"}},\"missing_bucket\":true," +
                     "\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}")
             );
@@ -1438,7 +1445,7 @@ public void testGroupByConvertScalarWithAlias() {
                 ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                     .replaceAll("\\s+", ""),
                 endsWith("{\"source\":\"InternalSqlScriptUtils.dateTimeChrono(" +
-                    "InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)\",\"lang\":\"painless\"," +
+                            "InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)\",\"lang\":\"painless\"," +
                     "\"params\":{\"v0\":\"date\",\"v1\":\"Z\",\"v2\":\"MINUTE_OF_HOUR\"}}," +
                     "\"missing_bucket\":true,\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}")
             );
@@ -1456,7 +1463,7 @@ public void testGroupByConvertScalarWithNumericRef() {
             ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                 .replaceAll("\\s+", ""),
             endsWith("{\"source\":\"InternalSqlScriptUtils.cast(InternalSqlScriptUtils.abs(InternalSqlScriptUtils.dateTimeChrono" +
-                "(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
+                        "(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)),params.v3)\",\"lang\":\"painless\"," +
                 "\"params\":{\"v0\":\"date\",\"v1\":\"Z\",\"v2\":\"YEAR\",\"v3\":\"LONG\"}},\"missing_bucket\":true," +
                 "\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}")
         );
@@ -1472,7 +1479,8 @@ public void testGroupByConstantScalar() {
         assertThat(
             ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                 .replaceAll("\\s+", ""),
-            endsWith("{\"script\":{\"source\":\"InternalSqlScriptUtils.mul(params.v0,InternalSqlScriptUtils.docValue(doc,params.v1))\"," +
+                endsWith("{\"script\":{\"source\":\"InternalSqlScriptUtils.mul(params.v0,InternalQlScriptUtils.docValue(doc,params.v1))\","
+                        +
                 "\"lang\":\"painless\",\"params\":{\"v0\":3.141592653589793,\"v1\":\"int\"}},\"missing_bucket\":true," +
                 "\"value_type\":\"double\",\"order\":\"asc\"}}}]}}}")
         );
@@ -1489,7 +1497,8 @@ public void testGroupByConstantScalarWithAlias() {
             assertThat(
                 ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                     .replaceAll("\\s+", ""),
-                endsWith("{\"script\":{\"source\":\"InternalSqlScriptUtils.mul(params.v0,InternalSqlScriptUtils.docValue(doc,params.v1))" +
+                    endsWith("{\"script\":{\"source\":\"InternalSqlScriptUtils.mul(params.v0,InternalQlScriptUtils.docValue(doc,params.v1))"
+                            +
                     "\",\"lang\":\"painless\",\"params\":{\"v0\":3.141592653589793,\"v1\":\"int\"}},\"missing_bucket\":true," +
                     "\"value_type\":\"double\",\"order\":\"asc\"}}}]}}}")
             );
@@ -1505,7 +1514,7 @@ public void testGroupByConstantScalarWithAlias() {
             assertThat(
                 ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                     .replaceAll("\\s+", ""),
-                endsWith("{\"source\":\"InternalSqlScriptUtils.gt(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1)\"," +
+                    endsWith("{\"source\":\"InternalQlScriptUtils.gt(InternalQlScriptUtils.docValue(doc,params.v0),params.v1)\"," +
                     "\"lang\":\"painless\",\"params\":{\"v0\":\"int\",\"v1\":3}}," +
                     "\"missing_bucket\":true,\"value_type\":\"boolean\",\"order\":\"asc\"}}}]}}}")
             );
@@ -1523,7 +1532,8 @@ public void testGroupByConstantScalarWithNumericRef() {
             assertThat(
                 ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                     .replaceAll("\\s+", ""),
-                endsWith("{\"script\":{\"source\":\"InternalSqlScriptUtils.mul(params.v0,InternalSqlScriptUtils.docValue(doc,params.v1))" +
+                    endsWith("{\"script\":{\"source\":\"InternalSqlScriptUtils.mul(params.v0,InternalQlScriptUtils.docValue(doc,params.v1))"
+                            +
                     "\",\"lang\":\"painless\",\"params\":{\"v0\":3.141592653589793,\"v1\":\"int\"}},\"missing_bucket\":true," +
                     "\"value_type\":\"double\",\"order\":\"asc\"}}}]}}}")
             );
@@ -1537,7 +1547,7 @@ public void testGroupByConstantScalarWithNumericRef() {
             assertThat(
                 ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                     .replaceAll("\\s+", ""),
-                endsWith("{\"source\":\"InternalSqlScriptUtils.mul(params.v0,InternalSqlScriptUtils.docValue(doc,params.v1))\"," +
+                    endsWith("{\"source\":\"InternalSqlScriptUtils.mul(params.v0,InternalQlScriptUtils.docValue(doc,params.v1))\"," +
                     "\"lang\":\"painless\",\"params\":{\"v0\":3.141592653589793,\"v1\":\"int\"}}," +
                     "\"missing_bucket\":true,\"value_type\":\"double\",\"order\":\"asc\"}}}]}}}")
             );
@@ -1551,8 +1561,8 @@ public void testGroupByConstantScalarWithNumericRef() {
             assertThat(
                 ((EsQueryExec) p).queryContainer().aggs().asAggBuilder().toString()
                     .replaceAll("\\s+", ""),
-                endsWith("{\"source\":\"InternalSqlScriptUtils.add(InternalSqlScriptUtils.docValue(doc,params.v0)," +
-                    "InternalSqlScriptUtils.intervalDayTime(params.v1,params.v2))\"," +
+                    endsWith("{\"source\":\"InternalSqlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0)," +
+                            "InternalSqlScriptUtils.intervalDayTime(params.v1,params.v2))\"," +
                     "\"lang\":\"painless\",\"params\":{\"v0\":\"date\",\"v1\":\"PT24H\",\"v2\":\"INTERVAL_DAY\"}}," +
                     "\"missing_bucket\":true,\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}")
             );
@@ -1568,8 +1578,8 @@ public void testOrderByWithCastWithMissingRefs() {
         assertThat(
             ((EsQueryExec) p).queryContainer().toString()
                 .replaceAll("\\s+", ""),
-            endsWith("\"sort\":[{\"_script\":{\"script\":{\"source\":\"InternalSqlScriptUtils.nullSafeSortString(InternalSqlScriptUtils" +
-                ".cast(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1))\",\"lang\":\"painless\"," +
+            endsWith("\"sort\":[{\"_script\":{\"script\":{\"source\":\"InternalQlScriptUtils.nullSafeSortString(InternalSqlScriptUtils" +
+                        ".cast(InternalQlScriptUtils.docValue(doc,params.v0),params.v1))\",\"lang\":\"painless\"," +
                 "\"params\":{\"v0\":\"date\",\"v1\":\"TIME\"}},\"type\":\"string\",\"order\":\"asc\"}},{\"int\":{\"order\":\"asc\"," +
                 "\"missing\":\"_last\",\"unmapped_type\":\"integer\"}}]}")
         );
@@ -1711,9 +1721,9 @@ public void testZonedDateTimeInScripts() {
         assertEquals(EsQueryExec.class, p.getClass());
         EsQueryExec eqe = (EsQueryExec) p;
         assertThat(eqe.queryContainer().toString().replaceAll("\\s+", ""), containsString(
-                "\"script\":{\"script\":{\"source\":\"InternalSqlScriptUtils.nullSafeFilter("
-                + "InternalSqlScriptUtils.gt(InternalSqlScriptUtils.add(InternalSqlScriptUtils.docValue(doc,params.v0),"
-                + "InternalSqlScriptUtils.intervalYearMonth(params.v1,params.v2)),InternalSqlScriptUtils.asDateTime(params.v3)))\","
+                "\"script\":{\"script\":{\"source\":\"InternalQlScriptUtils.nullSafeFilter("
+                        + "InternalQlScriptUtils.gt(InternalSqlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),"
+                        + "InternalSqlScriptUtils.intervalYearMonth(params.v1,params.v2)),InternalSqlScriptUtils.asDateTime(params.v3)))\","
                 + "\"lang\":\"painless\","
                 + "\"params\":{\"v0\":\"date\",\"v1\":\"P1Y\",\"v2\":\"INTERVAL_YEAR\",\"v3\":\"2019-03-11T12:34:56.000Z\"}},"));
     }
@@ -1728,8 +1738,8 @@ public void testChronoFieldBasedDateTimeFunctionsWithMathIntervalAndGroupBy() {
         EsQueryExec eqe = (EsQueryExec) p;
         assertThat(eqe.queryContainer().toString().replaceAll("\\s+", ""), containsString(
                 "{\"terms\":{\"script\":{\"source\":\"InternalSqlScriptUtils.dateTimeChrono("
-                + "InternalSqlScriptUtils.add(InternalSqlScriptUtils.docValue(doc,params.v0),"
-                + "InternalSqlScriptUtils.intervalYearMonth(params.v1,params.v2)),params.v3,params.v4)\","
+                        + "InternalSqlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),"
+                        + "InternalSqlScriptUtils.intervalYearMonth(params.v1,params.v2)),params.v3,params.v4)\","
                 + "\"lang\":\"painless\",\"params\":{\"v0\":\"date\",\"v1\":\"P1Y\",\"v2\":\"INTERVAL_YEAR\","
                 + "\"v3\":\"Z\",\"v4\":\"" + randomFunction.chronoField().name() + "\"}},\"missing_bucket\":true,"
                 + "\"value_type\":\"long\",\"order\":\"asc\"}}}]}}}}"));
@@ -1747,8 +1757,8 @@ public void testDateTimeFunctionsWithMathIntervalAndGroupBy() {
         EsQueryExec eqe = (EsQueryExec) p;
         assertThat(eqe.queryContainer().toString().replaceAll("\\s+", ""), containsString(
                 "{\"terms\":{\"script\":{\"source\":\"InternalSqlScriptUtils." + scriptMethods[pos]
-                + "(InternalSqlScriptUtils.add(InternalSqlScriptUtils.docValue(doc,params.v0),"
-                + "InternalSqlScriptUtils.intervalYearMonth(params.v1,params.v2)),params.v3)\",\"lang\":\"painless\","
+                        + "(InternalSqlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),"
+                        + "InternalSqlScriptUtils.intervalYearMonth(params.v1,params.v2)),params.v3)\",\"lang\":\"painless\","
                 + "\"params\":{\"v0\":\"date\",\"v1\":\"P1Y\",\"v2\":\"INTERVAL_YEAR\",\"v3\":\"Z\"}},\"missing_bucket\":true,"));
     }
 
@@ -1770,7 +1780,7 @@ public void testHavingWithColumnImplicitGrouping() {
         assertTrue("Should be tracking hits", eqe.queryContainer().shouldTrackHits());
         assertEquals(1, eqe.output().size());
         assertThat(eqe.queryContainer().toString().replaceAll("\\s+", ""), containsString(
-                "\"script\":{\"source\":\"InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(params.a0,params.v0))\","
+                "\"script\":{\"source\":\"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(params.a0,params.v0))\","
                 + "\"lang\":\"painless\",\"params\":{\"v0\":0}}"));
     }
 }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
index abe07c04869b7..4207ebba0a464 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json
@@ -21,15 +21,13 @@
       ]
     },
     "params":{
-      "wait_for_completion":{
+      "wait_for_completion_timeout":{
         "type":"time",
-        "description":"Specify the time that the request should block waiting for the final response",
-        "default": "1s"
+        "description":"Specify the time that the request should block waiting for the final response"
       },
       "keep_alive": {
         "type": "time",
-        "description": "Specify the time interval in which the results (partial or final) for this search will be available",
-        "default": "5d"
+        "description": "Specify the time interval in which the results (partial or final) for this search will be available"
       },
       "typed_keys":{
         "type":"boolean",
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
index 468ece1dfd2d3..c7cddf2ca7c0e 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json
@@ -27,18 +27,20 @@
       ]
     },
     "params":{
-      "wait_for_completion":{
+      "wait_for_completion_timeout":{
         "type":"time",
         "description":"Specify the time that the request should block waiting for the final response",
         "default": "1s"
       },
-      "clean_on_completion":{
+      "keep_on_completion":{
         "type":"boolean",
-        "description":"Control whether the response should not be stored in the cluster if it completed within the provided [wait_for_completion] time (default: true)"
+        "description":"Control whether the response should be stored in the cluster if it completed within the provided [wait_for_completion] time (default: true)",
+        "default":false
       },
       "keep_alive": {
         "type": "time",
-        "description": "Update the time interval in which the results (partial or final) for this search will be available"
+        "description": "Update the time interval in which the results (partial or final) for this search will be available",
+        "default": "5d"
       },
       "batched_reduce_size":{
         "type":"number",
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/cat.transform.json
similarity index 98%
rename from x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json
rename to x-pack/plugin/src/test/resources/rest-api-spec/api/cat.transform.json
index 9cdbc620eb93f..98a2fcac1c281 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/transform.cat_transform.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/cat.transform.json
@@ -1,5 +1,5 @@
 {
-  "transform.cat_transform":{
+  "cat.transform":{
     "documentation":{
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-transforms.html"
     },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/string_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/string_stats.yml
new file mode 100644
index 0000000000000..daa34a6d021e6
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/string_stats.yml
@@ -0,0 +1,53 @@
+setup:
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              value:
+                type: keyword
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"value": "a"}'
+          - '{"index": {}}'
+          - '{"value": "b"}'
+
+---
+"basic":
+  - do:
+      search:
+        size: 0
+        body:
+          aggs:
+            my_agg:
+              string_stats:
+                field: value
+  - match: { aggregations.my_agg.count: 2 }
+  - match: { aggregations.my_agg.min_length: 1 }
+  - match: { aggregations.my_agg.max_length: 1 }
+  - match: { aggregations.my_agg.avg_length: 1.0 }
+  - match: { aggregations.my_agg.entropy: 1.0 }
+
+---
+"show_distribution":
+  - do:
+      search:
+        size: 0
+        body:
+          aggs:
+            my_agg:
+              string_stats:
+                field: value
+                show_distribution: true
+  - match: { aggregations.my_agg.count: 2 }
+  - match: { aggregations.my_agg.min_length: 1 }
+  - match: { aggregations.my_agg.max_length: 1 }
+  - match: { aggregations.my_agg.avg_length: 1.0 }
+  - match: { aggregations.my_agg.entropy: 1.0 }
+  - match: { aggregations.my_agg.distribution.a: 0.5 }
+  - match: { aggregations.my_agg.distribution.b: 0.5 }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
index c373217b06c99..54c076d480b02 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/async_search/10_basic.yml
@@ -43,7 +43,7 @@
       async_search.submit:
         index: test-*
         batched_reduce_size: 2
-        wait_for_completion: 10s
+        wait_for_completion_timeout: 10s
         body:
           query:
             match_all: {}
@@ -63,8 +63,8 @@
       async_search.submit:
         index: test-*
         batched_reduce_size: 2
-        wait_for_completion: 10s
-        clean_on_completion: false
+        wait_for_completion_timeout: 10s
+        keep_on_completion: true
         body:
           aggs:
             max:
@@ -82,8 +82,8 @@
       async_search.submit:
         index: test-*
         batched_reduce_size: 2
-        wait_for_completion: 10s
-        clean_on_completion: false
+        wait_for_completion_timeout: 10s
+        keep_on_completion: true
         typed_keys: true
         body:
           aggs:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
index a5a99b30391ea..5ec3c978b73d6 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
@@ -938,7 +938,7 @@ setup:
         id: "missing_config"
 
 ---
-"Test delete given config and state documents":
+"Test delete given config, state and stats documents":
   - skip:
       features: headers
   - do:
@@ -976,11 +976,33 @@ setup:
           {
           }
 
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      index:
+        index: .ml-stats-000001
+        id: "delete_foo_stats_1"
+        body: >
+          {
+            "job_id": "delete_foo"
+          }
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      index:
+        index: .ml-stats-000001
+        id: "delete_foo_stats_2"
+        body: >
+          {
+            "job_id": "delete_foo"
+          }
+
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       indices.refresh:
-        index: .ml-state*
+        index: .ml-stat*
 
   - do:
       search:
@@ -997,6 +1019,21 @@ setup:
 
   - match: { hits.total.value: 2 }
 
+  - do:
+      search:
+        index: .ml-stats-*
+        body:
+          size: 0
+          query:
+            bool:
+              filter:
+                ids:
+                  values:
+                    - "delete_foo_stats_1"
+                    - "delete_foo_stats_2"
+
+  - match: { hits.total.value: 2 }
+
   - do:
       ml.delete_data_frame_analytics:
         id: "delete_foo"
@@ -1016,6 +1053,21 @@ setup:
                     - "delete_foo_regression_state#1"
   - match: { hits.total.value: 0 }
 
+  - do:
+      search:
+        index: .ml-stats-*
+        body:
+          size: 0
+          query:
+            bool:
+              filter:
+                ids:
+                  values:
+                    - "delete_foo_stats_1"
+                    - "delete_foo_stats_2"
+
+  - match: { hits.total.value: 0 }
+
 ---
 "Test max model memory limit":
   - skip:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml
index 8253f12dda70e..f0407860b6a58 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/estimate_model_memory.yml
@@ -12,7 +12,7 @@
               "airline": 50000
             }
           }
-  - match: { model_memory_estimate: "3135mb" }
+  - match: { model_memory_estimate: "2094mb" }
 
 ---
 "Test by field also influencer":
@@ -32,7 +32,7 @@
               "airline": 500
             }
           }
-  - match: { model_memory_estimate: "3135mb" }
+  - match: { model_memory_estimate: "2094mb" }
 
 ---
 "Test by field with independent influencer":
@@ -52,7 +52,63 @@
               "country": 500
             }
           }
-  - match: { model_memory_estimate: "3140mb" }
+  - match: { model_memory_estimate: "2099mb" }
+
+---
+"Test over field":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "over_field_name": "airline"}]
+            },
+            "overall_cardinality": {
+              "airline": 50000
+            }
+          }
+  - match: { model_memory_estimate: "35mb" }
+
+---
+"Test over field also influencer":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "over_field_name": "airline"}],
+              "influencers": [ "airline" ]
+            },
+            "overall_cardinality": {
+              "airline": 50000
+            },
+            "max_bucket_cardinality": {
+              "airline": 500
+            }
+          }
+  - match: { model_memory_estimate: "35mb" }
+
+---
+"Test over field with independent influencer":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "over_field_name": "airline"}],
+              "influencers": [ "country" ]
+            },
+            "overall_cardinality": {
+              "airline": 50000
+            },
+            "max_bucket_cardinality": {
+              "country": 500
+            }
+          }
+  - match: { model_memory_estimate: "40mb" }
 
 ---
 "Test partition field":
@@ -125,7 +181,7 @@
               "country": 600
             }
           }
-  - match: { model_memory_estimate: "150010mb" }
+  - match: { model_memory_estimate: "100060mb" }
 
 ---
 "Test by and partition fields also influencers":
@@ -147,7 +203,7 @@
               "country": 40
             }
           }
-  - match: { model_memory_estimate: "150010mb" }
+  - match: { model_memory_estimate: "100060mb" }
 
 ---
 "Test by and partition fields with independent influencer":
@@ -168,5 +224,65 @@
               "src_ip": 500
             }
           }
-  - match: { model_memory_estimate: "150015mb" }
+  - match: { model_memory_estimate: "100065mb" }
+
+---
+"Test over and partition field":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "over_field_name": "airline", "partition_field_name": "country"}]
+            },
+            "overall_cardinality": {
+              "airline": 4000,
+              "country": 600
+            }
+          }
+  - match: { model_memory_estimate: "1220mb" }
+
+---
+"Test over and partition fields also influencers":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "over_field_name": "airline", "partition_field_name": "country"}],
+              "influencers": [ "airline", "country" ]
+            },
+            "overall_cardinality": {
+              "airline": 4000,
+              "country": 600
+            },
+            "max_bucket_cardinality": {
+              "airline": 60,
+              "country": 40
+            }
+          }
+  - match: { model_memory_estimate: "1220mb" }
+
+---
+"Test over and partition fields with independent influencer":
+  - do:
+      ml.estimate_model_memory:
+        body: >
+          {
+            "analysis_config": {
+              "bucket_span": "1h",
+              "detectors": [{"function": "max", "field_name": "responsetime", "over_field_name": "airline", "partition_field_name": "country"}],
+              "influencers": [ "src_ip" ]
+            },
+            "overall_cardinality": {
+              "airline": 4000,
+              "country": 600
+            },
+            "max_bucket_cardinality": {
+              "src_ip": 500
+            }
+          }
+  - match: { model_memory_estimate: "1225mb" }
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
index 7b365dbe92dc2..c566bfed6284c 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
@@ -142,7 +142,7 @@
   - do:
       ml.open_job:
         job_id: job-model-memory-limit-as-string
-  - match: { opened: false }
+  - match: { opened: true }
 
   - do:
       headers:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
index be7ace82c6887..b95f518158ed6 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
@@ -75,6 +75,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 
@@ -127,6 +129,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 
@@ -179,6 +183,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
index 4ca935d7bf84b..cb81fe483c278 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
@@ -76,6 +76,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 ---
@@ -217,6 +219,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
index 530bf6414bbed..56d73acb6acd1 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
@@ -76,6 +76,8 @@ setup:
             index_total: 0
             search_time_in_ms: 0
             search_total: 0
+            processing_time_in_ms: 0
+            processing_total: 0
           status:
             job_state: "stopped"
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_cat_apis.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_cat_apis.yml
index 37517b2c822f6..4b349f9c2e739 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_cat_apis.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_cat_apis.yml
@@ -51,7 +51,7 @@ teardown:
 ---
 "Test cat transform stats hiding headers":
   - do:
-      transform.cat_transform:
+      cat.transform:
         transform_id: "airline-transform-stats"
   - match:
       $body: |
@@ -61,7 +61,7 @@ teardown:
 ---
 "Test cat transform stats with column selection":
   - do:
-      transform.cat_transform:
+      cat.transform:
         transform_id: "airline-transform-stats"
         v: true
         h: id,version,source_index,dest_index,search_total,index_total,dt,cdtea,indexed_documents_exp_avg
@@ -90,7 +90,7 @@ teardown:
             "description": "description"
           }
   - do:
-      transform.cat_transform:
+      cat.transform:
         transform_id: "airline-transform-batch"
         v: true
   - match:
@@ -126,7 +126,7 @@ teardown:
             }
           }
   - do:
-      transform.cat_transform:
+      cat.transform:
         transform_id: "airline-transform-continuous"
         v: true
   - match:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
index c67a79c8218da..486082ef3ab59 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml
@@ -1,8 +1,8 @@
 setup:
   - skip:
       features: headers
-      version: " - 7.9.99"
-      reason: "wildcard fields were added from 8.0"
+      version: " - 7.6.99"
+      reason: "wildcard fields were added from 7.7"
 
   - do:
       indices.create:
@@ -10,10 +10,20 @@ setup:
         body:
           settings:
             number_of_replicas: 0
+            analysis:
+              normalizer:
+                lowercase:
+                  type: custom
+                  char_filter: []
+                  filter: ["lowercase"]
           mappings:
             properties:
               my_wildcard:
                  type: wildcard
+                 normalizer: lowercase
+                 fields:
+                   case_sensitive:
+                     type: wildcard
   - do:
       index:
         index: test-index
@@ -26,6 +36,12 @@ setup:
         id: 2
         body:
           my_wildcard: goodbye world
+  - do:
+      index:
+        index: test-index
+        id: 3
+        body:
+          my_wildcard: cAsE iNsEnSiTiVe World
 
   - do:
       indices.refresh: {}
@@ -80,6 +96,31 @@ setup:
               my_wildcard: {value: "*ello worl*" }
 
 
+  - match: {hits.total.value: 1}
+---
+"Case insensitive query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*Worl*" }
+
+
+  - match: {hits.total.value: 3}
+
+---
+"Case sensitive query":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard.case_sensitive: {value: "*Worl*" }
+
+
   - match: {hits.total.value: 1}
 
 ---
@@ -93,7 +134,7 @@ setup:
               my_wildcard: {value: "*ld" }
 
 
-  - match: {hits.total.value: 2}
+  - match: {hits.total.value: 3}
 
 ---
 "Long suffix query":
@@ -188,8 +229,8 @@ setup:
               terms: {field: "my_wildcard" }
 
 
-  - match: {hits.total.value: 2}
-  - length: { aggregations.top_vals.buckets: 2 }
+  - match: {hits.total.value: 3}
+  - length: { aggregations.top_vals.buckets: 3 }
 
 ---
 "Sort works":
@@ -199,10 +240,11 @@ setup:
           track_total_hits: true
           sort: [ { "my_wildcard": "desc" } ]
 
-  - match: { hits.total.value: 2 }
-  - length: { hits.hits: 2 }
+  - match: { hits.total.value: 3 }
+  - length: { hits.hits: 3 }
   - match: { hits.hits.0._id: "1" }
   - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "3" }
 
   - do:
       search:
@@ -210,9 +252,9 @@ setup:
           track_total_hits: true
           sort: [ { "my_wildcard": "asc" } ]
 
-  - match: { hits.total.value: 2 }
-  - length: { hits.hits: 2 }
-  - match: { hits.hits.0._id: "2" }
-  - match: { hits.hits.1._id: "1" }
-
+  - match: { hits.total.value: 3 }
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "1" }
 
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestSpecialCasesIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestSpecialCasesIT.java
index 5de3476df201b..d017f27ed2492 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestSpecialCasesIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestSpecialCasesIT.java
@@ -6,7 +6,11 @@
 
 package org.elasticsearch.xpack.transform.integration;
 
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.junit.Before;
 
@@ -14,6 +18,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.equalTo;
 
 public class TransformPivotRestSpecialCasesIT extends TransformRestTestCase {
@@ -102,4 +107,110 @@ public void testIndexTemplateMappingClash() throws Exception {
         Number actual = (Number) ((List<?>) XContentMapValues.extractValue("hits.hits._source.rating.avg", searchResult)).get(0);
         assertEquals(3.878048780, actual.doubleValue(), 0.000001);
     }
+
+    public void testSparseDataPercentiles() throws Exception {
+        String indexName = "cpu-utilization";
+        String transformIndex = "pivot-cpu";
+        String transformId = "pivot-cpu";
+
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            {
+                builder.startObject("mappings")
+                    .startObject("properties")
+                    .startObject("host")
+                    .field("type", "keyword")
+                    .endObject()
+                    .startObject("cpu")
+                    .field("type", "integer")
+                    .endObject()
+                    .endObject()
+                    .endObject();
+            }
+            builder.endObject();
+            final StringEntity entity = new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
+            Request req = new Request("PUT", indexName);
+            req.setEntity(entity);
+            client().performRequest(req);
+        }
+
+        final StringBuilder bulk = new StringBuilder();
+        bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
+        bulk.append("{\"host\":\"host-1\",\"cpu\": 22}\n");
+        bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
+        bulk.append("{\"host\":\"host-1\",\"cpu\": 55}\n");
+        bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
+        bulk.append("{\"host\":\"host-1\",\"cpu\": 23}\n");
+        bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
+        bulk.append("{\"host\":\"host-2\",\"cpu\": 0}\n");
+        bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
+        bulk.append("{\"host\":\"host-2\",\"cpu\": 99}\n");
+        bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
+        bulk.append("{\"host\":\"host-1\",\"cpu\": 28}\n");
+        bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
+        bulk.append("{\"host\":\"host-1\",\"cpu\": 77}\n");
+
+        // missing value for cpu
+        bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
+        bulk.append("{\"host\":\"host-3\"}\n");
+        bulk.append("\r\n");
+        final Request bulkRequest = new Request("POST", "/_bulk");
+        bulkRequest.addParameter("refresh", "true");
+        bulkRequest.setJsonEntity(bulk.toString());
+        client().performRequest(bulkRequest);
+
+        final Request createTransformRequest = new Request("PUT", getTransformEndpoint() + transformId);
+
+        String config = "{" + " \"source\": {\"index\":\"" + indexName + "\"}," + " \"dest\": {\"index\":\"" + transformIndex + "\"},";
+
+        config += " \"pivot\": {"
+            + "   \"group_by\": {"
+            + "     \"host\": {"
+            + "       \"terms\": {"
+            + "         \"field\": \"host\""
+            + " } } },"
+            + "   \"aggregations\": {"
+            + "     \"p\": {"
+            + "       \"percentiles\": {"
+            + "         \"field\": \"cpu\""
+            + " } }"
+            + " } }"
+            + "}";
+
+        createTransformRequest.setJsonEntity(config);
+        Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
+        assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+        startAndWaitForTransform(transformId, transformIndex);
+        assertTrue(indexExists(transformIndex));
+
+        Map<String, Object> indexStats = getAsMap(transformIndex + "/_stats");
+        assertEquals(3, XContentMapValues.extractValue("_all.total.docs.count", indexStats));
+
+        // get and check some data
+        Map<String, Object> searchResult = getAsMap(transformIndex + "/_search?q=host:host-1");
+
+        assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult));
+        @SuppressWarnings("unchecked")
+        Map<String, Object> percentiles = (Map<String, Object>) ((List<?>) XContentMapValues.extractValue(
+            "hits.hits._source.p",
+            searchResult
+        )).get(0);
+
+        assertEquals(28.0, (double) percentiles.get("50"), 0.000001);
+        assertEquals(77.0, (double) percentiles.get("99"), 0.000001);
+
+        searchResult = getAsMap(transformIndex + "/_search?q=host:host-3");
+        assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult));
+
+        @SuppressWarnings("unchecked")
+        Map<String, Object> percentilesEmpty = (Map<String, Object>) ((List<?>) XContentMapValues.extractValue(
+            "hits.hits._source.p",
+            searchResult
+        )).get(0);
+        assertTrue(percentilesEmpty.containsKey("50"));
+        assertNull(percentilesEmpty.get("50"));
+        assertTrue(percentilesEmpty.containsKey("99"));
+        assertNull(percentilesEmpty.get("99"));
+    }
 }
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java
index 765feca863262..d339e24fb74ad 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java
@@ -100,11 +100,6 @@ public void testUsage() throws Exception {
             assertEquals(2, XContentMapValues.extractValue("transform.transforms.stopped", statsMap));
             assertEquals(1, XContentMapValues.extractValue("transform.transforms.started", statsMap));
             for (String statName : PROVIDED_STATS) {
-                if (statName.equals(TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName())
-                    || statName.equals(TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName())) {
-                    continue;
-                }
-
                 // the trigger count can be higher if the scheduler kicked before usage has been called, therefore check for gte
                 if (statName.equals(TransformIndexerStats.NUM_INVOCATIONS.getPreferredName())) {
                     assertThat(
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
index f94237c9a5753..49bab468b9548 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
@@ -42,7 +42,6 @@
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
@@ -148,7 +147,6 @@ public class Transform extends Plugin implements SystemIndexPlugin, PersistentTa
      * These attributes should never be set directly, use the node setting counter parts instead.
      */
     public static final String TRANSFORM_ENABLED_NODE_ATTR = "transform.node";
-    public static final String TRANSFORM_REMOTE_ENABLED_NODE_ATTR = "transform.remote_connect";
 
     /**
      * Setting whether transform (the coordinator task) can run on this node and REST API's are available,
@@ -355,9 +353,8 @@ public List<Setting<?>> getSettings() {
     @Override
     public Settings additionalSettings() {
         String transformEnabledNodeAttribute = "node.attr." + TRANSFORM_ENABLED_NODE_ATTR;
-        String transformRemoteEnabledNodeAttribute = "node.attr." + TRANSFORM_REMOTE_ENABLED_NODE_ATTR;
 
-        if (settings.get(transformEnabledNodeAttribute) != null || settings.get(transformRemoteEnabledNodeAttribute) != null) {
+        if (settings.get(transformEnabledNodeAttribute) != null) {
             throw new IllegalArgumentException(
                 "Directly setting transform node attributes is not permitted, please use the documented node settings instead"
             );
@@ -370,7 +367,6 @@ public Settings additionalSettings() {
         Settings.Builder additionalSettings = Settings.builder();
 
         additionalSettings.put(transformEnabledNodeAttribute, TRANSFORM_ENABLED_NODE.get(settings));
-        additionalSettings.put(transformRemoteEnabledNodeAttribute, RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings));
 
         return additionalSettings.build();
     }
@@ -393,7 +389,7 @@ public List<Entry> getNamedXContent() {
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.singletonList(
             new SystemIndexDescriptor(TransformInternalIndexConstants.INDEX_NAME_PATTERN, "Contains Transform configuration data")
         );
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java
index 3dd12b74c93aa..847f37a5b9d4b 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java
@@ -51,8 +51,10 @@ public class TransformInfoTransportAction extends XPackInfoFeatureTransportActio
         TransformIndexerStats.NUM_INVOCATIONS.getPreferredName(),
         TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(),
         TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName(),
+        TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(),
         TransformIndexerStats.INDEX_TOTAL.getPreferredName(),
         TransformIndexerStats.SEARCH_TOTAL.getPreferredName(),
+        TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(),
         TransformIndexerStats.INDEX_FAILURES.getPreferredName(),
         TransformIndexerStats.SEARCH_FAILURES.getPreferredName(),
         TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
@@ -105,13 +107,15 @@ static TransformIndexerStats parseSearchAggs(SearchResponse searchResponse) {
             statisticsList.get(3).longValue(),  // numInvocations
             statisticsList.get(4).longValue(),  // indexTime
             statisticsList.get(5).longValue(),  // searchTime
-            statisticsList.get(6).longValue(),  // indexTotal
-            statisticsList.get(7).longValue(),  // searchTotal
-            statisticsList.get(8).longValue(),  // indexFailures
-            statisticsList.get(9).longValue(), // searchFailures
-            statisticsList.get(10), // exponential_avg_checkpoint_duration_ms
-            statisticsList.get(11), // exponential_avg_documents_indexed
-            statisticsList.get(12)  // exponential_avg_documents_processed
+            statisticsList.get(6).longValue(),  // processingTime
+            statisticsList.get(7).longValue(),  // indexTotal
+            statisticsList.get(8).longValue(),  // searchTotal
+            statisticsList.get(9).longValue(),  // processingTotal
+            statisticsList.get(10).longValue(),  // indexFailures
+            statisticsList.get(11).longValue(), // searchFailures
+            statisticsList.get(12), // exponential_avg_checkpoint_duration_ms
+            statisticsList.get(13), // exponential_avg_documents_indexed
+            statisticsList.get(14)  // exponential_avg_documents_processed
         );
     }
 
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java
index 46f18afa23c25..9fef913443312 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java
@@ -34,12 +34,12 @@
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.XPackField;
@@ -121,7 +121,7 @@ protected TransportPreviewTransformAction(
         this.sourceDestValidator = new SourceDestValidator(
             indexNameExpressionResolver,
             transportService.getRemoteClusterService(),
-            RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings)
+            Node.NODE_REMOTE_CLUSTER_CLIENT.get(settings)
                 ? new RemoteClusterLicenseChecker(client, XPackLicenseState::isTransformAllowedForOperationMode)
                 : null,
             clusterService.getNodeName(),
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java
index 23bd53e4efd30..8ef9804df762f 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java
@@ -31,11 +31,11 @@
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.XPackField;
@@ -138,7 +138,7 @@ protected TransportPutTransformAction(
         this.sourceDestValidator = new SourceDestValidator(
             indexNameExpressionResolver,
             transportService.getRemoteClusterService(),
-            RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings)
+            Node.NODE_REMOTE_CLUSTER_CLIENT.get(settings)
                 ? new RemoteClusterLicenseChecker(client, XPackLicenseState::isTransformAllowedForOperationMode)
                 : null,
             clusterService.getNodeName(),
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
index 9598e2f400e6f..deb88268eb8e5 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
@@ -31,12 +31,12 @@
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.XPackField;
@@ -137,7 +137,7 @@ protected TransportStartTransformAction(
         this.sourceDestValidator = new SourceDestValidator(
             indexNameExpressionResolver,
             transportService.getRemoteClusterService(),
-            RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings)
+            Node.NODE_REMOTE_CLUSTER_CLIENT.get(settings)
                 ? new RemoteClusterLicenseChecker(client, XPackLicenseState::isTransformAllowedForOperationMode)
                 : null,
             clusterService.getNodeName(),
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
index 1df334036cd9a..cf5f57e3f71d1 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
@@ -27,11 +27,11 @@
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.XPackField;
@@ -126,7 +126,7 @@ protected TransportUpdateTransformAction(
         this.sourceDestValidator = new SourceDestValidator(
             indexNameExpressionResolver,
             transportService.getRemoteClusterService(),
-            RemoteClusterService.ENABLE_REMOTE_CLUSTERS.get(settings)
+            Node.NODE_REMOTE_CLUSTER_CLIENT.get(settings)
                 ? new RemoteClusterLicenseChecker(client, XPackLicenseState::isTransformAllowedForOperationMode)
                 : null,
             clusterService.getNodeName(),
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
index 9874dc1f2586a..decd3e6282e63 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
@@ -58,6 +58,7 @@ public final class TransformInternalIndex {
      * version 3 (7.5): rename to .transform-internal-xxx
      * version 4 (7.6): state::should_stop_at_checkpoint
      *                  checkpoint::checkpoint
+     * version 5 (7.7): stats::processing_time_in_ms, stats::processing_total
      */
 
     // constants for mappings
@@ -238,11 +239,17 @@ private static XContentBuilder addTransformStoredDocMappings(XContentBuilder bui
                      .startObject(TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName())
                         .field(TYPE, LONG)
                     .endObject()
+                    .startObject(TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName())
+                        .field(TYPE, LONG)
+                     .endObject()
                      .startObject(TransformIndexerStats.INDEX_TOTAL.getPreferredName())
                         .field(TYPE, LONG)
                     .endObject()
                      .startObject(TransformIndexerStats.SEARCH_TOTAL.getPreferredName())
                         .field(TYPE, LONG)
+                    .endObject()
+                    .startObject(TransformIndexerStats.PROCESSING_TOTAL.getPreferredName())
+                        .field(TYPE, LONG)
                     .endObject()
                      .startObject(TransformIndexerStats.SEARCH_FAILURES.getPreferredName())
                         .field(TYPE, LONG)
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java
index 52bdd24d52b99..62501fccbae32 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java
@@ -146,7 +146,7 @@ public static boolean nodeCanRunThisTransform(DiscoveryNode node, TransformTaskP
         }
 
         // does the transform require a remote and remote is enabled?
-        if (params.requiresRemote() && Boolean.parseBoolean(nodeAttributes.get(Transform.TRANSFORM_REMOTE_ENABLED_NODE_ATTR)) == false) {
+        if (params.requiresRemote() && node.isRemoteClusterClient() == false) {
             if (explain != null) {
                 explain.put(node.getId(), "transform requires a remote connection but remote is disabled");
             }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java
index a0ae5518b53ce..b6cf82c4930c2 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java
@@ -205,11 +205,16 @@ static class PercentilesAggExtractor implements AggValueExtractor {
         @Override
         public Object value(Aggregation agg, Map<String, String> fieldTypeMap, String lookupFieldPrefix) {
             Percentiles aggregation = (Percentiles) agg;
-
             HashMap<String, Double> percentiles = new HashMap<>();
 
             for (Percentile p : aggregation) {
-                percentiles.put(OutputFieldNameConverter.fromDouble(p.getPercent()), p.getValue());
+                // in case of sparse data percentiles might not have data, in this case it returns NaN,
+                // we need to guard the output and set null in this case
+                if (Numbers.isValidDouble(p.getValue()) == false) {
+                    percentiles.put(OutputFieldNameConverter.fromDouble(p.getPercent()), null);
+                } else {
+                    percentiles.put(OutputFieldNameConverter.fromDouble(p.getPercent()), p.getValue());
+                }
             }
 
             return percentiles;
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java
index ceb30cf500792..7018c2d2ae2f1 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java
@@ -98,13 +98,15 @@ public void testParseSearchAggs() {
             4,  // numInvocations
             5,  // indexTime
             6,  // searchTime
-            7,  // indexTotal
-            8,  // searchTotal
-            9,  // indexFailures
-            10, // searchFailures
-            11.0,  // exponential_avg_checkpoint_duration_ms
-            12.0,  // exponential_avg_documents_indexed
-            13.0   // exponential_avg_documents_processed
+            7,  // processingTime
+            8,  // indexTotal
+            9,  // searchTotal
+            10, // processingTotal
+            11, // indexFailures
+            12, // searchFailures
+            13.0,  // exponential_avg_checkpoint_duration_ms
+            14.0,  // exponential_avg_documents_indexed
+            15.0   // exponential_avg_documents_processed
         );
 
         int currentStat = 1;
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformTests.java
index fdfd5a5a1c247..49dd17d8f9e59 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformTests.java
@@ -19,18 +19,12 @@ public void testNodeAttributes() {
         Settings.Builder builder = Settings.builder();
         boolean transformEnabled = randomBoolean();
         boolean transformPluginEnabled = randomBoolean();
-        boolean remoteEnabled = randomBoolean();
 
         // randomly use explicit or default setting
         if ((transformEnabled && randomBoolean()) == false) {
             builder.put("node.transform", transformEnabled);
         }
 
-        // randomly use explicit or default setting
-        if ((remoteEnabled && randomBoolean()) == false) {
-            builder.put("cluster.remote.connect", remoteEnabled);
-        }
-
         if (transformPluginEnabled == false) {
             builder.put("xpack.transform.enabled", transformPluginEnabled);
         }
@@ -42,23 +36,14 @@ public void testNodeAttributes() {
             transformPluginEnabled && transformEnabled,
             Boolean.parseBoolean(transform.additionalSettings().get("node.attr.transform.node"))
         );
-        assertEquals(
-            transformPluginEnabled && remoteEnabled,
-            Boolean.parseBoolean(transform.additionalSettings().get("node.attr.transform.remote_connect"))
-        );
     }
 
     public void testNodeAttributesDirectlyGiven() {
         Settings.Builder builder = Settings.builder();
-
-        if (randomBoolean()) {
-            builder.put("node.attr.transform.node", randomBoolean());
-        } else {
-            builder.put("node.attr.transform.remote_connect", randomBoolean());
-        }
+        builder.put("node.attr.transform.node", randomBoolean());
 
         Transform transform = createTransform(builder.build());
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> transform.additionalSettings());
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, transform::additionalSettings);
         assertThat(
             e.getMessage(),
             equalTo("Directly setting transform node attributes is not permitted, please use the documented node settings instead")
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java
index e6ce52fa5b9ca..219cc7c807f1b 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java
@@ -324,7 +324,8 @@ public void testDoProcessAggNullCheck() {
             0,
             0,
             ShardSearchFailure.EMPTY_ARRAY,
-            SearchResponse.Clusters.EMPTY
+            SearchResponse.Clusters.EMPTY,
+            null
         );
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         Function<SearchRequest, SearchResponse> searchFunction = searchRequest -> searchResponse;
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java
index 9bfadf85c3005..e4964862d3b24 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java
@@ -243,19 +243,14 @@ private DiscoveryNodes.Builder buildNodes(
         boolean dedicatedTransformNode,
         boolean pastDataNode,
         boolean transformRemoteNodes,
-        boolean transformLocanOnlyNodes,
+        boolean transformLocalOnlyNodes,
         boolean currentDataNode
     ) {
 
         Map<String, String> transformNodeAttributes = new HashMap<>();
         transformNodeAttributes.put(Transform.TRANSFORM_ENABLED_NODE_ATTR, "true");
-        transformNodeAttributes.put(Transform.TRANSFORM_REMOTE_ENABLED_NODE_ATTR, "true");
         Map<String, String> transformNodeAttributesDisabled = new HashMap<>();
         transformNodeAttributesDisabled.put(Transform.TRANSFORM_ENABLED_NODE_ATTR, "false");
-        transformNodeAttributesDisabled.put(Transform.TRANSFORM_REMOTE_ENABLED_NODE_ATTR, "true");
-        Map<String, String> transformNodeAttributesNoRemote = new HashMap<>();
-        transformNodeAttributesNoRemote.put(Transform.TRANSFORM_ENABLED_NODE_ATTR, "true");
-        transformNodeAttributesNoRemote.put(Transform.TRANSFORM_REMOTE_ENABLED_NODE_ATTR, "false");
 
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
 
@@ -265,7 +260,7 @@ private DiscoveryNodes.Builder buildNodes(
                     "dedicated-transform-node",
                     buildNewFakeTransportAddress(),
                     transformNodeAttributes,
-                    Collections.singleton(DiscoveryNodeRole.MASTER_ROLE),
+                    new HashSet<>(Arrays.asList(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)),
                     Version.CURRENT
                 )
             );
@@ -277,7 +272,9 @@ private DiscoveryNodes.Builder buildNodes(
                     "past-data-node-1",
                     buildNewFakeTransportAddress(),
                     transformNodeAttributes,
-                    new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE)),
+                    new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE,
+                        DiscoveryNodeRole.MASTER_ROLE,
+                        DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)),
                     Version.V_7_7_0
                 )
             );
@@ -289,7 +286,7 @@ private DiscoveryNodes.Builder buildNodes(
                     "current-data-node-with-2-tasks",
                     buildNewFakeTransportAddress(),
                     transformNodeAttributes,
-                    new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE)),
+                    new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)),
                     Version.CURRENT
                 )
             )
@@ -298,18 +295,18 @@ private DiscoveryNodes.Builder buildNodes(
                         "current-data-node-with-1-tasks",
                         buildNewFakeTransportAddress(),
                         transformNodeAttributes,
-                        new HashSet<>(Arrays.asList(DiscoveryNodeRole.MASTER_ROLE)),
+                        new HashSet<>(Arrays.asList(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)),
                         Version.CURRENT
                     )
                 );
         }
 
-        if (transformLocanOnlyNodes) {
+        if (transformLocalOnlyNodes) {
             nodes.add(
                 new DiscoveryNode(
                     "current-data-node-with-0-tasks-transform-remote-disabled",
                     buildNewFakeTransportAddress(),
-                    transformNodeAttributesNoRemote,
+                    transformNodeAttributes,
                     new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE)),
                     Version.CURRENT
                 )
@@ -322,7 +319,7 @@ private DiscoveryNodes.Builder buildNodes(
                     "current-data-node-with-transform-disabled",
                     buildNewFakeTransportAddress(),
                     transformNodeAttributesDisabled,
-                    Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE),
+                    Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE),
                     Version.CURRENT
                 )
             );
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java
index 485384378e9b8..0040d5b7c7119 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java
@@ -798,6 +798,11 @@ public void testPercentilesAggExtractor() {
         );
     }
 
+    public void testPercentilesAggExtractorNaN() {
+        Aggregation agg = createPercentilesAgg("p_agg", Arrays.asList(new Percentile(1, Double.NaN), new Percentile(50, Double.NaN)));
+        assertThat(AggregationResultUtils.getExtractor(agg).value(agg, Collections.emptyMap(), ""), equalTo(asMap("1", null, "50", null)));
+    }
+
     public static SingleBucketAggregation createSingleBucketAgg(String name, long docCount, Aggregation... subAggregations) {
         SingleBucketAggregation agg = mock(SingleBucketAggregation.class);
         when(agg.getDocCount()).thenReturn(docCount);
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java
index 4f361c5e9d0d9..c09bbf5342ec1 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java
@@ -186,6 +186,7 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
                     0,
                     0,
                     searchFailures.toArray(new ShardSearchFailure[searchFailures.size()]),
+                    null,
                     null
                 );
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
index c22be9400da15..b9c44c22fd7a1 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
@@ -690,7 +690,7 @@ public void reload(Settings settings) {
     }
 
     @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return List.of(
             new SystemIndexDescriptor(Watch.INDEX, "Contains Watch definitions"),
             new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, "Used to track current and queued Watch execution")
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java
index 75e5bc1073e69..0556b8535e428 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java
@@ -64,7 +64,7 @@ public void testWatcherMetadataParsingDoesNotSwallowOtherMetaData() throws Excep
         builder = metaDataBuilder.build().toXContent(builder, params);
         builder.endObject();
         // deserialize metadata again
-        MetaData metaData = MetaData.Builder.fromXContent(createParser(builder), randomBoolean());
+        MetaData metaData = MetaData.Builder.fromXContent(createParser(builder));
         // check that custom metadata still present
         assertThat(metaData.custom(watcherMetaData.getWriteableName()), notNullValue());
         assertThat(metaData.custom(repositoriesMetaData.getWriteableName()), notNullValue());
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java
index 375cfa2ef7aef..192f2a777e8bb 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java
@@ -166,7 +166,7 @@ void stopExecutor() {
         // empty scroll response, no further scrolling needed
         SearchResponseSections scrollSearchSections = new SearchResponseSections(SearchHits.empty(), null, null, false, false, null, 1);
         SearchResponse scrollSearchResponse = new SearchResponse(scrollSearchSections, "scrollId", 1, 1, 0, 10,
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
         doAnswer(invocation -> {
             ActionListener<SearchResponse> listener = (ActionListener<SearchResponse>) invocation.getArguments()[2];
             listener.onResponse(scrollSearchResponse);
@@ -198,7 +198,7 @@ void stopExecutor() {
         SearchHits searchHits = new SearchHits(hits, new TotalHits(count, TotalHits.Relation.EQUAL_TO), 1.0f);
         SearchResponseSections sections = new SearchResponseSections(searchHits, null, null, false, false, null, 1);
         SearchResponse searchResponse = new SearchResponse(sections, "scrollId", 1, 1, 0, 10, ShardSearchFailure.EMPTY_ARRAY,
-                SearchResponse.Clusters.EMPTY);
+                SearchResponse.Clusters.EMPTY, null);
         doAnswer(invocation -> {
             ActionListener<SearchResponse> listener = (ActionListener<SearchResponse>) invocation.getArguments()[2];
             listener.onResponse(searchResponse);
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java
index 2df11261e22fa..3581b80cea307 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java
@@ -83,7 +83,7 @@ public void testExecuteAccessHits() throws Exception {
                 new SearchHits(new SearchHit[]{hit}, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1f),
             null, null, null, false, false, 1);
         SearchResponse response = new SearchResponse(internalSearchResponse, "", 3, 3, 0,
-            500L, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+            500L, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
 
         WatchExecutionContext ctx = mockExecutionContext("_watch_name", new Payload.XContent(response, ToXContent.EMPTY_PARAMS));
         assertThat(condition.execute(ctx).met(), is(true));
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionTests.java
index 4a7b50c2b18de..310753a315d07 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionTests.java
@@ -94,7 +94,7 @@ public void init() throws IOException {
     public void testExecute() throws Exception {
         ScriptCondition condition = new ScriptCondition(mockScript("ctx.payload.hits.total.value > 1"), scriptService);
         SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, ShardSearchFailure.EMPTY_ARRAY,
-                SearchResponse.Clusters.EMPTY);
+                SearchResponse.Clusters.EMPTY, null);
         WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response, Settings.EMPTY_PARAMS));
         assertFalse(condition.execute(ctx).met());
     }
@@ -104,7 +104,7 @@ public void testExecuteMergedParams() throws Exception {
             "ctx.payload.hits.total.value > params.threshold", singletonMap("threshold", 1));
         ScriptCondition executable = new ScriptCondition(script, scriptService);
         SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, ShardSearchFailure.EMPTY_ARRAY,
-                SearchResponse.Clusters.EMPTY);
+                SearchResponse.Clusters.EMPTY, null);
         WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response, Settings.EMPTY_PARAMS));
         assertFalse(executable.execute(ctx).met());
     }
@@ -118,7 +118,7 @@ public void testParserValid() throws Exception {
         ExecutableCondition executable = ScriptCondition.parse(scriptService, "_watch", parser);
 
         SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, ShardSearchFailure.EMPTY_ARRAY,
-                SearchResponse.Clusters.EMPTY);
+                SearchResponse.Clusters.EMPTY, null);
         WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response, Settings.EMPTY_PARAMS));
 
         assertFalse(executable.execute(ctx).met());
@@ -183,7 +183,7 @@ public void testScriptConditionThrowException() throws Exception {
         ScriptCondition condition = new ScriptCondition(
                 mockScript("null.foo"), scriptService);
         SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, ShardSearchFailure.EMPTY_ARRAY,
-                SearchResponse.Clusters.EMPTY);
+                SearchResponse.Clusters.EMPTY, null);
         WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response, ToXContent.EMPTY_PARAMS));
         ScriptException exception = expectThrows(ScriptException.class, () -> condition.execute(ctx));
         assertThat(exception.getMessage(), containsString("Error evaluating null.foo"));
@@ -193,7 +193,7 @@ public void testScriptConditionAccessCtx() throws Exception {
         ScriptCondition condition = new ScriptCondition(
             mockScript("ctx.trigger.scheduled_time.toInstant().toEpochMill() < new Date().time"), scriptService);
         SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, ShardSearchFailure.EMPTY_ARRAY,
-                SearchResponse.Clusters.EMPTY);
+                SearchResponse.Clusters.EMPTY, null);
         WatchExecutionContext ctx = mockExecutionContext("_name", ZonedDateTime.now(ZoneOffset.UTC),
             new Payload.XContent(response, ToXContent.EMPTY_PARAMS));
         Thread.sleep(10);
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java
index 657b89949bba6..c33bd3050588a 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java
@@ -234,8 +234,8 @@ public void testFindTriggeredWatchesGoodCase() {
         hit.sourceRef(source);
         hits = new SearchHits(new SearchHit[]{hit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f);
         SearchResponse searchResponse2 = new SearchResponse(
-            new InternalSearchResponse(hits, null, null, null, false, null, 1), "_scrollId1", 1, 1, 0, 1, null, null);
-        SearchResponse searchResponse3 = new SearchResponse(InternalSearchResponse.empty(), "_scrollId2", 1, 1, 0, 1, null, null);
+            new InternalSearchResponse(hits, null, null, null, false, null, 1), "_scrollId1", 1, 1, 0, 1, null, null, null);
+        SearchResponse searchResponse3 = new SearchResponse(InternalSearchResponse.empty(), "_scrollId2", 1, 1, 0, 1, null, null, null);
 
         doAnswer(invocation -> {
             SearchScrollRequest request = (SearchScrollRequest) invocation.getArguments()[1];
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java
index e45b1a7b91ee1..e33d1213a46db 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java
@@ -90,7 +90,7 @@ public void testExecute() throws Exception {
         ArgumentCaptor<SearchRequest> requestCaptor = ArgumentCaptor.forClass(SearchRequest.class);
         PlainActionFuture<SearchResponse> searchFuture = PlainActionFuture.newFuture();
         SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), "", 1, 1, 0, 1234,
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
         searchFuture.onResponse(searchResponse);
         when(client.search(requestCaptor.capture())).thenReturn(searchFuture);
 
@@ -118,7 +118,7 @@ public void testDifferentSearchType() throws Exception {
         ArgumentCaptor<SearchRequest> requestCaptor = ArgumentCaptor.forClass(SearchRequest.class);
         PlainActionFuture<SearchResponse> searchFuture = PlainActionFuture.newFuture();
         SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), "", 1, 1, 0, 1234,
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
         searchFuture.onResponse(searchResponse);
         when(client.search(requestCaptor.capture())).thenReturn(searchFuture);
 
@@ -160,7 +160,7 @@ public void testThatEmptyRequestBodyWorks() throws Exception {
         ArgumentCaptor<SearchRequest> requestCaptor = ArgumentCaptor.forClass(SearchRequest.class);
         PlainActionFuture<SearchResponse> searchFuture = PlainActionFuture.newFuture();
         SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), "", 1, 1, 0, 1234,
-                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
+                ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY, null);
         searchFuture.onResponse(searchResponse);
         when(client.search(requestCaptor.capture())).thenReturn(searchFuture);
 
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
index e489d8a35bb9f..568e4f502dbe0 100644
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -53,6 +54,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -64,6 +66,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.index.mapper.TypeParsers.parseField;
 
@@ -100,6 +103,9 @@ public static class Defaults {
 
     public static class Builder extends FieldMapper.Builder<Builder, WildcardFieldMapper> {
         protected int ignoreAbove = Defaults.IGNORE_ABOVE;
+        private IndexAnalyzers indexAnalyzers;
+        private String normalizerName;
+        
 
         public Builder(String name) {
             super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@@ -164,10 +170,23 @@ protected void setupFieldType(BuilderContext context) {
         public WildcardFieldType fieldType() {
             return (WildcardFieldType) super.fieldType();
         }
+        
+        public Builder normalizer(IndexAnalyzers indexAnalyzers, String name) {
+            this.indexAnalyzers = indexAnalyzers;
+            this.normalizerName = name;
+            return builder;
+        }        
 
         @Override
         public WildcardFieldMapper build(BuilderContext context) {
-            setupFieldType(context);            
+            setupFieldType(context);   
+            if (normalizerName != null) {
+                NamedAnalyzer normalizer = indexAnalyzers.getNormalizer(normalizerName);
+                if (normalizer == null) {
+                    throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
+                }
+                fieldType().setNormalizer(normalizer);
+            }            
             return new WildcardFieldMapper(
                     name, fieldType, defaultFieldType, ignoreAbove, 
                     context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
@@ -188,6 +207,11 @@ public static class TypeParser implements Mapper.TypeParser {
                 if (propName.equals("ignore_above")) {
                     builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
                     iterator.remove();
+                } else if (propName.equals("normalizer")) {
+                    if (propNode != null) {
+                        builder.normalizer(parserContext.getIndexAnalyzers(), propNode.toString());
+                    }
+                    iterator.remove();
                 }
             }            
             
@@ -198,6 +222,8 @@ public static class TypeParser implements Mapper.TypeParser {
      public static final char TOKEN_START_OR_END_CHAR = 0;    
     
      public static final class WildcardFieldType extends MappedFieldType {
+         
+        private NamedAnalyzer normalizer = null;
 
         public WildcardFieldType() {            
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
@@ -206,6 +232,7 @@ public WildcardFieldType() {
 
         protected WildcardFieldType(WildcardFieldType ref) {
             super(ref);
+            this.normalizer = ref.normalizer;
         }
 
         public WildcardFieldType clone() {
@@ -213,7 +240,39 @@ public WildcardFieldType clone() {
             return result;
         }
         
-                
+
+        @Override
+        public boolean equals(Object o) {
+            if (super.equals(o) == false) {
+                return false;
+            }
+            WildcardFieldType other = (WildcardFieldType) o;
+            return Objects.equals(normalizer, other.normalizer);
+        }
+
+        @Override
+        public int hashCode() {
+            return 31 * super.hashCode() + Objects.hash(normalizer);
+        }       
+        
+        private NamedAnalyzer normalizer() {
+            return normalizer;
+        }
+
+        public void setNormalizer(NamedAnalyzer normalizer) {
+            checkIfFrozen();
+            this.normalizer = normalizer;
+        }                
+
+        @Override
+        public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts) {
+            super.checkCompatibility(otherFT, conflicts);
+            WildcardFieldType other = (WildcardFieldType) otherFT;
+            if (Objects.equals(normalizer, other.normalizer) == false) {
+                conflicts.add("mapper [" + name() + "] has different [normalizer]");
+            }
+        }
+        
         // Holds parsed information about the wildcard pattern
         static class PatternStructure {
             boolean openStart, openEnd, hasSymbols;            
@@ -327,6 +386,9 @@ public boolean equals(Object obj) {
 
         @Override
         public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QueryShardContext context) {
+            if (normalizer != null) {
+                wildcardPattern = StringFieldType.normalizeWildcardPattern(name(), wildcardPattern, normalizer);
+            }
             PatternStructure patternStructure = new PatternStructure(wildcardPattern);            
             ArrayList<String> tokens = new ArrayList<>();
 
@@ -467,7 +529,32 @@ public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fiel
                         CircuitBreakerService breakerService, MapperService mapperService) {
                     return new WildcardBytesBinaryDVIndexFieldData(indexSettings.getIndex(), fieldType.name());
                 }};
-        }        
+        }
+
+
+        String normalize(String value) throws IOException {
+            if (normalizer != null) {
+                try (TokenStream ts = normalizer.tokenStream(name(), value)) {
+                    final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+                    ts.reset();
+                    if (ts.incrementToken() == false) {
+                      throw new IllegalStateException("The normalization token stream is "
+                          + "expected to produce exactly 1 token, but got 0 for analyzer "
+                          + normalizer + " and input \"" + value + "\"");
+                    }
+                    final String newValue = termAtt.toString();
+                    if (ts.incrementToken()) {
+                      throw new IllegalStateException("The normalization token stream is "
+                          + "expected to produce exactly 1 token, but got 2+ for analyzer "
+                          + normalizer + " and input \"" + value + "\"");
+                    }
+                    ts.end();
+                    return newValue;
+                }
+            }
+            return value;
+        }       
+        
     }
      
     static class  WildcardBytesBinaryDVIndexFieldData extends BytesBinaryDVIndexFieldData{
@@ -521,6 +608,11 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
             builder.field("ignore_above", ignoreAbove);
         }
+        if (fieldType().normalizer() != null) {
+            builder.field("normalizer", fieldType().normalizer().name());
+        } else if (includeDefaults) {
+            builder.nullField("normalizer");
+        }        
     }
     
     @Override
@@ -544,10 +636,11 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
     // For internal use by Lucene only - used to define ngram index
     final MappedFieldType ngramFieldType;
     
-    void createFields(String value, Document parseDoc, List<IndexableField>fields) {
+    void createFields(String value, Document parseDoc, List<IndexableField>fields) throws IOException {
         if (value == null || value.length() > ignoreAbove) {
             return;
         }
+        value = fieldType().normalize(value);
         String ngramValue = TOKEN_START_OR_END_CHAR + value + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;
         Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType);
         fields.add(ngramField);
diff --git a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java
index 4694340047467..f5fb588de10cf 100644
--- a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java
+++ b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java
@@ -148,7 +148,7 @@ private void setupSpnegoAuthSchemeSupport(HttpAsyncClientBuilder httpClientBuild
      * returns {@link LoginContext}
      *
      * @return {@link LoginContext}
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException if the login triggers a checked exception
      */
     public synchronized LoginContext login() throws PrivilegedActionException {
         if (this.loginContext == null) {
@@ -181,9 +181,9 @@ public synchronized LoginContext login() throws PrivilegedActionException {
      *            Subject.doAs
      * @param acc the {@link AccessControlContext} to be tied to the specified
      *            subject and action see
-     *            {@link Subject#doAsPrivileged(Subject, PrivilegedExceptionAction, AccessControlContext)
+     *            {@link Subject#doAsPrivileged(Subject, PrivilegedExceptionAction, AccessControlContext)}
      * @return the value returned by the PrivilegedExceptionAction's run method
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException if the specified action's run method threw a checked exception
      */
     static <T> T doAsPrivilegedWrapper(final Subject subject, final PrivilegedExceptionAction<T> action, final AccessControlContext acc)
             throws PrivilegedActionException {
diff --git a/x-pack/qa/multi-cluster-search-security/build.gradle b/x-pack/qa/multi-cluster-search-security/build.gradle
index 715d8857bac1b..75f8af8f44b65 100644
--- a/x-pack/qa/multi-cluster-search-security/build.gradle
+++ b/x-pack/qa/multi-cluster-search-security/build.gradle
@@ -23,7 +23,7 @@ task 'remote-cluster'(type: RestIntegTestTask) {
 testClusters.'remote-cluster' {
   testDistribution = 'DEFAULT'
   numberOfNodes = 2
-  setting 'cluster.remote.connect', "false"
+  setting 'node.remote_cluster_client', "false"
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
@@ -53,7 +53,7 @@ testClusters.'mixed-cluster' {
     testClusters.'remote-cluster'.getAllTransportPortURI().collect { "\"$it\"" }.toString()
   }
   setting 'cluster.remote.connections_per_cluster', "1"
-  setting 'cluster.remote.connect', "true"
+  setting 'node.remote_cluster_client', "true"
 
   user username: "test_user", password: "x-pack-test-password"
 }
diff --git a/x-pack/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/qa/multi-cluster-tests-with-security/build.gradle
index b6a3088f250aa..7dc422447bd1a 100644
--- a/x-pack/qa/multi-cluster-tests-with-security/build.gradle
+++ b/x-pack/qa/multi-cluster-tests-with-security/build.gradle
@@ -24,7 +24,7 @@ task 'remote-cluster'(type: RestIntegTestTask) {
 testClusters.'remote-cluster' {
   testDistribution = 'DEFAULT'
   numberOfNodes = 2
-  setting 'cluster.remote.connect', "false"
+  setting 'node.remote_cluster_client', "false"
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
@@ -52,7 +52,7 @@ testClusters.'mixed-cluster' {
     testClusters.'remote-cluster'.getAllTransportPortURI().collect { "\"$it\"" }.toString()
   }
   setting 'cluster.remote.connections_per_cluster', "1"
-  setting 'cluster.remote.connect', "true"
+  setting 'node.remote_cluster_client', "true"
 
   user username: "test_user", password: "x-pack-test-password"
 }
diff --git a/x-pack/qa/oidc-op-tests/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java b/x-pack/qa/oidc-op-tests/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java
index 4b0e3afc813ad..a082e39ec24d7 100644
--- a/x-pack/qa/oidc-op-tests/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java
+++ b/x-pack/qa/oidc-op-tests/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java
@@ -327,7 +327,7 @@ private Tuple<String, String> completeAuthentication(String redirectUri, String
         logger.info(" OpenIDConnect authentication response {}", responseBody);
         assertNotNull(responseBody.get("access_token"));
         assertNotNull(responseBody.get("refresh_token"));
-        return new Tuple(responseBody.get("access_token"), responseBody.get("refresh_token"));
+        return Tuple.tuple(responseBody.get("access_token").toString(), responseBody.get("refresh_token").toString());
     }
 
     private Request buildRequest(String method, String endpoint, Map<String, ?> body, Header... headers) throws IOException {
diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle
index 58ab5515b14cb..969fdba727ea8 100644
--- a/x-pack/qa/reindex-tests-with-security/build.gradle
+++ b/x-pack/qa/reindex-tests-with-security/build.gradle
@@ -39,10 +39,8 @@ testClusters.integTest {
   setting 'reindex.ssl.truststore.password', 'password'
 
   // Workaround for JDK-8212885
-  rootProject.globalInfo.ready {
-    if (BuildParams.runtimeJavaVersion.isJava12Compatible() == false) {
-      setting 'reindex.ssl.supported_protocols', 'TLSv1.2'
-    }
+  if (BuildParams.runtimeJavaVersion.isJava12Compatible() == false) {
+    setting 'reindex.ssl.supported_protocols', 'TLSv1.2'
   }
 
   extraConfigFile 'roles.yml', file('roles.yml')
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml
index d220c0b1a51ab..6056a243445cd 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml
@@ -279,9 +279,9 @@ setup:
 
   - do:
       indices.get_mapping:
-        index: .transform-internal-004
-  - match: { \.transform-internal-004.mappings.dynamic: "false" }
-  - match: { \.transform-internal-004.mappings.properties.id.type: "keyword" }
+        index: .transform-internal-005
+  - match: { \.transform-internal-005.mappings.dynamic: "false" }
+  - match: { \.transform-internal-005.mappings.properties.id.type: "keyword" }
   - do:
       indices.get_mapping:
         index: .transform-notifications-000002
diff --git a/x-pack/qa/saml-idp-tests/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java b/x-pack/qa/saml-idp-tests/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java
index 12b4c1849ad23..4abc3f7a70519 100644
--- a/x-pack/qa/saml-idp-tests/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java
+++ b/x-pack/qa/saml-idp-tests/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java
@@ -297,7 +297,7 @@ private Tuple<String, String> loginViaSaml(String acs) throws Exception {
     }
 
     /**
-     * Verifies that the provided "Access Token" (see {@link org.elasticsearch.xpack.security.authc.TokenService})
+     * Verifies that the provided "Access Token" (see org.elasticsearch.xpack.security.authc.TokenService)
      * is for the expected user with the expected name and roles if the user was created from Role-Mapping
      */
     private void verifyElasticsearchAccessTokenForRoleMapping(String accessToken) throws IOException {
@@ -315,7 +315,7 @@ private void verifyElasticsearchAccessTokenForRoleMapping(String accessToken) th
     }
 
     /**
-     * Verifies that the provided "Access Token" (see {@link org.elasticsearch.xpack.security.authc.TokenService})
+     * Verifies that the provided "Access Token" (see org.elasticsearch.xpack.security.authc.TokenService)
      * is for the expected user with the expected name and roles if the user was retrieved from the native realm
      */
     private void verifyElasticsearchAccessTokenForAuthorizingRealms(String accessToken) throws IOException {
@@ -602,7 +602,7 @@ private Response samlAuthenticate(HttpExchange http) throws IOException {
         assertThat(id, notNullValue());
         assertThat(realmName, notNullValue());
 
-        final MapBuilder<String, Object> bodyBuilder = new MapBuilder()
+        final MapBuilder<String, Object> bodyBuilder = new MapBuilder<String, Object>()
             .put("content", saml)
             .put("ids", Collections.singletonList(id));
         if (randomBoolean()) {
@@ -643,7 +643,7 @@ private Tuple<String, String> getCookie(HttpExchange http) throws IOException {
             final List<Cookie> parsed = new DefaultCookieSpec().parse(header, origin);
             return parsed.stream().filter(c -> SAML_REQUEST_COOKIE.equals(c.getName())).map(c -> {
                 String[] values = c.getValue().split("&");
-                return new Tuple(values[0], values[1]);
+                return Tuple.tuple(values[0], values[1]);
             }).findFirst().orElse(null);
         } catch (MalformedCookieException e) {
             throw new IOException("Cannot read cookies", e);
diff --git a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
index 8c5a3078e84f0..c89e889e813b9 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
+++ b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
@@ -257,6 +257,7 @@ public void testSettingsFilter() throws IOException {
         assertThat(settings, not(hasKey("ssl")));
     }
 
+    @SuppressWarnings("unchecked")
     private String randomNodeHttpAddress() throws IOException {
         Response response = client().performRequest(new Request("GET", "/_nodes"));
         assertOK(response);

From 5430ab2d7794e1625285d51d896b113699ad7c01 Mon Sep 17 00:00:00 2001
From: jimczi <jim.ferenczi@elastic.co>
Date: Thu, 26 Mar 2020 13:30:48 +0100
Subject: [PATCH 07/47] fix checkstyle after backport

---
 .../src/main/java/org/elasticsearch/search/SearchService.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index dff940b9a8c90..7218c5af03dff 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedSupplier;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -680,7 +679,6 @@ final SearchContext createContext(ReaderContext reader,
                                       boolean includeAggregations) throws IOException {
         final DefaultSearchContext context = createSearchContext(reader, request, defaultSearchTimeout);
         try {
-            context.setTask(task);
             if (request.scroll() != null) {
                 context.scrollContext().scroll = request.scroll();
             }

From 3b0760b2852b469d6a267977718c983724a9727a Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sun, 12 Apr 2020 10:39:41 -0400
Subject: [PATCH 08/47] Adjust hlrc tests after merge

---
 .../java/org/elasticsearch/client/RestHighLevelClientTests.java  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index bc35b206f9149..d04d4025b02e6 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -861,7 +861,6 @@ public void testApiNamingConventions() throws Exception {
             "indices.create_data_stream",
             "indices.get_data_streams",
             "indices.delete_data_stream",
-            "indices.delete_data_stream",
             "clear_reader"
         };
         //These API are not required for high-level client feature completeness

From ff4689b5c0f931347762bb136d780f6f789f528c Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 13 Apr 2020 11:16:43 -0400
Subject: [PATCH 09/47] enable bwc

---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 3757e708c6fc8..95b399d7ce4ef 100644
--- a/build.gradle
+++ b/build.gradle
@@ -183,8 +183,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/53989" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

From defa2e77567bddbcc89d05147ef2c8883e474e2a Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Tue, 14 Apr 2020 12:12:07 -0400
Subject: [PATCH 10/47] Restore missing rewrite when create search context

---
 .../main/java/org/elasticsearch/search/SearchService.java    | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index b0c7964d3d3e1..fa71f294fda1c 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -732,6 +732,11 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
                 indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE);
             searchContext = new DefaultSearchContext(reader, request, shardTarget, clusterService, indexService, indexShard,
                 bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase, lowLevelCancellation);
+            // we clone the query shard context here just for rewriting otherwise we
+            // might end up with incorrect state since we are using now() or script services
+            // during rewrite and normalized / evaluate templates etc.
+            QueryShardContext context = new QueryShardContext(searchContext.getQueryShardContext());
+            Rewriteable.rewrite(request.getRewriteable(), context, true);
             assert searchContext.getQueryShardContext().isCacheable();
             success = true;
         } finally {

From aa14abee1c1c28c084664a64fd193bff27af334b Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Tue, 21 Apr 2020 16:03:50 -0400
Subject: [PATCH 11/47] Add open reader contexts API (#55265)

This commit adds a new API that allows opening reader contexts
in a separate step before using them in subsequent search requests
---
 .../client/RestHighLevelClientTests.java      |   3 +-
 .../search/request/reader_context.asciidoc    |  92 +++++++++
 .../rest-api-spec/api/clear_reader.json       |   6 +-
 .../rest-api-spec/api/open_reader.json        |  61 ++++++
 .../test/search/330_reader_context.yml        |   9 +-
 .../elasticsearch/action/ActionModule.java    |   4 +
 .../search/AbstractSearchAsyncAction.java     |   8 +-
 .../action/search/OpenReaderRequest.java      | 121 ++++++++++++
 .../action/search/OpenReaderResponse.java     |  61 ++++++
 .../action/search/SearchRequest.java          |  13 +-
 .../action/search/SearchTransportService.java |  18 ++
 .../search/TransportOpenReaderAction.java     | 186 ++++++++++++++++++
 .../action/search/TransportSearchAction.java  |  60 ++++--
 .../action/search/RestOpenReaderAction.java   |  59 ++++++
 .../elasticsearch/search/SearchService.java   |  58 +++++-
 .../search/builder/SearchSourceBuilder.java   |  26 ++-
 .../search/internal/ShardSearchRequest.java   |   2 +
 .../action/search/MockSearchPhaseContext.java |   3 +-
 .../action/search/SearchRequestTests.java     |  44 +++++
 .../search/SearchServiceTests.java            |  10 +
 .../search/query/QuerySearchResultTests.java  |   5 +-
 .../SearchWithReaderContextIT.java            |  40 ++--
 22 files changed, 820 insertions(+), 69 deletions(-)
 create mode 100644 docs/reference/search/request/reader_context.asciidoc
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/open_reader.json
 create mode 100644 server/src/main/java/org/elasticsearch/action/search/OpenReaderRequest.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/search/OpenReaderResponse.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java
 create mode 100644 server/src/main/java/org/elasticsearch/rest/action/search/RestOpenReaderAction.java

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index d04d4025b02e6..525f83caa73c4 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -861,7 +861,8 @@ public void testApiNamingConventions() throws Exception {
             "indices.create_data_stream",
             "indices.get_data_streams",
             "indices.delete_data_stream",
-            "clear_reader"
+            "clear_reader",
+            "open_reader"
         };
         //These API are not required for high-level client feature completeness
         String[] notRequiredApi = new String[] {
diff --git a/docs/reference/search/request/reader_context.asciidoc b/docs/reference/search/request/reader_context.asciidoc
new file mode 100644
index 0000000000000..6fb397fceceb2
--- /dev/null
+++ b/docs/reference/search/request/reader_context.asciidoc
@@ -0,0 +1,92 @@
+[[reader-context]]
+==== Reader Context
+
+By default, a search request executes against the latest point in time readers of the
+target indices. Sometimes it's preferred to execute multiple search requests using
+the same point in time readers. For example, the combined result of the initial and
+subsequent search_after requests is more consistent if they use the same point in time
+readers.
+
+Point in time readers must be opened in a separate step before being used in subsequent
+search requests. The keep_alive parameter tells Elasticsearch how long it should keep
+the ``reader_context'' alive, e.g. `?keep_alive=5m`.
+
+[source,console]
+--------------------------------------------------
+POST /twitter/_open_reader?keep_alive=1m
+--------------------------------------------------
+// TEST[setup:twitter]
+
+The result from the above request includes a `reader_id`, which should
+be passed to the `id` of the `reader` parameter of a search request.
+
+POST /_search <1>
+{
+    "size": 100,
+    "query": {
+        "match" : {
+            "title" : "elasticsearch"
+        }
+    },
+    "reader": {
+	    "id":  "46ToAwEHdHdpdHRlchZSa1dRZDFELVNLbTZkQVJudUtGMFFnAAAWNVAwS09JWTdTRUdibWE1ZC1id0tjQRRURnhPWjNFQmI3bmVOeVZReS1tRAAAAAAAAAAB", <2>
+	    "keep_alive": "1m"  <3>
+    }
+}
+
+<1> A search request with `reader.id` must not specify `index`, `routing`,
+and `preferences` as these parameters are copied from the `reader.id`.
+<2> The `reader.id` parameter tells Elasticsearch to execute the request using
+the point-in-time readers from this id.
+<3> The `reader.keep_alive` parameter tells Elasticsearch how long it should extend
+the time to live of these readers.
+
+IMPORTANT: The open_reader request and each subsequent search request can return
+different `reader_id`; thus always use the most recently received `reader_id`
+for the next search request.
+
+[[reader-keep-alive]]
+===== Keeping the reader context alive
+The `keep_alive` parameter (passed to a `open_reader` request and search
+request) extends the time to live of open reader contexts. The value
+(e.g. `1m`, see <<time-units>>) does not need to be long enough to
+process all data -- it just needs to be long enough for the next request.
+
+Normally, the background merge process optimizes the index by merging together
+smaller segments to create new, bigger segments. Once the smaller segments are
+no longer needed they are deleted. However, open reader contexts prevents the
+old segments from being deleted since they are still in use.
+
+TIP: Keeping older segments alive means that more disk space and file handles
+are needed. Ensure that you have configured your nodes to have ample free file
+handles. See <<file-descriptors>>.
+
+Additionally, if a segment contains deleted or updated documents then the reader
+context must keep track of whether each document in the segment was live at the
+time of the initial search request. Ensure that your nodes have sufficient heap
+space if you have many open reader contexts on an index that is subject to ongoing
+deletes or updates.
+
+You can check how many reader contexts are open with the
+<<cluster-nodes-stats,nodes stats API>>:
+
+[source,console]
+---------------------------------------
+GET /_nodes/stats/indices/search
+---------------------------------------
+
+===== Clear reader API
+
+Reader contexts are automatically closed when the `reader.keep_alive` has
+been elapsed. However keeping readers open has a cost, as discussed in the
+<<reader-keep-alive,previous section>> so readers should be explicitly
+closed as soon as they are no longer used in search requests.
+
+[source,console]
+---------------------------------------
+DELETE /_search/reader
+{
+    "id" : "46ToAwEHdHdpdHRlchZSa1dRZDFELVNLbTZkQVJudUtGMFFnAAAWNVAwS09JWTdTRUdibWE1ZC1id0tjQRRURnhPWjNFQmI3bmVOeVZReS1tRAAAAAAAAAAB"
+}
+---------------------------------------
+// TEST[catch:missing]
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json b/rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json
index 353d82a0ff9d5..0799d22a81e0d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json
@@ -1,8 +1,8 @@
 {
   "clear_reader":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-body.html#_clear_scroll_api",
-      "description":"Explicitly clears the point in time readers"
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/reader-context.html",
+      "description":"Explicitly close the point in time readers"
     },
     "stability":"beta",
     "url":{
@@ -17,7 +17,7 @@
     },
     "params":{},
     "body":{
-      "description": "A point in time reader to clear"
+      "description": "point in time readers to clear"
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/open_reader.json b/rest-api-spec/src/main/resources/rest-api-spec/api/open_reader.json
new file mode 100644
index 0000000000000..435e40e8d9281
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/open_reader.json
@@ -0,0 +1,61 @@
+{
+  "open_reader":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/reader-context.html",
+      "description":"Open and keep point-in-time readers that can be used in subsequent search requests"
+    },
+    "stability":"beta",
+    "url":{
+      "paths":[
+        {
+          "path":"/_open_reader",
+          "methods":[
+            "POST"
+          ]
+        },
+        {
+          "path":"/{index}/_open_reader",
+          "methods":[
+            "POST"
+          ],
+          "parts":{
+            "index":{
+              "type":"list",
+              "description":"A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "preference":{
+        "type":"string",
+        "description":"Specify the node or shard the operation should be performed on (default: random)"
+      },
+      "routing":{
+        "type":"string",
+        "description":"Specific routing value"
+      },
+      "ignore_unavailable":{
+        "type":"boolean",
+        "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+      },
+      "expand_wildcards":{
+        "type":"enum",
+        "options":[
+          "open",
+          "closed",
+          "hidden",
+          "none",
+          "all"
+        ],
+        "default":"open",
+        "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
+      },
+      "keep_alive": {
+        "type": "string",
+        "description": "Specific the time to live for the point-in-time readers"
+      }
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml
index a0834565df258..6da2ea1152ad9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml
@@ -29,10 +29,15 @@ setup:
   - skip:
       version: " - 7.9.99"
       reason: "reader context is introduced in 8.0"
+  - do:
+      open_reader:
+        index: test
+        keep_alive: 1m
+  - set: {reader_id: reader_id}
+
   - do:
       search:
         rest_total_hits_as_int: true
-        index: test
         body:
           size: 1
           query:
@@ -40,6 +45,7 @@ setup:
               foo: bar
           sort: [{ age: desc }, { id: desc }]
           reader:
+            id: "$reader_id"
             keep_alive: 1m
 
   - match: {hits.total: 3 }
@@ -47,7 +53,6 @@ setup:
   - match: {hits.hits.0._index: test }
   - match: {hits.hits.0._id: "172" }
   - match: {hits.hits.0.sort: [24, 172] }
-  - set: {reader_id: reader_id}
 
   - do:
       index:
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index e916ce7f99ed8..f710304295ef7 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -207,6 +207,7 @@
 import org.elasticsearch.action.search.TransportClearReaderAction;
 import org.elasticsearch.action.search.TransportClearScrollAction;
 import org.elasticsearch.action.search.TransportMultiSearchAction;
+import org.elasticsearch.action.search.TransportOpenReaderAction;
 import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.search.TransportSearchScrollAction;
 import org.elasticsearch.action.support.ActionFilters;
@@ -364,6 +365,7 @@
 import org.elasticsearch.rest.action.search.RestCountAction;
 import org.elasticsearch.rest.action.search.RestExplainAction;
 import org.elasticsearch.rest.action.search.RestMultiSearchAction;
+import org.elasticsearch.rest.action.search.RestOpenReaderAction;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.rest.action.search.RestSearchScrollAction;
 import org.elasticsearch.tasks.Task;
@@ -581,6 +583,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(MultiSearchAction.INSTANCE, TransportMultiSearchAction.class);
         actions.register(ExplainAction.INSTANCE, TransportExplainAction.class);
         actions.register(ClearScrollAction.INSTANCE, TransportClearScrollAction.class);
+        actions.register(TransportOpenReaderAction.INSTANCE, TransportOpenReaderAction.class);
         actions.register(ClearReaderAction.INSTANCE, TransportClearReaderAction.class);
         actions.register(RecoveryAction.INSTANCE, TransportRecoveryAction.class);
         actions.register(NodesReloadSecureSettingsAction.INSTANCE, TransportNodesReloadSecureSettingsAction.class);
@@ -734,6 +737,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestSearchAction());
         registerHandler.accept(new RestSearchScrollAction());
         registerHandler.accept(new RestClearScrollAction());
+        registerHandler.accept(new RestOpenReaderAction());
         registerHandler.accept(new RestClearReaderAction());
         registerHandler.accept(new RestMultiSearchAction(settings));
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index fef296de2c9c9..77deb13df71cb 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -533,17 +533,21 @@ protected final SearchResponse buildSearchResponse(InternalSearchResponse intern
                                                        ShardSearchFailure[] failures) {
         final Version minNodeVersion = clusterState.nodes().getMinNodeVersion();
         String scrollId = request.scroll() != null ? TransportSearchHelper.buildScrollId(queryResults, minNodeVersion) : null;
-        String readerId = request.reader() != null ? TransportSearchHelper.encodeReaderIds(queryResults, minNodeVersion) : null;
+        String readerId = includeReaderIdInResponse() ? TransportSearchHelper.encodeReaderIds(queryResults, minNodeVersion) : null;
         return new SearchResponse(internalSearchResponse, scrollId, getNumShards(), successfulOps.get(),
             skippedOps.get(), buildTookInMillis(), failures, clusters, readerId);
     }
 
+    boolean includeReaderIdInResponse() {
+        return request.reader() != null;
+    }
+
     @Override
     public void sendSearchResponse(InternalSearchResponse internalSearchResponse, AtomicArray<SearchPhaseResult> queryResults) {
         ShardSearchFailure[] failures = buildShardFailures();
         Boolean allowPartialResults = request.allowPartialSearchResults();
         assert allowPartialResults != null : "SearchRequest missing setting for allowPartialSearchResults";
-        if ((request.reader() == null || request.reader().getId() == null) && allowPartialResults == false && failures.length > 0) {
+        if (request.reader() == null && allowPartialResults == false && failures.length > 0) {
             raisePhaseFailure(new SearchPhaseExecutionException("", "Shard failures", null, failures));
         } else {
             listener.onResponse(buildSearchResponse(internalSearchResponse, queryResults, failures));
diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenReaderRequest.java b/server/src/main/java/org/elasticsearch/action/search/OpenReaderRequest.java
new file mode 100644
index 0000000000000..c911b3124af66
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/search/OpenReaderRequest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+public final class OpenReaderRequest extends ActionRequest implements IndicesRequest {
+    private final String[] indices;
+    private final IndicesOptions indicesOptions;
+    private final TimeValue keepAlive;
+
+    @Nullable
+    private final String routing;
+    @Nullable
+    private final String preference;
+
+    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed();
+
+    public OpenReaderRequest(String[] indices, IndicesOptions indicesOptions, TimeValue keepAlive, String routing, String preference) {
+        this.indices = Objects.requireNonNull(indices);
+        this.indicesOptions = Objects.requireNonNull(indicesOptions);
+        this.keepAlive = keepAlive;
+        this.routing = routing;
+        this.preference = preference;
+    }
+
+    public OpenReaderRequest(StreamInput in) throws IOException {
+        super(in);
+        this.indices = in.readStringArray();
+        this.indicesOptions = IndicesOptions.readIndicesOptions(in);
+        this.keepAlive = in.readTimeValue();
+        this.routing = in.readOptionalString();
+        this.preference = in.readOptionalString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringArray(indices);
+        indicesOptions.writeIndicesOptions(out);
+        out.writeTimeValue(keepAlive);
+        out.writeOptionalString(routing);
+        out.writeOptionalString(preference);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (indices.length == 0) {
+            validationException = addValidationError("[index] is not specified", validationException);
+        }
+        if (keepAlive == null) {
+            validationException = addValidationError("[keep_alive] is not specified", validationException);
+        }
+        return validationException;
+    }
+
+    @Override
+    public String[] indices() {
+        return indices;
+    }
+
+    @Override
+    public IndicesOptions indicesOptions() {
+        return indicesOptions;
+    }
+
+    TimeValue keepAlive() {
+        return keepAlive;
+    }
+
+    String routing() {
+        return routing;
+    }
+
+    String preference() {
+        return preference;
+    }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new SearchTask(id, type, action, null, parentTaskId, headers) {
+            @Override
+            public String getDescription() {
+                return "open reader: indices [" + String.join(",", indices) + "] keep_alive [" + keepAlive + "]";
+            }
+        };
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenReaderResponse.java b/server/src/main/java/org/elasticsearch/action/search/OpenReaderResponse.java
new file mode 100644
index 0000000000000..43114f7d14f19
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/search/OpenReaderResponse.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+public final class OpenReaderResponse extends ActionResponse implements ToXContentObject {
+    private static final ParseField READER_ID = new ParseField("reader_id");
+
+    private final String readerId;
+
+    public OpenReaderResponse(String readerId) {
+        this.readerId = readerId;
+    }
+
+    public OpenReaderResponse(StreamInput in) throws IOException {
+        super(in);
+        readerId = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(readerId);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(READER_ID.getPreferredName(), readerId);
+        builder.endObject();
+        return builder;
+    }
+
+    public String getReaderId() {
+        return readerId;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 906cd7c39280e..952a143fe83e9 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -277,15 +277,24 @@ public ActionRequestValidationException validate() {
                 validationException =
                     addValidationError("[request_cache] cannot be used in a scroll context", validationException);
             }
+            if (reader() != null) {
+                validationException = addValidationError("using [reader] is not allowed in a scroll context", validationException);
+            }
         }
         if (source != null) {
             if (source.aggregations() != null) {
                 validationException = source.aggregations().validate(validationException);
             }
         }
-        if (reader() != null && reader().getId() != null) {
+        if (reader() != null) {
             if (indices.length > 0) {
-                validationException = addValidationError("[indices] cannot be used with reader contexts", validationException);
+                validationException = addValidationError("[index] cannot be used with reader contexts", validationException);
+            }
+            if (routing() != null) {
+                validationException = addValidationError("[routing] cannot be used with reader contexts", validationException);
+            }
+            if (preference() != null) {
+                validationException = addValidationError("[preference] cannot be used with reader contexts", validationException);
             }
         }
         return validationException;
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 9503c63e7f57b..be697cc78f79b 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -78,6 +78,7 @@ public class SearchTransportService {
     public static final String FETCH_ID_SCROLL_ACTION_NAME = "indices:data/read/search[phase/fetch/id/scroll]";
     public static final String FETCH_ID_ACTION_NAME = "indices:data/read/search[phase/fetch/id]";
     public static final String QUERY_CAN_MATCH_NAME = "indices:data/read/search[can_match]";
+    public static final String SHARD_OPEN_READER_NAME = "indices:data/read/shard_open_reader";
 
     private final TransportService transportService;
     private final BiFunction<Transport.Connection, SearchActionListener, ActionListener> responseWrapper;
@@ -168,6 +169,13 @@ public void sendExecuteFetchScroll(Transport.Connection connection, final ShardF
         sendExecuteFetch(connection, FETCH_ID_SCROLL_ACTION_NAME, request, task, listener);
     }
 
+    void sendShardOpenReader(Transport.Connection connection, SearchTask task,
+                             TransportOpenReaderAction.ShardOpenReaderRequest request,
+                             ActionListener<TransportOpenReaderAction.ShardOpenReaderResponse> listener) {
+        transportService.sendChildRequest(connection, SHARD_OPEN_READER_NAME, request, task,
+            new ActionListenerResponseHandler<>(listener, TransportOpenReaderAction.ShardOpenReaderResponse::new));
+    }
+
     private void sendExecuteFetch(Transport.Connection connection, String action, final ShardFetchRequest request, SearchTask task,
                                   final SearchActionListener<FetchSearchResult> listener) {
         transportService.sendChildRequest(connection, action, request, task,
@@ -362,6 +370,16 @@ public static void registerRequestHandler(TransportService transportService, Sea
                 searchService.canMatch(request, new ChannelActionListener<>(channel, QUERY_CAN_MATCH_NAME, request));
             });
         TransportActionProxy.registerProxyAction(transportService, QUERY_CAN_MATCH_NAME, SearchService.CanMatchResponse::new);
+
+        transportService.registerRequestHandler(SHARD_OPEN_READER_NAME, ThreadPool.Names.SAME,
+            TransportOpenReaderAction.ShardOpenReaderRequest::new,
+            (request, channel, task) -> {
+                searchService.openReaderContext(request.searchShardTarget.getShardId(), request.keepAlive,
+                    ActionListener.map(new ChannelActionListener<>(channel, SHARD_OPEN_READER_NAME, request),
+                        contextId -> new TransportOpenReaderAction.ShardOpenReaderResponse(contextId, request.searchShardTarget)));
+            });
+        TransportActionProxy.registerProxyAction(
+            transportService, SHARD_OPEN_READER_NAME, TransportOpenReaderAction.ShardOpenReaderResponse::new);
     }
 
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java
new file mode 100644
index 0000000000000..b551202279a87
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.function.BiFunction;
+
+public class TransportOpenReaderAction extends HandledTransportAction<OpenReaderRequest, OpenReaderResponse> {
+    public static final String NAME = "indices:data/read/open_reader";
+    public static final ActionType<OpenReaderResponse> INSTANCE = new ActionType<>(NAME, OpenReaderResponse::new);
+
+    private final TransportSearchAction transportSearchAction;
+    private final SearchTransportService searchTransportService;
+
+    @Inject
+    public TransportOpenReaderAction(TransportService transportService, SearchTransportService searchTransportService,
+                                     ActionFilters actionFilters, TransportSearchAction transportSearchAction) {
+        super(NAME, transportService, actionFilters, OpenReaderRequest::new);
+        this.transportSearchAction = transportSearchAction;
+        this.searchTransportService = searchTransportService;
+    }
+
+    @Override
+    protected void doExecute(Task task, OpenReaderRequest openReaderRequest, ActionListener<OpenReaderResponse> listener) {
+        final TransportSearchAction.SearchAsyncActionProvider actionProvider = new TransportSearchAction.SearchAsyncActionProvider() {
+            @Override
+            public AbstractSearchAsyncAction<? extends SearchPhaseResult> asyncSearchAction(
+                SearchTask task, SearchRequest searchRequest, GroupShardsIterator<SearchShardIterator> shardIterators,
+                TransportSearchAction.SearchTimeProvider timeProvider, BiFunction<String, String, Transport.Connection> connectionLookup,
+                ClusterState clusterState, Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
+                Map<String, Set<String>> indexRoutings, ActionListener<SearchResponse> listener, boolean preFilter,
+                ThreadPool threadPool, SearchResponse.Clusters clusters) {
+                final Executor executor = threadPool.executor(ThreadPool.Names.SEARCH);
+                return new OpenReaderSearchPhase(openReaderRequest, logger, searchTransportService, connectionLookup,
+                    aliasFilter, concreteIndexBoosts, indexRoutings, executor, searchRequest, listener, shardIterators,
+                    timeProvider, clusterState, task, clusters);
+            }
+        };
+        final SearchRequest searchRequest = new SearchRequest()
+            .indices(openReaderRequest.indices())
+            .indicesOptions(openReaderRequest.indicesOptions())
+            .preference(openReaderRequest.preference())
+            .routing(openReaderRequest.routing())
+            .allowPartialSearchResults(false);
+        transportSearchAction.executeRequest(task, searchRequest, actionProvider,
+            ActionListener.map(listener, r -> new OpenReaderResponse(r.getReaderId())));
+    }
+
+    static final class ShardOpenReaderRequest extends TransportRequest implements IndicesRequest {
+        final SearchShardTarget searchShardTarget;
+        final TimeValue keepAlive;
+
+        ShardOpenReaderRequest(SearchShardTarget searchShardTarget, TimeValue keepAlive) {
+            this.searchShardTarget = searchShardTarget;
+            this.keepAlive = keepAlive;
+        }
+
+        ShardOpenReaderRequest(StreamInput in) throws IOException {
+            super(in);
+            searchShardTarget = new SearchShardTarget(in);
+            keepAlive = in.readTimeValue();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            searchShardTarget.writeTo(out);
+            out.writeTimeValue(keepAlive);
+        }
+
+        @Override
+        public String[] indices() {
+            return searchShardTarget.getOriginalIndices().indices();
+        }
+
+        @Override
+        public IndicesOptions indicesOptions() {
+            return searchShardTarget.getOriginalIndices().indicesOptions();
+        }
+    }
+
+    static final class ShardOpenReaderResponse extends SearchPhaseResult {
+        ShardOpenReaderResponse(SearchContextId contextId, SearchShardTarget searchShardTarget) {
+            this.contextId = contextId;
+            setSearchShardTarget(searchShardTarget);
+        }
+
+        ShardOpenReaderResponse(StreamInput in) throws IOException {
+            super(in);
+            contextId = new SearchContextId(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            contextId.writeTo(out);
+        }
+    }
+
+    static final class OpenReaderSearchPhase extends AbstractSearchAsyncAction<SearchPhaseResult> {
+        final OpenReaderRequest openReaderRequest;
+
+        OpenReaderSearchPhase(OpenReaderRequest openReaderRequest, Logger logger, SearchTransportService searchTransportService,
+                              BiFunction<String, String, Transport.Connection> nodeIdToConnection,
+                              Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
+                              Map<String, Set<String>> indexRoutings, Executor executor, SearchRequest request,
+                              ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
+                              TransportSearchAction.SearchTimeProvider timeProvider, ClusterState clusterState,
+                              SearchTask task, SearchResponse.Clusters clusters) {
+            super("open_reader", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings,
+                executor, request, listener, shardsIts, timeProvider, clusterState, task,
+                new ArraySearchPhaseResults<>(shardsIts.size()), shardsIts.size(), clusters);
+            this.openReaderRequest = openReaderRequest;
+        }
+
+        @Override
+        protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting shard,
+                                           SearchActionListener<SearchPhaseResult> listener) {
+            final Transport.Connection connection = getConnection(shardIt.getClusterAlias(), shard.currentNodeId());
+            final SearchShardTarget searchShardTarget = shardIt.newSearchShardTarget(shard.currentNodeId());
+            final ShardOpenReaderRequest shardRequest = new ShardOpenReaderRequest(searchShardTarget, openReaderRequest.keepAlive());
+            getSearchTransport().sendShardOpenReader(connection, getTask(), shardRequest, ActionListener.map(listener, r -> r));
+        }
+
+        @Override
+        protected SearchPhase getNextPhase(SearchPhaseResults<SearchPhaseResult> results, SearchPhaseContext context) {
+            return new SearchPhase(getName()) {
+                @Override
+                public void run() {
+                    final AtomicArray<SearchPhaseResult> atomicArray = results.getAtomicArray();
+                    sendSearchResponse(InternalSearchResponse.empty(), atomicArray);
+                }
+            };
+        }
+
+        @Override
+        boolean includeReaderIdInResponse() {
+            return true;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 20f249160a72f..87e0711bbdc54 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -200,6 +200,11 @@ long buildTookInMillis() {
 
     @Override
     protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<SearchResponse> listener) {
+        executeRequest(task, searchRequest, this::searchAsyncAction, listener);
+    }
+
+    void executeRequest(Task task, SearchRequest searchRequest,
+                        SearchAsyncActionProvider searchAsyncActionProvider, ActionListener<SearchResponse> listener) {
         final long relativeStartNanos = System.nanoTime();
         final SearchTimeProvider timeProvider =
             new SearchTimeProvider(searchRequest.getOrCreateAbsoluteStartMillis(), relativeStartNanos, System::nanoTime);
@@ -211,24 +216,25 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<
             }
             final Map<ShardId, ReaderIdForNode> readerContexts;
             final Map<String, OriginalIndices> remoteClusterIndices;
-            if (searchRequest.reader() != null && searchRequest.reader().getId() != null) {
+            if (searchRequest.reader() != null) {
                 readerContexts = TransportSearchHelper.decodeReaderIds(searchRequest.reader().getId());
                 remoteClusterIndices = indicesFromReaderContexts(readerContexts, searchRequest.indicesOptions());
             } else {
                 readerContexts = null;
-                remoteClusterIndices = remoteClusterService.groupIndices(searchRequest.indicesOptions(),
-                    searchRequest.indices());
+                remoteClusterIndices = remoteClusterService.groupIndices(searchRequest.indicesOptions(), searchRequest.indices());
             }
             OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
             final ClusterState clusterState = clusterService.state();
             if (remoteClusterIndices.isEmpty()) {
-                executeLocalSearch(task, timeProvider, searchRequest, localIndices, clusterState, listener, readerContexts);
+                executeLocalSearch(
+                    task, timeProvider, searchRequest, localIndices, clusterState, listener, readerContexts, searchAsyncActionProvider);
             } else {
                 if (shouldMinimizeRoundtrips(searchRequest)) {
                     ccsRemoteReduce(searchRequest, localIndices, remoteClusterIndices, timeProvider,
-                            searchService.aggReduceContextBuilder(searchRequest),
-                            remoteClusterService, threadPool, listener,
-                            (r, l) -> executeLocalSearch(task, timeProvider, r, localIndices, clusterState, l, readerContexts));
+                        searchService.aggReduceContextBuilder(searchRequest),
+                        remoteClusterService, threadPool, listener,
+                        (r, l) -> executeLocalSearch(
+                            task, timeProvider, r, localIndices, clusterState, l, readerContexts, searchAsyncActionProvider));
                 } else {
                     AtomicInteger skippedClusters = new AtomicInteger(0);
                     collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(),
@@ -242,9 +248,10 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<
                                 int localClusters = localIndices == null ? 0 : 1;
                                 int totalClusters = remoteClusterIndices.size() + localClusters;
                                 int successfulClusters = searchShardsResponses.size() + localClusters;
-                                executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices,
-                                    remoteShardIterators, clusterNodeLookup, clusterState, remoteAliasFilters, listener,
-                                    new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters.get()), readerContexts);
+                                executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices, remoteShardIterators,
+                                    clusterNodeLookup, clusterState, remoteAliasFilters, listener,
+                                    new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters.get()),
+                                    readerContexts, searchAsyncActionProvider);
                             },
                             listener::onFailure));
                 }
@@ -417,9 +424,10 @@ SearchResponse createFinalResponse() {
 
     private void executeLocalSearch(Task task, SearchTimeProvider timeProvider, SearchRequest searchRequest, OriginalIndices localIndices,
                                     ClusterState clusterState, ActionListener<SearchResponse> listener,
-                                    Map<ShardId, ReaderIdForNode> readerContexts) {
+                                    Map<ShardId, ReaderIdForNode> readerContexts, SearchAsyncActionProvider searchAsyncActionProvider) {
         executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, Collections.emptyList(),
-            (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener, SearchResponse.Clusters.EMPTY, readerContexts);
+            (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener, SearchResponse.Clusters.EMPTY,
+            readerContexts, searchAsyncActionProvider);
     }
 
     static BiFunction<String, String, DiscoveryNode> processRemoteShards(Map<String, ClusterSearchShardsResponse> searchShardsResponses,
@@ -483,7 +491,8 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
                                OriginalIndices localIndices, List<SearchShardIterator> remoteShardIterators,
                                BiFunction<String, String, DiscoveryNode> remoteConnections, ClusterState clusterState,
                                Map<String, AliasFilter> remoteAliasMap, ActionListener<SearchResponse> listener,
-                               SearchResponse.Clusters clusters, Map<ShardId, ReaderIdForNode> readerContexts) {
+                               SearchResponse.Clusters clusters, Map<ShardId, ReaderIdForNode> readerContexts,
+                               SearchAsyncActionProvider searchAsyncActionProvider) {
 
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
         if (searchRequest.allowPartialSearchResults() == null) {
@@ -494,20 +503,21 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
         // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
         // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
         // of just for the _search api
-        final TimeValue keepAlive = searchRequest.reader() != null ? searchRequest.reader().getKeepAlive() : null;
         final List<SearchShardIterator> localShardIterators;
         final Map<String, AliasFilter> aliasFilter;
         final Map<String, Set<String>> indexRoutings;
 
         boolean preFilterSearchShards;
         if (readerContexts != null) {
+            assert searchRequest.reader() != null;
             aliasFilter = new HashMap<>();
             for (ShardId shardId : readerContexts.keySet()) {
                 aliasFilter.put(shardId.getIndex().getUUID(), AliasFilter.EMPTY);
             }
             indexRoutings = Map.of();
             localShardIterators = searchShardsFromReaderContexts(
-                clusterState, localIndices, searchRequest.getLocalClusterAlias(), readerContexts, keepAlive);
+                clusterState, localIndices, searchRequest.getLocalClusterAlias(), readerContexts, searchRequest.reader().getKeepAlive());
+            // TODO: support can_match with reader contexts after https://github.com/elastic/elasticsearch/pull/54966
             preFilterSearchShards = false;
         } else {
             final Index[] indices = resolveLocalIndices(localIndices, searchRequest.indicesOptions(), clusterState, timeProvider);
@@ -523,7 +533,7 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
                 concreteIndices, routingMap, searchRequest.preference(), searchService.getResponseCollectorService(), nodeSearchCounts);
             localShardIterators = StreamSupport.stream(localShardRoutings.spliterator(), false)
                 .map(it -> new SearchShardIterator(
-                    searchRequest.getLocalClusterAlias(), it.shardId(), it.getShardRoutings(), localIndices, null, keepAlive))
+                    searchRequest.getLocalClusterAlias(), it.shardId(), it.getShardRoutings(), localIndices, null, null))
                 .collect(Collectors.toList());
             aliasFilter = buildPerIndexAliasFilter(searchRequest, clusterState, indices, remoteAliasMap);
             indexRoutings = routingMap;
@@ -558,9 +568,10 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
         final DiscoveryNodes nodes = clusterState.nodes();
         BiFunction<String, String, Transport.Connection> connectionLookup = buildConnectionLookup(searchRequest.getLocalClusterAlias(),
             nodes::get, remoteConnections, searchTransportService::getConnection);
-        searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState,
-            Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, indexRoutings, listener, preFilterSearchShards, clusters)
-            .start();
+        searchAsyncActionProvider.asyncSearchAction(
+            task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState,
+            Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, indexRoutings, listener,
+            preFilterSearchShards, threadPool, clusters).start();
     }
 
     static BiFunction<String, String, Transport.Connection> buildConnectionLookup(String requestClusterAlias,
@@ -619,6 +630,15 @@ static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(List<Search
         return GroupShardsIterator.sortAndCreate(shards);
     }
 
+    interface SearchAsyncActionProvider {
+        AbstractSearchAsyncAction<? extends SearchPhaseResult> asyncSearchAction(
+            SearchTask task, SearchRequest searchRequest, GroupShardsIterator<SearchShardIterator> shardIterators,
+            SearchTimeProvider timeProvider, BiFunction<String, String, Transport.Connection> connectionLookup,
+            ClusterState clusterState, Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
+            Map<String, Set<String>> indexRoutings, ActionListener<SearchResponse> listener, boolean preFilter,
+            ThreadPool threadPool, SearchResponse.Clusters clusters);
+    }
+
     private AbstractSearchAsyncAction<? extends SearchPhaseResult> searchAsyncAction(SearchTask task, SearchRequest searchRequest,
                                                         GroupShardsIterator<SearchShardIterator> shardIterators,
                                                         SearchTimeProvider timeProvider,
@@ -629,6 +649,7 @@ private AbstractSearchAsyncAction<? extends SearchPhaseResult> searchAsyncAction
                                                         Map<String, Set<String>> indexRoutings,
                                                         ActionListener<SearchResponse> listener,
                                                         boolean preFilter,
+                                                        ThreadPool threadPool,
                                                         SearchResponse.Clusters clusters) {
         Executor executor = threadPool.executor(ThreadPool.Names.SEARCH);
         if (preFilter) {
@@ -647,6 +668,7 @@ private AbstractSearchAsyncAction<? extends SearchPhaseResult> searchAsyncAction
                     indexRoutings,
                     listener,
                     false,
+                    threadPool,
                     clusters);
                 return new SearchPhase(action.getName()) {
                     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestOpenReaderAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestOpenReaderAction.java
new file mode 100644
index 0000000000000..7294347fc1e1e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestOpenReaderAction.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.search;
+
+import org.elasticsearch.action.search.OpenReaderRequest;
+import org.elasticsearch.action.search.TransportOpenReaderAction;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestOpenReaderAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "open_reader_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(POST, "/{index}/_open_reader"));
+    }
+
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
+        final IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, OpenReaderRequest.DEFAULT_INDICES_OPTIONS);
+        final String routing = request.param("routing");
+        final String preference = request.param("preference");
+        final TimeValue keepAlive = TimeValue.parseTimeValue(request.param("keep_alive"), null, "keep_alive");
+        final OpenReaderRequest openReaderRequest = new OpenReaderRequest(indices, indicesOptions, keepAlive, routing, preference);
+        return channel -> client.execute(TransportOpenReaderAction.INSTANCE, openReaderRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 7fe3284d17c38..5ef4d2130f98e 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -63,6 +63,7 @@
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.SearchOperationListener;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
@@ -482,8 +483,9 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
             try (SearchContext context = createContext(reader, reader.getShardSearchRequest(null), task, false);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
                 if (request.scroll() != null && request.scroll().keepAlive() != null) {
-                    checkKeepAliveLimit(request.scroll().keepAlive().millis());
-                    reader.keepAlive(request.scroll().keepAlive().millis());
+                    final long keepAlive = request.scroll().keepAlive().millis();
+                    checkKeepAliveLimit(keepAlive);
+                    reader.keepAlive(keepAlive);
                 }
                 reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
                 context.searcher().setAggregatedDfs(reader.getAggregatedDfs(null));
@@ -612,8 +614,10 @@ private ReaderContext findReaderContext(SearchContextId id) throws SearchContext
 
     final ReaderContext createOrGetReaderContext(ShardSearchRequest request, boolean keepStatesInContext) {
         if (request.readerId() != null) {
-            ReaderContext readerContext = findReaderContext(request.readerId());
-            final long keepAlive = getKeepAlive(request);
+            assert keepStatesInContext == false;
+            // NORELEASE: either wrap the searcher with the right security context or make sure the user owns it.
+            final ReaderContext readerContext = findReaderContext(request.readerId());
+            final long keepAlive = request.keepAlive().millis();
             checkKeepAliveLimit(keepAlive);
             readerContext.keepAlive(keepAlive);
             return readerContext;
@@ -626,6 +630,8 @@ final ReaderContext createOrGetReaderContext(ShardSearchRequest request, boolean
 
     final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexShard shard,
                                                   Engine.Searcher engineSearcher, boolean keepStatesInContext) {
+        assert request.readerId() == null;
+        assert request.keepAlive() == null;
         ReaderContext readerContext = null;
         Releasable decreaseScrollContexts = null;
         try {
@@ -638,17 +644,19 @@ final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexS
                             + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
                 }
             }
-            final long keepAlive = getKeepAlive(request);
-            checkKeepAliveLimit(keepAlive);
             if (keepStatesInContext || request.scroll() != null) {
+                long keepAlive = defaultKeepAlive;
+                if (request.scroll() != null && request.scroll().keepAlive() != null) {
+                    keepAlive = request.scroll().keepAlive().millis();
+                    checkKeepAliveLimit(keepAlive);
+                }
                 readerContext = new LegacyReaderContext(idGenerator.incrementAndGet(), shard, engineSearcher, request, keepAlive);
                 if (request.scroll() != null) {
                     readerContext.addOnClose(decreaseScrollContexts);
                     decreaseScrollContexts = null;
                 }
             } else {
-                readerContext = new ReaderContext(idGenerator.incrementAndGet(), shard, engineSearcher, keepAlive,
-                    request.keepAlive() == null);
+                readerContext = new ReaderContext(idGenerator.incrementAndGet(), shard, engineSearcher, defaultKeepAlive, true);
             }
             engineSearcher = null;
             final ReaderContext finalReaderContext = readerContext;
@@ -674,6 +682,36 @@ final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexS
         }
     }
 
+    /**
+     * Opens the reader context for given shardId. The newly opened reader context will be keep
+     * until the {@code keepAlive} elapsed unless it is manually released.
+     */
+    public void openReaderContext(ShardId shardId, TimeValue keepAlive, ActionListener<SearchContextId> listener) {
+        checkKeepAliveLimit(keepAlive.millis());
+        final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
+        final IndexShard shard = indexService.getShard(shardId.id());
+        final SearchOperationListener searchOperationListener = shard.getSearchOperationListener();
+        shard.awaitShardSearchActive(ignored ->
+            runAsync(shard, () -> {
+                Releasable releasable = null;
+                try {
+                    final Engine.Searcher engineSearcher = shard.acquireSearcher("search");
+                    releasable = engineSearcher;
+                    final ReaderContext readerContext = new ReaderContext(
+                        idGenerator.incrementAndGet(), shard, engineSearcher, keepAlive.millis(), false);
+                    releasable = readerContext;
+                    searchOperationListener.onNewReaderContext(readerContext);
+                    readerContext.addOnClose(() -> searchOperationListener.onFreeReaderContext(readerContext));
+                    putReaderContext(readerContext);
+                    releasable = null;
+                    return readerContext.id();
+                } finally {
+                    Releasables.close(releasable);
+                }
+            }, listener)
+        );
+    }
+
     final SearchContext createContext(ReaderContext reader,
                                       ShardSearchRequest request,
                                       SearchShardTask task,
@@ -784,8 +822,6 @@ public void freeAllScrollContexts() {
     private long getKeepAlive(ShardSearchRequest request) {
         if (request.scroll() != null && request.scroll().keepAlive() != null) {
             return request.scroll().keepAlive().millis();
-        } else if (request.keepAlive() != null) {
-            return request.keepAlive().millis();
         } else {
             return defaultKeepAlive;
         }
@@ -1085,6 +1121,8 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Set<String
      */
     public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException {
         assert request.searchType() == SearchType.QUERY_THEN_FETCH : "unexpected search type: " + request.searchType();
+        // TODO: support can_match with reader contexts after https://github.com/elastic/elasticsearch/pull/54966
+        assert request.readerId() == null : "request with reader_id bypass can_match phase";
         IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
         IndexShard indexShard = indexService.getShard(request.shardId().getId());
         // we don't want to use the reader wrapper since it could run costly operations
diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index a8fc8a24be20a..1b20917e785f3 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -1629,37 +1629,33 @@ private static final class XContentParams {
         private final TimeValue keepAlive;
 
         public ReaderBuilder(String id, TimeValue keepAlive) {
-            if (id == null && keepAlive == null) {
-                throw new IllegalArgumentException("id or keep_alive must be specified");
-            }
-            this.id = id;
-            this.keepAlive = keepAlive;
+            this.id = Objects.requireNonNull(id);
+            this.keepAlive = Objects.requireNonNull(keepAlive);
         }
 
         public ReaderBuilder(StreamInput in) throws IOException {
-            id = in.readOptionalString();
-            keepAlive = in.readOptionalTimeValue();
+            id = in.readString();
+            keepAlive = in.readTimeValue();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeOptionalString(id);
-            out.writeOptionalTimeValue(keepAlive);
+            out.writeString(id);
+            out.writeTimeValue(keepAlive);
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            if (id != null) {
-                builder.field(ID_FIELD.getPreferredName(), id);
-            }
-            if (keepAlive != null) {
-                builder.field(KEEP_ALIVE_FIELD.getPreferredName(), keepAlive);
-            }
+            builder.field(ID_FIELD.getPreferredName(), id);
+            builder.field(KEEP_ALIVE_FIELD.getPreferredName(), keepAlive);
             return builder;
         }
 
         public static ReaderBuilder fromXContent(XContentParser parser) throws IOException {
             final XContentParams params = PARSER.parse(parser, null);
+            if (params.id == null || params.keepAlive == null) {
+                throw new IllegalArgumentException("id and keep_alive must be specified");
+            }
             return new ReaderBuilder(params.id, params.keepAlive);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index b57f8cff7ccc5..254c0ab300e6a 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -176,6 +176,7 @@ private ShardSearchRequest(OriginalIndices originalIndices,
         this.originalIndices = originalIndices;
         this.readerId = readerId;
         this.keepAlive = keepAlive;
+        assert (readerId != null) == (keepAlive != null);
     }
 
     public ShardSearchRequest(StreamInput in) throws IOException {
@@ -216,6 +217,7 @@ public ShardSearchRequest(StreamInput in) throws IOException {
             this.keepAlive = null;
         }
         originalIndices = OriginalIndices.readOriginalIndices(in);
+        assert (readerId != null) == (keepAlive != null);
     }
 
     public ShardSearchRequest(ShardSearchRequest clone) {
diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
index ed5b6f41023c1..21b6363c8015d 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
@@ -88,8 +88,7 @@ public SearchRequest getRequest() {
     @Override
     public void sendSearchResponse(InternalSearchResponse internalSearchResponse, AtomicArray<SearchPhaseResult> queryResults) {
         String scrollId = getRequest().scroll() != null ? TransportSearchHelper.buildScrollId(queryResults, Version.CURRENT) : null;
-        String readerContextId = getRequest().source() != null && getRequest().source().reader() != null ?
-            TransportSearchHelper.buildScrollId(queryResults, Version.CURRENT) : null;
+        String readerContextId = getRequest().reader() != null ? TransportSearchHelper.buildScrollId(queryResults, Version.CURRENT) : null;
         searchResponse.set(new SearchResponse(internalSearchResponse, scrollId, numShards, numSuccess.get(), 0, 0,
             failures.toArray(ShardSearchFailure.EMPTY_ARRAY), SearchResponse.Clusters.EMPTY, readerContextId));
     }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
index 5ff66a0709701..6f99c7c981fa7 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
@@ -164,6 +164,50 @@ public void testValidate() throws IOException {
             assertEquals(1, validationErrors.validationErrors().size());
             assertEquals("using [rescore] is not allowed in a scroll context", validationErrors.validationErrors().get(0));
         }
+        {
+            // Reader context with scroll
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder()
+                    .reader(new SearchSourceBuilder.ReaderBuilder("id", TimeValue.timeValueMillis(randomIntBetween(1, 10)))))
+                .scroll(TimeValue.timeValueMillis(randomIntBetween(1, 100)));
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertEquals(1, validationErrors.validationErrors().size());
+            assertEquals("using [reader] is not allowed in a scroll context", validationErrors.validationErrors().get(0));
+        }
+        {
+            // Reader context with indices
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder()
+                    .reader(new SearchSourceBuilder.ReaderBuilder("id", TimeValue.timeValueMillis(randomIntBetween(1, 10)))))
+                .indices("test");
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertEquals(1, validationErrors.validationErrors().size());
+            assertEquals("[index] cannot be used with reader contexts", validationErrors.validationErrors().get(0));
+        }
+        {
+            // Reader context with preference
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder()
+                    .reader(new SearchSourceBuilder.ReaderBuilder("id", TimeValue.timeValueMillis(randomIntBetween(1, 10)))))
+                .preference("test");
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertEquals(1, validationErrors.validationErrors().size());
+            assertEquals("[preference] cannot be used with reader contexts", validationErrors.validationErrors().get(0));
+        }
+        {
+            // Reader context with routing
+            SearchRequest searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder()
+                    .reader(new SearchSourceBuilder.ReaderBuilder("id", TimeValue.timeValueMillis(randomIntBetween(1, 10)))))
+                .routing("test");
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertEquals(1, validationErrors.validationErrors().size());
+            assertEquals("[routing] cannot be used with reader contexts", validationErrors.validationErrors().get(0));
+        }
     }
 
     public void testCopyConstructor() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index b56e97f4d2292..11804fdca1ba1 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -1020,6 +1020,16 @@ OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
         latch.await();
     }
 
+    public void testOpenReaderContext() {
+        createIndex("index");
+        SearchService searchService = getInstanceFromNode(SearchService.class);
+        PlainActionFuture<SearchContextId> future = new PlainActionFuture<>();
+        searchService.openReaderContext(new ShardId(resolveIndex("index"), 0), TimeValue.timeValueMinutes(between(1, 10)), future);
+        future.actionGet();
+        assertThat(searchService.getActiveContexts(), equalTo(1));
+        assertTrue(searchService.freeReaderContext(future.actionGet()));
+    }
+
     private ReaderContext createReaderContext(IndexShard shard) {
         Engine.Searcher searcher = shard.acquireSearcher("test");
         return new ReaderContext(randomNonNegativeLong(), shard, searcher, randomNonNegativeLong(), false);
diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
index 3fbe1c5b8a4b4..90ff475263e3c 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.OriginalIndicesTests;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
@@ -57,8 +58,8 @@ public QuerySearchResultTests() {
     private static QuerySearchResult createTestInstance() throws Exception {
         ShardId shardId = new ShardId("index", "uuid", randomInt());
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean());
-        ShardSearchRequest shardSearchRequest = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shardId, 1,
-            new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, randomNonNegativeLong(), null, new String[0]);
+        ShardSearchRequest shardSearchRequest = new ShardSearchRequest(OriginalIndicesTests.randomOriginalIndices(), searchRequest,
+            shardId, 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, randomNonNegativeLong(), null, new String[0]);
         QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.base64UUID(), randomLong()),
             new SearchShardTarget("node", shardId, null, OriginalIndices.NONE), shardSearchRequest);
         if (randomBoolean()) {
diff --git a/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java b/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
index 2e4df00d16aa5..ffc1d83200e85 100644
--- a/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
+++ b/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
@@ -23,9 +23,12 @@
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.search.ClearReaderAction;
 import org.elasticsearch.action.search.ClearReaderRequest;
+import org.elasticsearch.action.search.OpenReaderRequest;
+import org.elasticsearch.action.search.OpenReaderResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.action.search.TransportOpenReaderAction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -58,7 +61,9 @@ public void testBasic() {
             client().prepareIndex("test").setId(id).setSource("value", i).get();
         }
         refresh("test");
-        SearchResponse resp1 = client().prepareSearch("test").setReader(null, TimeValue.timeValueMinutes(2)).get();
+        String readerId = openReaders(new String[]{"test"}, TimeValue.timeValueMinutes(2));
+        SearchResponse resp1 = client().prepareSearch().setPreference(null).setReader(readerId, TimeValue.timeValueMinutes(2)).get();
+        assertThat(resp1.getReaderId(), equalTo(readerId));
         assertHitCount(resp1, numDocs);
         int deletedDocs = 0;
         for (int i = 0; i < numDocs; i++) {
@@ -70,18 +75,19 @@ public void testBasic() {
         }
         refresh("test");
         if (randomBoolean()) {
-            SearchResponse resp2 = client().prepareSearch("test").setQuery(new MatchAllQueryBuilder()).get();
+            SearchResponse resp2 = client().prepareSearch("test").setPreference(null).setQuery(new MatchAllQueryBuilder()).get();
             assertNoFailures(resp2);
             assertHitCount(resp2, numDocs - deletedDocs);
         }
         try {
-            SearchResponse resp3 = client().prepareSearch().setQuery(new MatchAllQueryBuilder())
+            SearchResponse resp3 = client().prepareSearch().setPreference(null).setQuery(new MatchAllQueryBuilder())
                 .setReader(resp1.getReaderId(), TimeValue.timeValueMinutes(2))
                 .get();
             assertNoFailures(resp3);
             assertHitCount(resp3, numDocs);
+            assertThat(resp3.getReaderId(), equalTo(readerId));
         } finally {
-            clearReaderId(resp1.getReaderId());
+            clearReaderId(readerId);
         }
     }
 
@@ -97,7 +103,8 @@ public void testMultipleIndices() {
             client().prepareIndex(index).setId(id).setSource("value", i).get();
         }
         refresh();
-        SearchResponse resp1 = client().prepareSearch("*").setReader(null, TimeValue.timeValueMinutes(2)).get();
+        String readerId = openReaders(new String[]{"*"}, TimeValue.timeValueMinutes(2));
+        SearchResponse resp1 = client().prepareSearch().setPreference(null).setReader(readerId, TimeValue.timeValueMinutes(2)).get();
         assertNoFailures(resp1);
         assertHitCount(resp1, numDocs);
         int moreDocs = randomIntBetween(10, 50);
@@ -112,7 +119,8 @@ public void testMultipleIndices() {
             assertNoFailures(resp2);
             assertHitCount(resp2, numDocs + moreDocs);
 
-            SearchResponse resp3 = client().prepareSearch().setReader(resp1.getReaderId(), null).get();
+            SearchResponse resp3 = client().prepareSearch().setPreference(null)
+                .setReader(resp1.getReaderId(), TimeValue.timeValueMinutes(1)).get();
             assertNoFailures(resp3);
             assertHitCount(resp3, numDocs);
         } finally {
@@ -128,8 +136,9 @@ public void testReaderIdNotFound() throws Exception {
             client().prepareIndex("index").setId(id).setSource("value", i).get();
         }
         refresh();
-        SearchResponse resp1 = client().prepareSearch("index")
-            .setReader(null, TimeValue.timeValueSeconds(randomIntBetween(2, 5))).get();
+        String readerId = openReaders(new String[]{"index"}, TimeValue.timeValueSeconds(5));
+        SearchResponse resp1 = client().prepareSearch().setPreference(null)
+            .setReader(readerId, TimeValue.timeValueMillis(randomIntBetween(0, 10))).get();
         assertNoFailures(resp1);
         assertHitCount(resp1, index1);
         if (rarely()) {
@@ -141,7 +150,7 @@ public void testReaderIdNotFound() throws Exception {
             clearReaderId(resp1.getReaderId());
         }
         SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class,
-            () -> client().prepareSearch().setReader(resp1.getReaderId(), null).get());
+            () -> client().prepareSearch().setPreference(null).setReader(resp1.getReaderId(), TimeValue.timeValueMinutes(1)).get());
         for (ShardSearchFailure failure : e.shardFailures()) {
             assertThat(ExceptionsHelper.unwrapCause(failure.getCause()), instanceOf(SearchContextMissingException.class));
         }
@@ -163,7 +172,8 @@ public void testIndexNotFound() {
             client().prepareIndex("index-2").setId(id).setSource("value", i).get();
         }
         refresh();
-        SearchResponse resp1 = client().prepareSearch("index-*").setReader(null, TimeValue.timeValueMinutes(2)).get();
+        String readerId = openReaders(new String[]{"index-*"}, TimeValue.timeValueMinutes(2));
+        SearchResponse resp1 = client().prepareSearch().setPreference(null).setReader(readerId, TimeValue.timeValueMinutes(2)).get();
         assertNoFailures(resp1);
         assertHitCount(resp1, index1 + index2);
         client().admin().indices().prepareDelete("index-1").get();
@@ -173,10 +183,18 @@ public void testIndexNotFound() {
             assertHitCount(resp2, index2);
 
         }
-        expectThrows(IndexNotFoundException.class, () -> client().prepareSearch().setReader(resp1.getReaderId(), null).get());
+        expectThrows(IndexNotFoundException.class, () -> client().prepareSearch()
+            .setPreference(null)
+            .setReader(resp1.getReaderId(), TimeValue.timeValueMinutes(1)).get());
         clearReaderId(resp1.getReaderId());
     }
 
+    private String openReaders(String[] indices, TimeValue keepAlive) {
+        OpenReaderRequest request = new OpenReaderRequest(indices, OpenReaderRequest.DEFAULT_INDICES_OPTIONS, keepAlive, null, null);
+        final OpenReaderResponse response = client().execute(TransportOpenReaderAction.INSTANCE, request).actionGet();
+        return response.getReaderId();
+    }
+
     private void clearReaderId(String readerId) {
         client().execute(ClearReaderAction.INSTANCE, new ClearReaderRequest(readerId)).actionGet();
     }

From 51f9542d25b8e0d0fe9cc80e13d18dae3ecf067d Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 28 Apr 2020 00:11:41 +0200
Subject: [PATCH 12/47] Adds the ability to acquire readers in IndexShard
 (#54966)

This change adds the ability to acquire a point in time reader on an engine.
This is needed for frozen indices that lazily loads the reader on every
phase of the search requests. Acquiring a reader on a frozen index ensures
that the engine will not be closed until the reader is released, leaving the
directory reader unopened until a call to acquire a searcher is made.
When the searcher is closed, the underlyinng directory reader is also closed
unless another requests on the same frozen shard is in-flight. This ensures
that the directory reader of frozen indices is opened only when requests are
executed (they consume a thread in the search throttled pool).
---
 .../action/search/SearchTransportService.java |   4 +-
 .../search/TransportOpenReaderAction.java     |  30 +-
 .../elasticsearch/index/engine/Engine.java    | 109 ++--
 .../index/search/stats/ShardSearchStats.java  |   7 +-
 .../elasticsearch/index/shard/IndexShard.java |  16 +-
 .../index/shard/SearchOperationListener.java  |  28 +-
 .../search/DefaultSearchContext.java          |   6 +-
 .../elasticsearch/search/SearchService.java   | 286 +++++-----
 .../search/internal/LegacyReaderContext.java  |  33 +-
 .../search/internal/ReaderContext.java        |  52 +-
 .../search/internal/ScrollContext.java        |  32 --
 .../search/internal/SearchContext.java        |   4 +-
 .../search/internal/ShardSearchRequest.java   |   2 +-
 .../shard/SearchOperationListenerTests.java   |  24 +-
 .../search/DefaultSearchContextTests.java     |  33 +-
 .../search/SearchServiceTests.java            |  30 +-
 .../search/internal/ScrollContextTests.java   |  36 --
 .../test/engine/MockInternalEngine.java       |   6 +
 .../index/engine/FrozenEngine.java            | 489 +++---------------
 .../xpack/frozen/FrozenIndices.java           |   9 -
 .../index/engine/FrozenEngineTests.java       | 204 ++++----
 .../index/engine/FrozenIndexTests.java        |  49 +-
 .../SecuritySearchOperationListener.java      |  42 +-
 .../DocumentLevelSecurityTests.java           |  43 ++
 .../integration/FieldLevelSecurityTests.java  |  53 +-
 .../SecuritySearchOperationListenerTests.java | 234 +++++----
 26 files changed, 847 insertions(+), 1014 deletions(-)
 delete mode 100644 server/src/test/java/org/elasticsearch/search/internal/ScrollContextTests.java

diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index be697cc78f79b..38a0288c5e2f0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -374,9 +374,9 @@ public static void registerRequestHandler(TransportService transportService, Sea
         transportService.registerRequestHandler(SHARD_OPEN_READER_NAME, ThreadPool.Names.SAME,
             TransportOpenReaderAction.ShardOpenReaderRequest::new,
             (request, channel, task) -> {
-                searchService.openReaderContext(request.searchShardTarget.getShardId(), request.keepAlive,
+                searchService.openReaderContext(request.getShardId(), request.keepAlive,
                     ActionListener.map(new ChannelActionListener<>(channel, SHARD_OPEN_READER_NAME, request),
-                        contextId -> new TransportOpenReaderAction.ShardOpenReaderResponse(contextId, request.searchShardTarget)));
+                        contextId -> new TransportOpenReaderAction.ShardOpenReaderResponse(contextId)));
             });
         TransportActionProxy.registerProxyAction(
             transportService, SHARD_OPEN_READER_NAME, TransportOpenReaderAction.ShardOpenReaderResponse::new);
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java
index b551202279a87..b827610893617 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -34,6 +35,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.AliasFilter;
@@ -93,42 +95,49 @@ public AbstractSearchAsyncAction<? extends SearchPhaseResult> asyncSearchAction(
     }
 
     static final class ShardOpenReaderRequest extends TransportRequest implements IndicesRequest {
-        final SearchShardTarget searchShardTarget;
+        final ShardId shardId;
+        final OriginalIndices originalIndices;
         final TimeValue keepAlive;
 
-        ShardOpenReaderRequest(SearchShardTarget searchShardTarget, TimeValue keepAlive) {
-            this.searchShardTarget = searchShardTarget;
+        ShardOpenReaderRequest(ShardId shardId, OriginalIndices originalIndices, TimeValue keepAlive) {
+            this.shardId = shardId;
+            this.originalIndices = originalIndices;
             this.keepAlive = keepAlive;
         }
 
         ShardOpenReaderRequest(StreamInput in) throws IOException {
             super(in);
-            searchShardTarget = new SearchShardTarget(in);
+            shardId = new ShardId(in);
+            originalIndices = OriginalIndices.readOriginalIndices(in);
             keepAlive = in.readTimeValue();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            searchShardTarget.writeTo(out);
+            shardId.writeTo(out);
+            OriginalIndices.writeOriginalIndices(originalIndices, out);
             out.writeTimeValue(keepAlive);
         }
 
+        public ShardId getShardId() {
+            return shardId;
+        }
+
         @Override
         public String[] indices() {
-            return searchShardTarget.getOriginalIndices().indices();
+            return originalIndices.indices();
         }
 
         @Override
         public IndicesOptions indicesOptions() {
-            return searchShardTarget.getOriginalIndices().indicesOptions();
+            return originalIndices.indicesOptions();
         }
     }
 
     static final class ShardOpenReaderResponse extends SearchPhaseResult {
-        ShardOpenReaderResponse(SearchContextId contextId, SearchShardTarget searchShardTarget) {
+        ShardOpenReaderResponse(SearchContextId contextId) {
             this.contextId = contextId;
-            setSearchShardTarget(searchShardTarget);
         }
 
         ShardOpenReaderResponse(StreamInput in) throws IOException {
@@ -163,7 +172,8 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                                            SearchActionListener<SearchPhaseResult> listener) {
             final Transport.Connection connection = getConnection(shardIt.getClusterAlias(), shard.currentNodeId());
             final SearchShardTarget searchShardTarget = shardIt.newSearchShardTarget(shard.currentNodeId());
-            final ShardOpenReaderRequest shardRequest = new ShardOpenReaderRequest(searchShardTarget, openReaderRequest.keepAlive());
+            final ShardOpenReaderRequest shardRequest = new ShardOpenReaderRequest(searchShardTarget.getShardId(),
+                searchShardTarget.getOriginalIndices(), openReaderRequest.keepAlive());
             getSearchTransport().sendShardOpenReader(connection, getTask(), shardRequest, ActionListener.map(listener, r -> r));
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index bd90f73054c94..f795a8cf69b17 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -95,6 +95,7 @@
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.BiFunction;
+import java.util.function.Function;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
@@ -588,31 +589,17 @@ protected final GetResult getFromSearcher(Get get, BiFunction<String, SearcherSc
 
     public abstract GetResult get(Get get, BiFunction<String, SearcherScope, Searcher> searcherFactory) throws EngineException;
 
-
     /**
-     * Returns a new searcher instance. The consumer of this
-     * API is responsible for releasing the returned searcher in a
-     * safe manner, preferably in a try/finally block.
-     *
-     * @param source the source API or routing that triggers this searcher acquire
-     *
-     * @see Searcher#close()
+     * Acquires a point-in-time reader that can be used to create {@link Engine.Searcher}s on demand.
      */
-    public final Searcher acquireSearcher(String source) throws EngineException {
-        return acquireSearcher(source, SearcherScope.EXTERNAL);
+    public final SearcherSupplier acquireSearcherSupplier(Function<Searcher, Searcher> wrapper) throws EngineException {
+        return acquireSearcherSupplier(wrapper, SearcherScope.EXTERNAL);
     }
 
     /**
-     * Returns a new searcher instance. The consumer of this
-     * API is responsible for releasing the returned searcher in a
-     * safe manner, preferably in a try/finally block.
-     *
-     * @param source the source API or routing that triggers this searcher acquire
-     * @param scope the scope of this searcher ie. if the searcher will be used for get or search purposes
-     *
-     * @see Searcher#close()
+     * Acquires a point-in-time reader that can be used to create {@link Engine.Searcher}s on demand.
      */
-    public Searcher acquireSearcher(String source, SearcherScope scope) throws EngineException {
+    public SearcherSupplier acquireSearcherSupplier(Function<Searcher, Searcher> wrapper, SearcherScope scope) throws EngineException {
         /* Acquire order here is store -> manager since we need
          * to make sure that the store is not closed before
          * the searcher is acquired. */
@@ -621,35 +608,64 @@ public Searcher acquireSearcher(String source, SearcherScope scope) throws Engin
         }
         Releasable releasable = store::decRef;
         try {
-            assert assertSearcherIsWarmedUp(source, scope);
             ReferenceManager<ElasticsearchDirectoryReader> referenceManager = getReferenceManager(scope);
-            final ElasticsearchDirectoryReader acquire = referenceManager.acquire();
+            ElasticsearchDirectoryReader acquire = referenceManager.acquire();
             AtomicBoolean released = new AtomicBoolean(false);
-            Searcher engineSearcher = new Searcher(source, acquire,
-                engineConfig.getSimilarity(), engineConfig.getQueryCache(), engineConfig.getQueryCachingPolicy(),
-                () -> {
-                if (released.compareAndSet(false, true)) {
-                    try {
-                        referenceManager.release(acquire);
-                    } finally {
-                        store.decRef();
+            SearcherSupplier reader = new SearcherSupplier(wrapper) {
+                @Override
+                public Searcher acquireSearcherInternal(String source) {
+                    assert assertSearcherIsWarmedUp(source, scope);
+                    return new Searcher(source, acquire, engineConfig.getSimilarity(), engineConfig.getQueryCache(),
+                        engineConfig.getQueryCachingPolicy(), () -> {});
+                }
+
+                @Override
+                public void close() {
+                    if (released.compareAndSet(false, true)) {
+                        try {
+                            referenceManager.release(acquire);
+                        } catch (IOException e) {
+                            throw new UncheckedIOException("failed to close", e);
+                        } catch (AlreadyClosedException e) {
+                            // This means there's a bug somewhere: don't suppress it
+                            throw new AssertionError(e);
+                        } finally {
+                            store.decRef();
+                        }
+                    } else {
+                        /* In general, readers should never be released twice or this would break reference counting. There is one rare case
+                         * when it might happen though: when the request and the Reaper thread would both try to release it in a very short
+                         * amount of time, this is why we only log a warning instead of throwing an exception. */
+                        logger.warn("Reader was released twice", new IllegalStateException("Double release"));
                     }
-                } else {
-                    /* In general, readers should never be released twice or this would break reference counting. There is one rare case
-                     * when it might happen though: when the request and the Reaper thread would both try to release it in a very short
-                     * amount of time, this is why we only log a warning instead of throwing an exception. */
-                    logger.warn("Searcher was released twice", new IllegalStateException("Double release"));
                 }
-              });
+            };
             releasable = null; // success - hand over the reference to the engine reader
-            return engineSearcher;
+            return reader;
         } catch (AlreadyClosedException ex) {
             throw ex;
         } catch (Exception ex) {
-            maybeFailEngine("acquire_searcher", ex);
+            maybeFailEngine("acquire_reader", ex);
             ensureOpen(ex); // throw EngineCloseException here if we are already closed
-            logger.error(() -> new ParameterizedMessage("failed to acquire searcher, source {}", source), ex);
-            throw new EngineException(shardId, "failed to acquire searcher, source " + source, ex);
+            logger.error(() -> new ParameterizedMessage("failed to acquire reader"), ex);
+            throw new EngineException(shardId, "failed to acquire reader", ex);
+        } finally {
+            Releasables.close(releasable);
+        }
+    }
+
+    public final Searcher acquireSearcher(String source) throws EngineException {
+        return acquireSearcher(source, SearcherScope.EXTERNAL);
+    }
+
+    public Searcher acquireSearcher(String source, SearcherScope scope) throws EngineException {
+        SearcherSupplier releasable = null;
+        try {
+            SearcherSupplier reader = releasable = acquireSearcherSupplier(Function.identity(), scope);
+            Searcher searcher = reader.acquireSearcher(source);
+            releasable = null;
+            return new Searcher(source, searcher.getDirectoryReader(), searcher.getSimilarity(),
+                searcher.getQueryCache(), searcher.getQueryCachingPolicy(), () -> Releasables.close(searcher, reader));
         } finally {
             Releasables.close(releasable);
         }
@@ -1158,6 +1174,21 @@ default void onFailedEngine(String reason, @Nullable Exception e) {
         }
     }
 
+    public abstract static class SearcherSupplier implements Releasable {
+        private final Function<Searcher, Searcher> wrapper;
+
+        public SearcherSupplier(Function<Searcher, Searcher> wrapper) {
+            this.wrapper = wrapper;
+        }
+
+        public final Searcher acquireSearcher(String source) {
+            final Searcher searcher = acquireSearcherInternal(source);
+            return "can_match".equals(source) ? searcher : wrapper.apply(searcher);
+        }
+
+        protected abstract Searcher acquireSearcherInternal(String source);
+    }
+
     public static final class Searcher extends IndexSearcher implements Releasable {
         private final String source;
         private final Closeable onClose;
diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
index 664b631249f68..e72a5182287e1 100644
--- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
+++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.search.internal.ReaderContext;
-import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.util.HashMap;
@@ -158,15 +157,15 @@ public void onFreeReaderContext(ReaderContext readerContext) {
     }
 
     @Override
-    public void onNewScrollContext(ScrollContext scrollContext) {
+    public void onNewScrollContext(ReaderContext readerContext) {
         totalStats.scrollCurrent.inc();
     }
 
     @Override
-    public void onFreeScrollContext(ScrollContext scrollContext) {
+    public void onFreeScrollContext(ReaderContext readerContext) {
         totalStats.scrollCurrent.dec();
         assert totalStats.scrollCurrent.count() >= 0;
-        totalStats.scrollMetric.inc(TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - scrollContext.getStartTimeInNano()));
+        totalStats.scrollMetric.inc(TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - readerContext.getStartTimeInNano()));
     }
 
     static final class StatsHolder {
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 984654261db43..b6b9884a15096 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1197,12 +1197,20 @@ public void failShard(String reason, @Nullable Exception e) {
     }
 
     /**
-     * Acquire a lightweight searcher which can be used to rewrite shard search requests.
+     * Acquires a point-in-time reader that can be used to create {@link Engine.Searcher}s on demand.
      */
-    public Engine.Searcher acquireCanMatchSearcher() {
+    public Engine.SearcherSupplier acquireSearcherSupplier() {
+        return acquireSearcherSupplier(Engine.SearcherScope.EXTERNAL);
+    }
+
+    /**
+     * Acquires a point-in-time reader that can be used to create {@link Engine.Searcher}s on demand.
+     */
+    public Engine.SearcherSupplier acquireSearcherSupplier(Engine.SearcherScope scope) {
         readAllowed();
         markSearcherAccessed();
-        return getEngine().acquireSearcher("can_match", Engine.SearcherScope.EXTERNAL);
+        final Engine engine = getEngine();
+        return engine.acquireSearcherSupplier(this::wrapSearcher, scope);
     }
 
     public Engine.Searcher acquireSearcher(String source) {
@@ -1218,7 +1226,7 @@ private Engine.Searcher acquireSearcher(String source, Engine.SearcherScope scop
         markSearcherAccessed();
         final Engine engine = getEngine();
         final Engine.Searcher searcher = engine.acquireSearcher(source, scope);
-        return wrapSearcher(searcher);
+        return "can_match".equals(source) ? searcher : wrapSearcher(searcher);
     }
 
     private Engine.Searcher wrapSearcher(Engine.Searcher searcher) {
diff --git a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
index 77e83edcdd18f..fff7e88e505cb 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
@@ -22,7 +22,6 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.search.internal.ReaderContext;
-import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.transport.TransportRequest;
 
@@ -93,28 +92,29 @@ default void onNewReaderContext(ReaderContext readerContext) {}
     default void onFreeReaderContext(ReaderContext readerContext) {}
 
     /**
-     * Executed when a new scroll search {@link SearchContext} was created
-     * @param scrollContext the created search context
+     * Executed when a new scroll search {@link ReaderContext} was created
+     * @param readerContext the created reader context
      */
-    default void onNewScrollContext(ScrollContext scrollContext) {}
+    default void onNewScrollContext(ReaderContext readerContext) {}
 
     /**
      * Executed when a scroll search {@link SearchContext} is freed.
      * This happens either when the scroll search execution finishes, if the
      * execution failed or if the search context as idle for and needs to be
      * cleaned up.
-     * @param scrollContext the freed search context
+     * @param readerContext the freed search context
      */
-    default void onFreeScrollContext(ScrollContext scrollContext) {}
+    default void onFreeScrollContext(ReaderContext readerContext) {}
 
     /**
      * Executed prior to using a {@link SearchContext} that has been retrieved
      * from the active contexts. If the context is deemed invalid a runtime
      * exception can be thrown, which will prevent the context from being used.
-     * @param context the context retrieved from the active contexts
+     * @param readerContext The reader context used by this request.
+     * @param searchContext The newly created {@link SearchContext}.
      * @param transportRequest the request that is going to use the search context
      */
-    default void validateSearchContext(SearchContext context, TransportRequest transportRequest) {}
+    default void validateSearchContext(ReaderContext readerContext, SearchContext searchContext, TransportRequest transportRequest) {}
 
     /**
      * Executed when a search context was freed. The implementor can implement
@@ -225,10 +225,10 @@ public void onFreeReaderContext(ReaderContext readerContext) {
         }
 
         @Override
-        public void onNewScrollContext(ScrollContext scrollContext) {
+        public void onNewScrollContext(ReaderContext readerContext) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onNewScrollContext(scrollContext);
+                    listener.onNewScrollContext(readerContext);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onNewScrollContext listener [{}] failed", listener), e);
                 }
@@ -236,10 +236,10 @@ public void onNewScrollContext(ScrollContext scrollContext) {
         }
 
         @Override
-        public void onFreeScrollContext(ScrollContext scrollContext) {
+        public void onFreeScrollContext(ReaderContext readerContext) {
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.onFreeScrollContext(scrollContext);
+                    listener.onFreeScrollContext(readerContext);
                 } catch (Exception e) {
                     logger.warn(() -> new ParameterizedMessage("onFreeScrollContext listener [{}] failed", listener), e);
                 }
@@ -247,11 +247,11 @@ public void onFreeScrollContext(ScrollContext scrollContext) {
         }
 
         @Override
-        public void validateSearchContext(SearchContext context, TransportRequest request) {
+        public void validateSearchContext(ReaderContext readerContext, SearchContext searchContext, TransportRequest request) {
             Exception exception = null;
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.validateSearchContext(context, request);
+                    listener.validateSearchContext(readerContext, searchContext, request);
                 } catch (Exception e) {
                     exception = ExceptionsHelper.useOrSuppress(exception, e);
                 }
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 2bdff5cafc965..c368590929a36 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -83,6 +83,7 @@
 final class DefaultSearchContext extends SearchContext {
 
     private final ReaderContext readerContext;
+    private final Engine.Searcher engineSearcher;
     private final ShardSearchRequest request;
     private final SearchShardTarget shardTarget;
     private final LongSupplier relativeTimeSupplier;
@@ -171,7 +172,7 @@ final class DefaultSearchContext extends SearchContext {
         this.indexShard = indexShard;
         this.indexService = indexService;
         this.clusterService = clusterService;
-        final Engine.Searcher engineSearcher = readerContext.engineSearcher();
+        this.engineSearcher = readerContext.acquireSearcher("search");
         this.searcher = new ContextIndexSearcher(engineSearcher.getIndexReader(), engineSearcher.getSimilarity(),
             engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy(), lowLevelCancellation);
         this.relativeTimeSupplier = relativeTimeSupplier;
@@ -184,6 +185,7 @@ final class DefaultSearchContext extends SearchContext {
 
     @Override
     public void doClose() {
+        engineSearcher.close();
         readerContext.decRef();
     }
 
@@ -304,7 +306,7 @@ public SearchContextId id() {
 
     @Override
     public String source() {
-        return readerContext.source();
+        return "search";
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 5ef4d2130f98e..f2617f13f9c1f 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -286,7 +286,6 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem
         if (reason == IndexRemovalReason.DELETED || reason == IndexRemovalReason.CLOSED || reason == IndexRemovalReason.REOPENED) {
             freeAllContextForIndex(index);
         }
-
     }
 
     protected void putReaderContext(ReaderContext context) {
@@ -323,7 +322,11 @@ public void executeDfsPhase(ShardSearchRequest request, boolean keepStatesInCont
             @Override
             public void onResponse(ShardSearchRequest rewritten) {
                 // fork the execution in the search thread pool
-                runAsync(shard, () -> executeDfsPhase(request, task, keepStatesInContext), listener);
+                try {
+                    runAsync(shard, () -> executeDfsPhase(request, task, keepStatesInContext), listener);
+                } catch (Exception exc) {
+                    listener.onFailure(exc);
+                }
             }
 
             @Override
@@ -369,40 +372,48 @@ public void executeQueryPhase(ShardSearchRequest request, boolean keepStatesInCo
         rewriteAndFetchShardRequest(shard, request, new ActionListener<ShardSearchRequest>() {
             @Override
             public void onResponse(ShardSearchRequest orig) {
-                if (orig.canReturnNullResponseIfMatchNoDocs()) {
-                    assert orig.shouldCreatePersistentReader() == false;
-                    // we clone the shard request and perform a quick rewrite using a lightweight
-                    // searcher since we are outside of the search thread pool.
-                    // If the request rewrites to "match none" we can shortcut the query phase
-                    // entirely. Otherwise we fork the execution in the search thread pool.
-                    ShardSearchRequest canMatchRequest = new ShardSearchRequest(orig);
-                    try (Engine.Searcher searcher = shard.acquireCanMatchSearcher()) {
-                        QueryShardContext context = indexService.newQueryShardContext(canMatchRequest.shardId().id(), searcher,
-                            canMatchRequest::nowInMillis, canMatchRequest.getClusterAlias());
-                        Rewriteable.rewrite(canMatchRequest.getRewriteable(), context, true);
-                    } catch (Exception exc) {
-                        listener.onFailure(exc);
-                        return;
-                    }
-                    if (canRewriteToMatchNone(canMatchRequest.source())
-                            && canMatchRequest.source().query() instanceof MatchNoneQueryBuilder) {
-                        if (canMatchRequest.readerId() != null) {
-                            ReaderContext readerContext = getReaderContext(canMatchRequest.readerId());
-                            if (readerContext == null) {
-                                listener.onFailure(new SearchContextMissingException(canMatchRequest.readerId()));
-                                return;
-                            }
-                            try (Releasable r = readerContext.markAsUsed()) {
-                                listener.onResponse(QuerySearchResult.nullInstance());
+                final ReaderContext readerContext = createOrGetReaderContext(orig, keepStatesInContext);
+                try {
+                    if (orig.canReturnNullResponseIfMatchNoDocs()) {
+                        assert orig.shouldCreatePersistentReader() == false;
+                        // we clone the shard request and perform a quick rewrite using a lightweight
+                        // searcher since we are outside of the search thread pool.
+                        // If the request rewrites to "match none" we can shortcut the query phase
+                        // entirely. Otherwise we fork the execution in the search thread pool.
+                        ShardSearchRequest canMatchRequest = new ShardSearchRequest(orig);
+                        try (Engine.Searcher searcher = readerContext.acquireSearcher("can_match")) {
+                            QueryShardContext context = indexService.newQueryShardContext(canMatchRequest.shardId().id(), searcher,
+                                canMatchRequest::nowInMillis, canMatchRequest.getClusterAlias());
+                            Rewriteable.rewrite(canMatchRequest.getRewriteable(), context, true);
+                        }
+                        if (canRewriteToMatchNone(canMatchRequest.source())
+                                && canMatchRequest.source().query() instanceof MatchNoneQueryBuilder) {
+                            if (orig.readerId() != null) {
+                                try (Releasable r = readerContext.markAsUsed()) {
+                                    listener.onResponse(QuerySearchResult.nullInstance());
+                                }
+                            } else {
+                                try {
+                                    listener.onResponse(QuerySearchResult.nullInstance());
+                                } finally {
+                                    // close and remove the ephemeral reader context
+                                    try (readerContext) {
+                                        removeReaderContext(readerContext.id().getId());
+                                    }
+                                }
                             }
-                        } else {
-                            listener.onResponse(QuerySearchResult.nullInstance());
+                            return;
                         }
-                        return;
+                    }
+                    // fork the execution in the search thread pool
+                    runAsync(shard, () -> executeQueryPhase(orig, task, readerContext), listener);
+                } catch (Exception exc) {
+                    try (readerContext) {
+                        removeReaderContext(readerContext.id().getId());
+                    } finally {
+                        listener.onFailure(exc);
                     }
                 }
-                // fork the execution in the search thread pool
-                runAsync(shard, () -> executeQueryPhase(orig, task, keepStatesInContext), listener);
             }
 
             @Override
@@ -414,20 +425,16 @@ public void onFailure(Exception exc) {
 
     private <T> void runAsync(IndexShard shard, CheckedSupplier<T, Exception> command, ActionListener<T> listener) {
         Executor executor = getExecutor(shard);
-        try {
-            executor.execute(() -> {
-                T result;
-                try {
-                    result = command.get();
-                } catch (Exception exc) {
-                    listener.onFailure(exc);
-                    return;
-                }
-                listener.onResponse(result);
-            });
-        } catch (Exception exc) {
-            listener.onFailure(exc);
-        }
+        executor.execute(() -> {
+            T result;
+            try {
+                result = command.get();
+            } catch (Exception exc) {
+                listener.onFailure(exc);
+                return;
+            }
+            listener.onResponse(result);
+        });
     }
 
     private <T> void runAsync(SearchContextId id, CheckedSupplier<T, IOException> executable, ActionListener<T> listener) {
@@ -436,8 +443,7 @@ private <T> void runAsync(SearchContextId id, CheckedSupplier<T, IOException> ex
 
     private SearchPhaseResult executeQueryPhase(ShardSearchRequest request,
                                                 SearchShardTask task,
-                                                boolean keepStatesInContext) throws Exception {
-        ReaderContext reader = createOrGetReaderContext(request, keepStatesInContext);
+                                                ReaderContext reader) throws Exception {
         try (SearchContext context = createContext(reader, request, task, true)) {
             final long afterQueryTime;
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
@@ -479,26 +485,26 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
         runAsync(request.contextId(), () -> {
-            final LegacyReaderContext reader = (LegacyReaderContext) findReaderContext(request.contextId());
-            try (SearchContext context = createContext(reader, reader.getShardSearchRequest(null), task, false);
-                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+            final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
+            try (SearchContext searchContext = createContext(readerContext, readerContext.getShardSearchRequest(null), task, false);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
                 if (request.scroll() != null && request.scroll().keepAlive() != null) {
                     final long keepAlive = request.scroll().keepAlive().millis();
                     checkKeepAliveLimit(keepAlive);
-                    reader.keepAlive(keepAlive);
+                    readerContext.keepAlive(keepAlive);
                 }
-                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
-                context.searcher().setAggregatedDfs(reader.getAggregatedDfs(null));
-                processScroll(request, reader, context);
-                queryPhase.execute(context);
+                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
+                searchContext.searcher().setAggregatedDfs(readerContext.getAggregatedDfs(null));
+                processScroll(request, readerContext, searchContext);
+                queryPhase.execute(searchContext);
                 executor.success();
-                final RescoreDocIds rescoreDocIds = context.rescoreDocIds();
-                context.queryResult().setRescoreDocIds(rescoreDocIds);
-                reader.setRescoreDocIds(rescoreDocIds);
-                return new ScrollQuerySearchResult(context.queryResult(), context.shardTarget());
+                final RescoreDocIds rescoreDocIds = searchContext.rescoreDocIds();
+                searchContext.queryResult().setRescoreDocIds(rescoreDocIds);
+                readerContext.setRescoreDocIds(rescoreDocIds);
+                return new ScrollQuerySearchResult(searchContext.queryResult(), searchContext.shardTarget());
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
-                processFailure(reader, e);
+                processFailure(readerContext, e);
                 throw e;
             }
         }, listener);
@@ -506,25 +512,26 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
 
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
         runAsync(request.contextId(), () -> {
-            final ReaderContext reader = findReaderContext(request.contextId());
-            reader.setAggregatedDfs(request.dfs());
-            try (SearchContext context = createContext(reader, reader.getShardSearchRequest(request.shardSearchRequest()), task, true);
-                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
-                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
-                context.searcher().setAggregatedDfs(request.dfs());
-                queryPhase.execute(context);
-                if (context.queryResult().hasSearchContext() == false && reader.singleSession()) {
+            final ReaderContext readerContext = findReaderContext(request.contextId());
+            readerContext.setAggregatedDfs(request.dfs());
+            try (SearchContext searchContext =
+                     createContext(readerContext, readerContext.getShardSearchRequest(request.shardSearchRequest()), task, true);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
+                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
+                searchContext.searcher().setAggregatedDfs(request.dfs());
+                queryPhase.execute(searchContext);
+                if (searchContext.queryResult().hasSearchContext() == false && readerContext.singleSession()) {
                     // no hits, we can release the context since there will be no fetch phase
-                    freeReaderContext(reader.id());
+                    freeReaderContext(readerContext.id());
                 }
                 executor.success();
-                final RescoreDocIds rescoreDocIds = context.rescoreDocIds();
-                context.queryResult().setRescoreDocIds(rescoreDocIds);
-                reader.setRescoreDocIds(rescoreDocIds);
-                return context.queryResult();
+                final RescoreDocIds rescoreDocIds = searchContext.rescoreDocIds();
+                searchContext.queryResult().setRescoreDocIds(rescoreDocIds);
+                readerContext.setRescoreDocIds(rescoreDocIds);
+                return searchContext.queryResult();
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
-                processFailure(reader, e);
+                processFailure(readerContext, e);
                 throw e;
             }
         }, listener);
@@ -542,24 +549,24 @@ private Executor getExecutor(IndexShard indexShard) {
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
         runAsync(request.contextId(), () -> {
-            final LegacyReaderContext reader = (LegacyReaderContext) findReaderContext(request.contextId());
-            try (SearchContext context = createContext(reader, reader.getShardSearchRequest(null), task, false);
-                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
+            final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
+            try (SearchContext searchContext = createContext(readerContext, readerContext.getShardSearchRequest(null), task, false);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
                 if (request.scroll() != null && request.scroll().keepAlive() != null) {
                     checkKeepAliveLimit(request.scroll().keepAlive().millis());
-                    reader.keepAlive(request.scroll().keepAlive().millis());
+                    readerContext.keepAlive(request.scroll().keepAlive().millis());
                 }
-                context.assignRescoreDocIds(reader.getRescoreDocIds(null));
-                context.searcher().setAggregatedDfs(reader.getAggregatedDfs(null));
-                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
-                processScroll(request, reader, context);
-                queryPhase.execute(context);
+                searchContext.assignRescoreDocIds(readerContext.getRescoreDocIds(null));
+                searchContext.searcher().setAggregatedDfs(readerContext.getAggregatedDfs(null));
+                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
+                processScroll(request, readerContext, searchContext);
+                queryPhase.execute(searchContext);
                 final long afterQueryTime = executor.success();
-                QueryFetchSearchResult fetchSearchResult = executeFetchPhase(reader, context, afterQueryTime);
-                return new ScrollQueryFetchSearchResult(fetchSearchResult, context.shardTarget());
+                QueryFetchSearchResult fetchSearchResult = executeFetchPhase(readerContext, searchContext, afterQueryTime);
+                return new ScrollQueryFetchSearchResult(fetchSearchResult, searchContext.shardTarget());
             } catch (Exception e) {
                 logger.trace("Fetch phase failed", e);
-                processFailure(reader, e);
+                processFailure(readerContext, e);
                 throw e;
             }
         }, listener);
@@ -567,27 +574,28 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
 
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
         runAsync(request.contextId(), () -> {
-            final ReaderContext reader = findReaderContext(request.contextId());
-            final ShardSearchRequest shardSearchRequest = reader.getShardSearchRequest(request.getShardSearchRequest());
-            try (SearchContext context = createContext(reader, shardSearchRequest, task, false)) {
-                reader.indexShard().getSearchOperationListener().validateSearchContext(context, request);
+            final ReaderContext readerContext = findReaderContext(request.contextId());
+            final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest());
+            try (SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false)) {
+                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
                 if (request.lastEmittedDoc() != null) {
-                    context.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
+                    searchContext.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
                 }
-                context.assignRescoreDocIds(reader.getRescoreDocIds(request.getRescoreDocIds()));
-                context.searcher().setAggregatedDfs(reader.getAggregatedDfs(request.getAggregatedDfs()));
-                context.docIdsToLoad(request.docIds(), 0, request.docIdsSize());
-                try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, System.nanoTime())) {
-                    fetchPhase.execute(context);
-                    if (reader.singleSession()) {
+                searchContext.assignRescoreDocIds(readerContext.getRescoreDocIds(request.getRescoreDocIds()));
+                searchContext.searcher().setAggregatedDfs(readerContext.getAggregatedDfs(request.getAggregatedDfs()));
+                searchContext.docIdsToLoad(request.docIds(), 0, request.docIdsSize());
+                try (SearchOperationListenerExecutor executor =
+                         new SearchOperationListenerExecutor(searchContext, true, System.nanoTime())) {
+                    fetchPhase.execute(searchContext);
+                    if (readerContext.singleSession()) {
                         freeReaderContext(request.contextId());
                     }
                     executor.success();
                 }
-                return context.fetchResult();
+                return searchContext.fetchResult();
             } catch (Exception e) {
                 logger.trace("Fetch phase failed", e);
-                processFailure(reader, e);
+                processFailure(readerContext, e);
                 throw e;
             }
         }, listener);
@@ -624,12 +632,12 @@ final ReaderContext createOrGetReaderContext(ShardSearchRequest request, boolean
         }
         IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
         IndexShard shard = indexService.getShard(request.shardId().id());
-        Engine.Searcher searcher = shard.acquireSearcher("search");
-        return createAndPutReaderContext(request, shard, searcher, keepStatesInContext);
+        Engine.SearcherSupplier reader = shard.acquireSearcherSupplier();
+        return createAndPutReaderContext(request, shard, reader, keepStatesInContext);
     }
 
     final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexShard shard,
-                                                  Engine.Searcher engineSearcher, boolean keepStatesInContext) {
+                                                  Engine.SearcherSupplier reader, boolean keepStatesInContext) {
         assert request.readerId() == null;
         assert request.keepAlive() == null;
         ReaderContext readerContext = null;
@@ -644,31 +652,29 @@ final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexS
                             + MAX_OPEN_SCROLL_CONTEXT.getKey() + "] setting.");
                 }
             }
+            final long keepAlive = getKeepAlive(request);
+            checkKeepAliveLimit(keepAlive);
             if (keepStatesInContext || request.scroll() != null) {
-                long keepAlive = defaultKeepAlive;
-                if (request.scroll() != null && request.scroll().keepAlive() != null) {
-                    keepAlive = request.scroll().keepAlive().millis();
-                    checkKeepAliveLimit(keepAlive);
-                }
-                readerContext = new LegacyReaderContext(idGenerator.incrementAndGet(), shard, engineSearcher, request, keepAlive);
+                readerContext = new LegacyReaderContext(idGenerator.incrementAndGet(), shard, reader, request, keepAlive);
                 if (request.scroll() != null) {
                     readerContext.addOnClose(decreaseScrollContexts);
                     decreaseScrollContexts = null;
                 }
             } else {
-                readerContext = new ReaderContext(idGenerator.incrementAndGet(), shard, engineSearcher, defaultKeepAlive, true);
+                readerContext = new ReaderContext(idGenerator.incrementAndGet(), shard, reader, keepAlive,
+                    request.keepAlive() == null);
             }
-            engineSearcher = null;
+            reader = null;
             final ReaderContext finalReaderContext = readerContext;
             final SearchOperationListener searchOperationListener = shard.getSearchOperationListener();
             searchOperationListener.onNewReaderContext(finalReaderContext);
             if (finalReaderContext.scrollContext() != null) {
-                searchOperationListener.onNewScrollContext(finalReaderContext.scrollContext());
+                searchOperationListener.onNewScrollContext(finalReaderContext);
             }
             readerContext.addOnClose(() -> {
                 try {
                     if (finalReaderContext.scrollContext() != null) {
-                        searchOperationListener.onFreeScrollContext(finalReaderContext.scrollContext());
+                        searchOperationListener.onFreeScrollContext(finalReaderContext);
                     }
                 } finally {
                     searchOperationListener.onFreeReaderContext(finalReaderContext);
@@ -678,7 +684,7 @@ final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexS
             readerContext = null;
             return finalReaderContext;
         } finally {
-            Releasables.close(engineSearcher, readerContext, decreaseScrollContexts);
+            Releasables.close(reader, readerContext, decreaseScrollContexts);
         }
     }
 
@@ -691,25 +697,25 @@ public void openReaderContext(ShardId shardId, TimeValue keepAlive, ActionListen
         final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         final IndexShard shard = indexService.getShard(shardId.id());
         final SearchOperationListener searchOperationListener = shard.getSearchOperationListener();
-        shard.awaitShardSearchActive(ignored ->
-            runAsync(shard, () -> {
-                Releasable releasable = null;
-                try {
-                    final Engine.Searcher engineSearcher = shard.acquireSearcher("search");
-                    releasable = engineSearcher;
-                    final ReaderContext readerContext = new ReaderContext(
-                        idGenerator.incrementAndGet(), shard, engineSearcher, keepAlive.millis(), false);
-                    releasable = readerContext;
-                    searchOperationListener.onNewReaderContext(readerContext);
-                    readerContext.addOnClose(() -> searchOperationListener.onFreeReaderContext(readerContext));
-                    putReaderContext(readerContext);
-                    releasable = null;
-                    return readerContext.id();
-                } finally {
-                    Releasables.close(releasable);
-                }
-            }, listener)
-        );
+        shard.awaitShardSearchActive(ignored -> {
+            Releasable releasable = null;
+            try {
+                final Engine.SearcherSupplier reader = shard.acquireSearcherSupplier();
+                releasable = reader;
+                final ReaderContext readerContext = new ReaderContext(
+                    idGenerator.incrementAndGet(), shard, reader, keepAlive.millis(), false);
+                releasable = readerContext;
+                searchOperationListener.onNewReaderContext(readerContext);
+                readerContext.addOnClose(() -> searchOperationListener.onFreeReaderContext(readerContext));
+                putReaderContext(readerContext);
+                releasable = null;
+                listener.onResponse(readerContext.id());
+            } catch (Exception exc) {
+                listener.onFailure(exc);
+            } finally {
+                Releasables.close(releasable);
+            }
+        });
     }
 
     final SearchContext createContext(ReaderContext reader,
@@ -749,12 +755,14 @@ final SearchContext createContext(ReaderContext reader,
 
     public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout) throws IOException {
         IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().getId());
-        Engine.Searcher engineSearcher = indexShard.acquireSearcher("search");
-        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, engineSearcher, -1L, true)) {
-            engineSearcher = null; // transfer ownership to readerContext
-            return createSearchContext(readerContext, request, timeout);
+        Engine.SearcherSupplier reader = indexShard.acquireSearcherSupplier();
+        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, reader, -1L, true)) {
+            reader = null; // transfer ownership to readerContext
+            DefaultSearchContext searchContext = createSearchContext(readerContext, request, timeout);
+            searchContext.addReleasable(readerContext);
+            return searchContext;
         } finally {
-            Releasables.close(engineSearcher);
+            Releasables.close(reader);
         }
     }
 
@@ -1127,7 +1135,7 @@ public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException
         IndexShard indexShard = indexService.getShard(request.shardId().getId());
         // we don't want to use the reader wrapper since it could run costly operations
         // and we can afford false positives.
-        try (Engine.Searcher searcher = indexShard.acquireCanMatchSearcher()) {
+        try (Engine.Searcher searcher = indexShard.acquireSearcher("can_match")) {
             final boolean aliasFilterCanMatch = request.getAliasFilter()
                 .getQueryBuilder() instanceof MatchNoneQueryBuilder == false;
             QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), searcher,
diff --git a/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
index d219c2c5621f4..5f719836a4a3c 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.search.internal;
 
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.search.RescoreDocIds;
@@ -32,9 +34,12 @@ public class LegacyReaderContext extends ReaderContext {
     private AggregatedDfs aggregatedDfs;
     private RescoreDocIds rescoreDocIds;
 
-    public LegacyReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher,
+    private Engine.Searcher searcher;
+    private Releasable onClose;
+
+    public LegacyReaderContext(long id, IndexShard indexShard, Engine.SearcherSupplier reader,
                                ShardSearchRequest shardSearchRequest, long keepAliveInMillis) {
-        super(id, indexShard, engineSearcher, keepAliveInMillis, false);
+        super(id, indexShard, reader, keepAliveInMillis, false);
         assert shardSearchRequest.readerId() == null;
         assert shardSearchRequest.keepAlive() == null;
         this.shardSearchRequest = Objects.requireNonNull(shardSearchRequest);
@@ -45,6 +50,30 @@ public LegacyReaderContext(long id, IndexShard indexShard, Engine.Searcher engin
         }
     }
 
+    @Override
+    public Engine.Searcher acquireSearcher(String source) {
+        if (scrollContext != null && "search".equals(source)) {
+            // Search scroll requests are special, they don't hold indices names so we have
+            // to reuse the searcher created on the request that initialized the scroll.
+            // This ensures that we wrap the searcher's reader with the user's permissions
+            // when they are available.
+            if (searcher == null) {
+                Engine.Searcher delegate = searcherSupplier.acquireSearcher(source);
+                onClose = delegate::close;
+                searcher = new Engine.Searcher(delegate.source(), delegate.getDirectoryReader(),
+                    delegate.getSimilarity(), delegate.getQueryCache(), delegate.getQueryCachingPolicy(), () -> {});
+            }
+            return searcher;
+        }
+        return super.acquireSearcher(source);
+    }
+
+    @Override
+    protected void closeInternal() {
+        Releasables.close(onClose);
+        super.closeInternal();
+    }
+
     @Override
     public ShardSearchRequest getShardSearchRequest(ShardSearchRequest other) {
         return shardSearchRequest;
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
index eb94b8f91e615..37593e7c172f3 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
@@ -28,7 +28,9 @@
 import org.elasticsearch.search.RescoreDocIds;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -47,7 +49,7 @@
 public class ReaderContext extends AbstractRefCounted implements Releasable {
     private final SearchContextId id;
     private final IndexShard indexShard;
-    private final Engine.Searcher engineSearcher;
+    protected final Engine.SearcherSupplier searcherSupplier;
     private final AtomicBoolean closed = new AtomicBoolean(false);
     private final boolean singleSession;
 
@@ -57,11 +59,19 @@ public class ReaderContext extends AbstractRefCounted implements Releasable {
 
     private final List<Releasable> onCloses = new CopyOnWriteArrayList<>();
 
-    public ReaderContext(long id, IndexShard indexShard, Engine.Searcher engineSearcher, long keepAliveInMillis, boolean singleSession) {
+    private final long startTimeInNano = System.nanoTime();
+
+    private Map<String, Object> context;
+
+    public ReaderContext(long id,
+                         IndexShard indexShard,
+                         Engine.SearcherSupplier searcherSupplier,
+                         long keepAliveInMillis,
+                         boolean singleSession) {
         super("reader_context");
         this.id = new SearchContextId(UUIDs.base64UUID(), id);
         this.indexShard = indexShard;
-        this.engineSearcher = engineSearcher;
+        this.searcherSupplier = searcherSupplier;
         this.singleSession = singleSession;
         this.keepAlive = new AtomicLong(keepAliveInMillis);
         this.lastAccessTime = new AtomicLong(nowInMillis());
@@ -75,14 +85,12 @@ private long nowInMillis() {
     public final void close() {
         if (closed.compareAndSet(false, true)) {
             decRef();
-        } else {
-            assert false : "ReaderContext was closed already";
         }
     }
 
     @Override
     protected void closeInternal() {
-        Releasables.close(Releasables.wrap(onCloses), engineSearcher);
+        Releasables.close(Releasables.wrap(onCloses), searcherSupplier);
     }
 
     public void addOnClose(Releasable releasable) {
@@ -93,16 +101,13 @@ public SearchContextId id() {
         return id;
     }
 
+
     public IndexShard indexShard() {
         return indexShard;
     }
 
-    public Engine.Searcher engineSearcher() {
-        return engineSearcher;
-    }
-
-    public String source() {
-        return engineSearcher.source();
+    public Engine.Searcher acquireSearcher(String source) {
+        return searcherSupplier.acquireSearcher(source);
     }
 
     public void keepAlive(long keepAlive) {
@@ -163,4 +168,27 @@ public void setRescoreDocIds(RescoreDocIds rescoreDocIds) {
     public boolean singleSession() {
         return singleSession;
     }
+
+    /**
+     * Returns the object or <code>null</code> if the given key does not have a
+     * value in the context
+     */
+    @SuppressWarnings("unchecked") // (T)object
+    public <T> T getFromContext(String key) {
+        return context != null ? (T) context.get(key) : null;
+    }
+
+    /**
+     * Puts the object into the context
+     */
+    public void putInContext(String key, Object value) {
+        if (context == null) {
+            context = new HashMap<>();
+        }
+        context.put(key, value);
+    }
+
+    public long getStartTimeInNano() {
+        return startTimeInNano;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java b/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
index 440408646d127..5b9c632d4e522 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
@@ -23,42 +23,10 @@
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.search.Scroll;
 
-import java.util.HashMap;
-import java.util.Map;
-
 /** Wrapper around information that needs to stay around when scrolling. */
 public final class ScrollContext {
-
-    private Map<String, Object> context = null;
-
     public TotalHits totalHits = null;
     public float maxScore = Float.NaN;
     public ScoreDoc lastEmittedDoc;
     public Scroll scroll;
-
-    private final long startTimeInNano = System.nanoTime();
-
-    /**
-     * Returns the object or <code>null</code> if the given key does not have a
-     * value in the context
-     */
-    @SuppressWarnings("unchecked") // (T)object
-    public <T> T getFromContext(String key) {
-        return context != null ? (T) context.get(key) : null;
-    }
-
-    /**
-     * Puts the object into the context
-     */
-    public void putInContext(String key, Object value) {
-        if (context == null) {
-            context = new HashMap<>();
-        }
-        context.put(key, value);
-    }
-
-
-    public long getStartTimeInNano() {
-        return startTimeInNano;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 47827350d2713..b6733c325fa13 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -82,9 +82,7 @@ public abstract class SearchContext implements Releasable {
     private final AtomicBoolean closed = new AtomicBoolean(false);
     private InnerHitsContext innerHitsContext;
 
-    protected SearchContext() {
-
-    }
+    protected SearchContext() {}
 
     public abstract void setTask(SearchShardTask task);
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index 254c0ab300e6a..5b16937749516 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -276,7 +276,7 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce
             out.writeBoolean(canReturnNullResponseIfMatchNoDocs);
             out.writeOptionalWriteable(bottomSortValues);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_8_0_0) && asKey == false) {
             out.writeOptionalWriteable(readerId);
             out.writeOptionalTimeValue(keepAlive);
         }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
index 34c528dfeda6a..10db9a7f80421 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.index.shard;
 
 import org.elasticsearch.search.internal.ReaderContext;
-import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
@@ -105,20 +104,21 @@ public void onFreeReaderContext(ReaderContext readerContext) {
             }
 
             @Override
-            public void onNewScrollContext(ScrollContext scrollContext) {
-                assertNotNull(scrollContext);
+            public void onNewScrollContext(ReaderContext readerContext) {
+                assertNotNull(readerContext);
                 newScrollContext.incrementAndGet();
             }
 
             @Override
-            public void onFreeScrollContext(ScrollContext scrollContext) {
-                assertNotNull(scrollContext);
+            public void onFreeScrollContext(ReaderContext readerContext) {
+                assertNotNull(readerContext);
                 freeScrollContext.incrementAndGet();
             }
 
             @Override
-            public void validateSearchContext(SearchContext context, TransportRequest request) {
-                assertNotNull(context);
+            public void validateSearchContext(ReaderContext readerContext, SearchContext searchContext, TransportRequest request) {
+                assertNotNull(readerContext);
+                assertNotNull(searchContext);
                 validateSearchContext.incrementAndGet();
             }
         };
@@ -232,7 +232,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onNewScrollContext(new ScrollContext());
+        compositeListener.onNewScrollContext(mock(ReaderContext.class));
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -258,7 +258,7 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, freeScrollContext.get());
         assertEquals(0, validateSearchContext.get());
 
-        compositeListener.onFreeScrollContext(new ScrollContext());
+        compositeListener.onFreeScrollContext(mock(ReaderContext.class));
         assertEquals(2, preFetch.get());
         assertEquals(2, preQuery.get());
         assertEquals(2, failedFetch.get());
@@ -272,10 +272,10 @@ public void validateSearchContext(SearchContext context, TransportRequest reques
         assertEquals(0, validateSearchContext.get());
 
         if (throwingListeners == 0) {
-            compositeListener.validateSearchContext(ctx, Empty.INSTANCE);
+            compositeListener.validateSearchContext(mock(ReaderContext.class), ctx, Empty.INSTANCE);
         } else {
-            RuntimeException expected =
-                expectThrows(RuntimeException.class, () -> compositeListener.validateSearchContext(ctx, Empty.INSTANCE));
+            RuntimeException expected = expectThrows(RuntimeException.class,
+                () -> compositeListener.validateSearchContext(mock(ReaderContext.class), ctx, Empty.INSTANCE));
             assertNull(expected.getMessage());
             assertEquals(throwingListeners - 1, expected.getSuppressed().length);
             if (throwingListeners > 1) {
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 38f790cb50c7a..de96b084f3e6c 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -60,7 +60,9 @@
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.io.IOException;
 import java.util.UUID;
+import java.util.function.Function;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.anyObject;
@@ -115,17 +117,28 @@ public void testPreProcess() throws Exception {
         BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
 
         try (Directory dir = newDirectory();
-             RandomIndexWriter w = new RandomIndexWriter(random(), dir);
-             IndexReader reader = w.getReader()) {
-
-            final Engine.Searcher engineSearcher = new Engine.Searcher("test", w.getReader(),
-                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(),
-                IndexSearcher.getDefaultQueryCachingPolicy(), reader);
+             RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
+
+            final Engine.SearcherSupplier engineReader = new Engine.SearcherSupplier(Function.identity()) {
+                @Override
+                public void close() {}
+
+                @Override
+                protected Engine.Searcher acquireSearcherInternal(String source) {
+                    try {
+                        IndexReader reader = w.getReader();
+                        return new Engine.Searcher("test", reader, IndexSearcher.getDefaultSimilarity(),
+                            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), reader);
+                    } catch (IOException exc) {
+                        throw new AssertionError(exc);
+                    }
+                }
+            };
 
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
             ReaderContext readerWithoutScroll = new ReaderContext(
-                randomNonNegativeLong(), indexShard, engineSearcher, randomNonNegativeLong(), false);
+                randomNonNegativeLong(), indexShard, engineReader, randomNonNegativeLong(), false);
             DefaultSearchContext contextWithoutScroll = new DefaultSearchContext(readerWithoutScroll, shardSearchRequest, target, null,
                 indexService, indexShard, bigArrays, null, timeout, null, false);
             contextWithoutScroll.from(300);
@@ -141,7 +154,7 @@ public void testPreProcess() throws Exception {
             // resultWindow greater than maxResultWindow and scrollContext isn't null
             when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMillis(randomInt(1000))));
             ReaderContext readerContext = new LegacyReaderContext(
-                randomNonNegativeLong(), indexShard, engineSearcher, shardSearchRequest, randomNonNegativeLong());
+                randomNonNegativeLong(), indexShard, engineReader, shardSearchRequest, randomNonNegativeLong());
             DefaultSearchContext context1 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null, indexService,
                 indexShard, bigArrays, null, timeout, null, false);
             context1.from(300);
@@ -173,7 +186,7 @@ public void testPreProcess() throws Exception {
                 + "] index level setting."));
 
             readerContext.close();
-            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, randomNonNegativeLong(), false);
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineReader, randomNonNegativeLong(), false);
             // rescore is null but sliceBuilder is not null
             DefaultSearchContext context2 = new DefaultSearchContext(readerContext, shardSearchRequest, target,
                 null, indexService, indexShard, bigArrays, null, timeout, null, false);
@@ -203,7 +216,7 @@ public void testPreProcess() throws Exception {
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
             readerContext.close();
-            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineSearcher, randomNonNegativeLong(), false);
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineReader, randomNonNegativeLong(), false);
             DefaultSearchContext context4 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
                 indexService, indexShard, bigArrays, null, timeout, null, false);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 11804fdca1ba1..0f9a7f7b55b4b 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -175,15 +175,6 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
         @Override
         public void onIndexModule(IndexModule indexModule) {
             indexModule.addSearchOperationListener(new SearchOperationListener() {
-                @Override
-                public void onNewReaderContext(ReaderContext readerContext) {
-                    if ("throttled_threadpool_index".equals(readerContext.indexShard().shardId().getIndex().getName())) {
-                        assertThat(Thread.currentThread().getName(), startsWith("elasticsearch[node_s_0][search_throttled]"));
-                    } else {
-                        assertThat(Thread.currentThread().getName(), startsWith("elasticsearch[node_s_0][search]"));
-                    }
-                }
-
                 @Override
                 public void onFetchPhase(SearchContext context, long tookInNanos) {
                     if ("throttled_threadpool_index".equals(context.indexShard().shardId().getIndex().getName())) {
@@ -522,7 +513,7 @@ public void testMaxOpenScrollContexts() throws Exception {
 
         final ShardScrollRequestTest request = new ShardScrollRequestTest(indexShard.shardId());
         ElasticsearchException ex = expectThrows(ElasticsearchException.class,
-            () -> service.createAndPutReaderContext(request, indexShard, indexShard.acquireSearcher("test"), randomBoolean()));
+            () -> service.createAndPutReaderContext(request, indexShard, indexShard.acquireSearcherSupplier(), randomBoolean()));
         assertEquals(
             "Trying to create too many scroll contexts. Must be less than or equal to: [" +
                 SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY) + "]. " +
@@ -545,11 +536,11 @@ public void testOpenScrollContextsConcurrently() throws Exception {
                 try {
                     latch.await();
                     for (; ; ) {
-                        Engine.Searcher searcher = indexShard.acquireSearcher("test");
+                        Engine.SearcherSupplier reader = indexShard.acquireSearcherSupplier();
                         try {
                             searchService.createAndPutReaderContext(
-                                new ShardScrollRequestTest(indexShard.shardId()), indexShard, searcher, true);
-                            searcher = null;
+                                new ShardScrollRequestTest(indexShard.shardId()), indexShard, reader, true);
+                            reader = null;
                         } catch (ElasticsearchException e) {
                             assertThat(e.getMessage(), equalTo(
                                 "Trying to create too many scroll contexts. Must be less than or equal to: " +
@@ -557,7 +548,7 @@ public void testOpenScrollContextsConcurrently() throws Exception {
                                     "This limit can be set by changing the [search.max_open_scroll_context] setting."));
                             return;
                         } finally {
-                            IOUtils.close(searcher);
+                            IOUtils.close(reader);
                         }
                     }
                 } catch (Exception e) {
@@ -992,10 +983,10 @@ public void testLookUpSearchContext() throws Exception {
                     ShardSearchRequest request = new ShardSearchRequest(
                         OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
                         indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
-                    final Engine.Searcher searcher = indexShard.acquireSearcher("test");
-                    final ReaderContext reader = searchService.createAndPutReaderContext(request, indexShard, searcher, randomBoolean());
-                    assertThat(reader.id().getId(), equalTo((long) (i + 1)));
-                    contextIds.add(reader.id());
+                    final ReaderContext context = searchService.createAndPutReaderContext(request, indexShard,
+                        indexShard.acquireSearcherSupplier(), randomBoolean());
+                    assertThat(context.id().getId(), equalTo((long) (i + 1)));
+                    contextIds.add(context.id());
                 }
                 assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
                 while (contextIds.isEmpty() == false) {
@@ -1031,7 +1022,6 @@ public void testOpenReaderContext() {
     }
 
     private ReaderContext createReaderContext(IndexShard shard) {
-        Engine.Searcher searcher = shard.acquireSearcher("test");
-        return new ReaderContext(randomNonNegativeLong(), shard, searcher, randomNonNegativeLong(), false);
+        return new ReaderContext(randomNonNegativeLong(), shard, shard.acquireSearcherSupplier(), randomNonNegativeLong(), false);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ScrollContextTests.java b/server/src/test/java/org/elasticsearch/search/internal/ScrollContextTests.java
deleted file mode 100644
index de4863dd92a08..0000000000000
--- a/server/src/test/java/org/elasticsearch/search/internal/ScrollContextTests.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.internal;
-
-import org.elasticsearch.test.ESTestCase;
-
-public class ScrollContextTests extends ESTestCase {
-
-    public void testStoringObjectsInScrollContext() {
-        final ScrollContext scrollContext = new ScrollContext();
-        final String key = randomAlphaOfLengthBetween(1, 16);
-        assertNull(scrollContext.getFromContext(key));
-
-        final String value = randomAlphaOfLength(6);
-        scrollContext.putInContext(key, value);
-
-        assertEquals(value, scrollContext.getFromContext(key));
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/MockInternalEngine.java b/test/framework/src/main/java/org/elasticsearch/test/engine/MockInternalEngine.java
index 42b7a4d8b102d..435feda33280a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/engine/MockInternalEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/engine/MockInternalEngine.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.index.engine.InternalEngine;
 
 import java.io.IOException;
+import java.util.function.Function;
 
 final class MockInternalEngine extends InternalEngine {
     private MockEngineSupport support;
@@ -81,4 +82,9 @@ public Engine.Searcher acquireSearcher(String source, SearcherScope scope) {
         final Engine.Searcher engineSearcher = super.acquireSearcher(source, scope);
         return support().wrapSearcher(engineSearcher);
     }
+
+    @Override
+    public SearcherSupplier acquireSearcherSupplier(Function<Searcher, Searcher> wrapper, SearcherScope scope) throws EngineException {
+        return super.acquireSearcherSupplier(wrapper.andThen(s -> support().wrapSearcher(s)), scope);
+    }
 }
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index 4cdf8c9c3af67..e99767b896db5 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -5,66 +5,37 @@
  */
 package org.elasticsearch.index.engine;
 
-import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.FilterDirectoryReader;
-import org.apache.lucene.index.FilterLeafReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.LeafMetaData;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.ReferenceManager;
-import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.shard.DocsStats;
-import org.elasticsearch.index.shard.SearchOperationListener;
-import org.elasticsearch.search.internal.ReaderContext;
-import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.index.store.Store;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.function.Function;
 
 /**
- * This is a stand-alone read-only engine that maintains a lazy loaded index reader that is opened on calls to
- * {@link Engine#acquireSearcher(String)}. The index reader opened is maintained until there are no reference to it anymore and then
- * releases itself from the engine. The readers returned from this engine are lazy which allows release after and reset before a search
- * phase starts. This allows releasing references as soon as possible on the search layer.
- *
- * Internally this class uses a set of wrapper abstractions to allow a reader that is used inside the {@link Engine.Searcher} returned from
- * {@link #acquireSearcher(String, SearcherScope)} to release and reset it's internal resources. This is necessary to for instance release
- * all SegmentReaders after a search phase finishes and reopen them before the next search phase starts. This together with a throttled
- * threadpool (search_throttled) guarantees that at most N frozen shards have a low level index reader open at the same time.
- *
- * In particular we have LazyDirectoryReader that wraps its LeafReaders (the actual segment readers) inside LazyLeafReaders. Each of the
- * LazyLeafReader delegates to segment LeafReader that can be reset (it's reference decremented and nulled out) on a search phase is
- * finished. Before the next search phase starts we can reopen the corresponding reader and reset the reference to execute the search phase.
- * This allows the SearchContext to hold on to the same LazyDirectoryReader across its lifecycle but under the hood resources (memory) is
- * released while the SearchContext phases are not executing.
- *
+ * This is a stand-alone read-only engine that maintains an index reader that is opened lazily on calls to
+ * {@link SearcherSupplier#acquireSearcher(String)}. The index reader opened is maintained until there are no reference to it anymore
+ * and then releases itself from the engine.
+ * This is necessary to for instance release all SegmentReaders after a search phase finishes and reopen them before the next search
+ * phase starts.
+ * This together with a throttled threadpool (search_throttled) guarantees that at most N frozen shards have a low level index reader
+ * open at the same time.
  * The internal reopen of readers is treated like a refresh and refresh listeners are called up-on reopen. This allows to consume refresh
  * stats in order to obtain the number of reopens.
  */
@@ -164,6 +135,11 @@ private synchronized void onReaderClosed(IndexReader.CacheKey key) {
         }
     }
 
+    @SuppressForbidden(reason = "we manage references explicitly here")
+    private synchronized void closeReader(IndexReader reader) throws IOException {
+        reader.decRef();
+    }
+
     private synchronized ElasticsearchDirectoryReader getOrOpenReader() throws IOException {
         ElasticsearchDirectoryReader reader = null;
         boolean success = false;
@@ -177,7 +153,7 @@ private synchronized ElasticsearchDirectoryReader getOrOpenReader() throws IOExc
                 reader = lastOpenedReader = wrapReader(dirReader, Function.identity());
                 processReader(reader);
                 reader.getReaderCacheHelper().addClosedListener(this::onReaderClosed);
-                for (ReferenceManager.RefreshListener listeners : config ().getInternalRefreshListener()) {
+                for (ReferenceManager.RefreshListener listeners : config().getInternalRefreshListener()) {
                     listeners.afterRefresh(true);
                 }
             }
@@ -199,404 +175,63 @@ private synchronized ElasticsearchDirectoryReader getReader() {
     }
 
     @Override
-    @SuppressWarnings("fallthrough")
-    @SuppressForbidden( reason = "we manage references explicitly here")
-    public Searcher acquireSearcher(String source, SearcherScope scope) throws EngineException {
+    public SearcherSupplier acquireSearcherSupplier(Function<Searcher, Searcher> wrapper, SearcherScope scope) throws EngineException {
+        final Store store = this.store;
         store.incRef();
-        boolean releaseRefeference = true;
-        try  {
-            final boolean maybeOpenReader;
-            switch (source) {
-                case "load_seq_no":
-                case "load_version":
-                    assert false : "this is a read-only engine";
-                case "doc_stats":
-                    assert false : "doc_stats are overwritten";
-                case "refresh_needed":
-                    assert false : "refresh_needed is always false";
-                case "segments":
-                case "segments_stats":
-                case "completion_stats":
-                case "can_match": // special case for can_match phase - we use the cached point values reader
-                    maybeOpenReader = false;
-                    break;
-                default:
-                    maybeOpenReader = true;
-            }
-            // special case we only want to report segment stats if we have a reader open. in that case we only get a reader if we still
-            // have one open at the time and can inc it's reference.
-            ElasticsearchDirectoryReader reader = maybeOpenReader ? getOrOpenReader() : getReader();
-            if (reader == null) {
-                // we just hand out a searcher on top of an empty reader that we opened for the ReadOnlyEngine in the #open(IndexCommit)
-                // method. this is the case when we don't have a reader open right now and we get a stats call any other that falls in
-                // the category that doesn't trigger a reopen
-                if ("can_match".equals(source)) {
-                    canMatchReader.incRef();
-                    return new Searcher(source, canMatchReader,
-                        engineConfig.getSimilarity(), engineConfig.getQueryCache(), engineConfig.getQueryCachingPolicy(),
-                        canMatchReader::decRef);
-                }
-                return super.acquireSearcher(source, scope);
-            } else {
-                try {
-                    LazyDirectoryReader lazyDirectoryReader = new LazyDirectoryReader(reader, this);
-                    Searcher newSearcher = new Searcher(source, lazyDirectoryReader,
-                        engineConfig.getSimilarity(), engineConfig.getQueryCache(), engineConfig.getQueryCachingPolicy(),
-                        () -> IOUtils.close(lazyDirectoryReader, store::decRef));
-                    releaseRefeference = false;
-                    return newSearcher;
-                } finally {
-                    if (releaseRefeference) {
-                        reader.decRef(); // don't call close here we manage reference ourselves
-                    }
-                }
-            }
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        } finally {
-            if (releaseRefeference) {
-                store.decRef();
-            }
-        }
-    }
-
-    static LazyDirectoryReader unwrapLazyReader(DirectoryReader reader) {
-        while (reader instanceof FilterDirectoryReader) {
-            if (reader instanceof LazyDirectoryReader) {
-                return (LazyDirectoryReader) reader;
-            }
-            reader = ((FilterDirectoryReader) reader).getDelegate();
-        }
-        return null;
-    }
-
-    /*
-     * We register this listener for a frozen index that will
-     *  1. reset the reader every time the search context is validated which happens when the context is looked up ie. on a fetch phase
-     *  etc.
-     *  2. register a releasable resource that is cleaned after each phase that releases the reader for this searcher
-     */
-    public static class ReacquireEngineSearcherListener implements SearchOperationListener {
-
-        @Override
-        public void validateSearchContext(SearchContext context, TransportRequest transportRequest) {
-            DirectoryReader dirReader = context.searcher().getDirectoryReader();
-            LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
-            if (lazyDirectoryReader != null) {
-                try {
-                    lazyDirectoryReader.reset();
-                } catch (IOException e) {
-                    throw new UncheckedIOException(e);
-                }
-            }
-        }
-
-        @Override
-        public void onFreeSearchContext(SearchContext context) {
-            DirectoryReader dirReader = context.searcher().getDirectoryReader();
-            LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
-            if (lazyDirectoryReader != null) {
+        return new SearcherSupplier(wrapper) {
+            @Override
+            @SuppressForbidden(reason = "we manage references explicitly here")
+            public Searcher acquireSearcherInternal(String source) {
                 try {
-                    lazyDirectoryReader.release();
-                } catch (IOException e) {
-                    throw new UncheckedIOException(e);
-                }
-            }
-        }
-
-        @Override
-        public void onNewReaderContext(ReaderContext readerContext) {
-            DirectoryReader dirReader = readerContext.engineSearcher().getDirectoryReader();
-            LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
-            if (lazyDirectoryReader != null) {
-                readerContext.addOnClose(() -> {
-                    try {
-                        lazyDirectoryReader.release();
-                    } catch (IOException e) {
-                        throw new UncheckedIOException(e);
-                    }
-                });
-            }
-        }
-    }
-
-    /**
-     * This class allows us to use the same high level reader across multiple search phases but replace the underpinnings
-     * on/after each search phase. This is really important otherwise we would hold on to multiple readers across phases.
-     *
-     * This reader and its leaf reader counterpart overrides FilterDirectory/LeafReader for convenience to be unwrapped but still
-     * overrides all it's delegate methods. We have tests to ensure we never miss an override but we need to in order to make sure
-     * the wrapper leaf readers don't register themself as close listeners on the wrapped ones otherwise we fail plugging in new readers
-     * on the next search phase.
-     */
-    static final class LazyDirectoryReader extends FilterDirectoryReader {
-
-        private final FrozenEngine engine;
-        private volatile DirectoryReader delegate; // volatile since it might be closed concurrently
-
-        private LazyDirectoryReader(DirectoryReader reader, FrozenEngine engine) throws IOException {
-            super(reader, new SubReaderWrapper() {
-                @Override
-                public LeafReader wrap(LeafReader reader) {
-                    return new LazyLeafReader(reader);
-                }
-            });
-            this.delegate = reader;
-            this.engine = engine;
-        }
-
-        @SuppressForbidden(reason = "we manage references explicitly here")
-        synchronized void release() throws IOException {
-            if (delegate != null) { // we are lenient here it's ok to double close
-                delegate.decRef();
-                delegate = null;
-                if (tryIncRef()) { // only do this if we are not closed already
-                    // we end up in this case when we are not closed but in an intermediate
-                    // state were we want to release all or the real leaf readers ie. in between search phases
-                    // but still want to keep this Lazy reference open. In oder to let the heavy real leaf
-                    // readers to be GCed we need to null our the references.
-                    try {
-                        for (LeafReaderContext leaf : leaves()) {
-                            LazyLeafReader reader = (LazyLeafReader) leaf.reader();
-                            reader.in = null;
-                        }
-                    } finally {
-                        decRef();
-                    }
-                }
-            }
-        }
-
-        void reset() throws IOException {
-            boolean success = false;
-            DirectoryReader reader = engine.getOrOpenReader();
-            try {
-                reset(reader);
-                success = true;
-            } finally {
-                if (success == false) {
-                    IOUtils.close(reader);
+                    return openSearcher(source, scope);
+                } catch (IOException exc) {
+                    throw new UncheckedIOException(exc);
                 }
             }
-        }
-
-        private synchronized void reset(DirectoryReader delegate) {
-            if (this.delegate != null) {
-                throw new AssertionError("lazy reader is not released");
-            }
-            assert (delegate instanceof LazyDirectoryReader) == false : "must not be a LazyDirectoryReader";
-            List<LeafReaderContext> leaves = delegate.leaves();
-            int ord = 0;
-            for (LeafReaderContext leaf : leaves()) {
-                LazyLeafReader reader = (LazyLeafReader) leaf.reader();
-                LeafReader newReader = leaves.get(ord++).reader();
-                assert reader.in == null;
-                reader.in = newReader;
-                assert reader.info.info.equals(Lucene.segmentReader(newReader).getSegmentInfo().info);
-            }
-            this.delegate = delegate;
-        }
 
-        @Override
-        protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) {
-            throw new UnsupportedOperationException();
-        }
-
-        void ensureOpenOrReset() {
-            // ensure we fail early and with good exceptions
-            ensureOpen();
-            if (delegate == null) {
-                throw new AlreadyClosedException("delegate is released");
+            @Override
+            public void close() {
+                store.decRef();
             }
-        }
-
-        @Override
-        public long getVersion() {
-            ensureOpenOrReset();
-            return delegate.getVersion();
-        }
-
-        @Override
-        public boolean isCurrent() throws IOException {
-            ensureOpenOrReset();
-            return delegate.isCurrent();
-        }
-
-        @Override
-        public IndexCommit getIndexCommit() throws IOException {
-            ensureOpenOrReset();
-            return delegate.getIndexCommit();
-        }
-
-        @Override
-        protected void doClose() throws IOException {
-            release();
-        }
-
-        @Override
-        public CacheHelper getReaderCacheHelper() {
-            ensureOpenOrReset();
-            return delegate.getReaderCacheHelper();
-        }
-
-        @Override
-        public DirectoryReader getDelegate() {
-            ensureOpenOrReset();
-            return delegate;
-        }
+        };
     }
 
-    /**
-     * We basically duplicate a FilterLeafReader here since we don't want the
-     * incoming reader to register with this reader as a parent reader. This would mean we barf if the incoming
-     * reader is closed and that is what we actually doing on purpose.
-     */
-    static final class LazyLeafReader extends FilterLeafReader {
-
-        private volatile LeafReader in;
-        private final SegmentCommitInfo info;
-        private final int numDocs;
-        private final int maxDocs;
-
-        private LazyLeafReader(LeafReader in) {
-            super(Lucene.emptyReader(in.maxDoc())); // empty reader here to make FilterLeafReader happy
-            this.info = Lucene.segmentReader(in).getSegmentInfo();
-            this.in = in;
-            numDocs = in.numDocs();
-            maxDocs = in.maxDoc();
-            // don't register in reader as a subreader here.
-        }
-
-        private void ensureOpenOrReleased() {
-            ensureOpen();
-            if (in == null) {
-                throw new AlreadyClosedException("leaf is already released");
+    @SuppressWarnings("fallthrough")
+    @SuppressForbidden(reason = "we manage references explicitly here")
+    private Engine.Searcher openSearcher(String source, SearcherScope scope) throws IOException {
+        boolean maybeOpenReader;
+        switch (source) {
+            case "load_seq_no":
+            case "load_version":
+                assert false : "this is a read-only engine";
+            case "doc_stats":
+                assert false : "doc_stats are overwritten";
+            case "refresh_needed":
+                assert false : "refresh_needed is always false";
+            case "segments":
+            case "segments_stats":
+            case "completion_stats":
+            case "can_match": // special case for can_match phase - we use the cached point values reader
+                maybeOpenReader = false;
+                break;
+            default:
+                maybeOpenReader = true;
+        }
+        ElasticsearchDirectoryReader reader = maybeOpenReader ? getOrOpenReader() : getReader();
+        if (reader == null) {
+            if ("can_match".equals(source)) {
+                canMatchReader.incRef();
+                return new Searcher(source, canMatchReader, engineConfig.getSimilarity(), engineConfig.getQueryCache(),
+                    engineConfig.getQueryCachingPolicy(), canMatchReader::decRef);
+            } else {
+                ReferenceManager<ElasticsearchDirectoryReader> manager = getReferenceManager(scope);
+                ElasticsearchDirectoryReader acquire = manager.acquire();
+                return new Searcher(source, acquire, engineConfig.getSimilarity(), engineConfig.getQueryCache(),
+                    engineConfig.getQueryCachingPolicy(), () -> manager.release(acquire));
             }
-        }
-
-        @Override
-        public Bits getLiveDocs() {
-            ensureOpenOrReleased();
-            return in.getLiveDocs();
-        }
-
-        @Override
-        public FieldInfos getFieldInfos() {
-            ensureOpenOrReleased();
-            return in.getFieldInfos();
-        }
-
-        @Override
-        public PointValues getPointValues(String field) throws IOException {
-            ensureOpenOrReleased();
-            return in.getPointValues(field);
-        }
-
-        @Override
-        public Fields getTermVectors(int docID)
-            throws IOException {
-            ensureOpenOrReleased();
-            return in.getTermVectors(docID);
-        }
-
-        @Override
-        public int numDocs() {
-            return numDocs;
-        }
-
-        @Override
-        public int maxDoc() {
-            return maxDocs;
-        }
-
-        @Override
-        public void document(int docID, StoredFieldVisitor visitor) throws IOException {
-            ensureOpenOrReleased();
-            in.document(docID, visitor);
-        }
-
-        @Override
-        protected void doClose() throws IOException {
-            in.close();
-        }
-
-        @Override
-        public CacheHelper getReaderCacheHelper() {
-            ensureOpenOrReleased();
-            return in.getReaderCacheHelper();
-        }
-
-        @Override
-        public CacheHelper getCoreCacheHelper() {
-            ensureOpenOrReleased();
-            return in.getCoreCacheHelper();
-        }
-
-        @Override
-        public Terms terms(String field) throws IOException {
-            ensureOpenOrReleased();
-            return in.terms(field);
-        }
-
-        @Override
-        public String toString() {
-            final StringBuilder buffer = new StringBuilder("LazyLeafReader(");
-            buffer.append(in);
-            buffer.append(')');
-            return buffer.toString();
-        }
-
-        @Override
-        public NumericDocValues getNumericDocValues(String field) throws IOException {
-            ensureOpenOrReleased();
-            return in.getNumericDocValues(field);
-        }
-
-        @Override
-        public BinaryDocValues getBinaryDocValues(String field) throws IOException {
-            ensureOpenOrReleased();
-            return in.getBinaryDocValues(field);
-        }
-
-        @Override
-        public SortedDocValues getSortedDocValues(String field) throws IOException {
-            ensureOpenOrReleased();
-            return in.getSortedDocValues(field);
-        }
-
-        @Override
-        public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
-            ensureOpenOrReleased();
-            return in.getSortedNumericDocValues(field);
-        }
-
-        @Override
-        public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
-            ensureOpenOrReleased();
-            return in.getSortedSetDocValues(field);
-        }
-
-        @Override
-        public NumericDocValues getNormValues(String field) throws IOException {
-            ensureOpenOrReleased();
-            return in.getNormValues(field);
-        }
-
-        @Override
-        public LeafMetaData getMetaData() {
-            ensureOpenOrReleased();
-            return in.getMetaData();
-        }
-
-        @Override
-        public void checkIntegrity() throws IOException {
-            ensureOpenOrReleased();
-            in.checkIntegrity();
-        }
-
-        @Override
-        public LeafReader getDelegate() {
-            return in;
+        } else {
+            return new Searcher(source, reader, engineConfig.getSimilarity(), engineConfig.getQueryCache(),
+                engineConfig.getQueryCachingPolicy(), () -> closeReader(reader));
         }
     }
 
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java
index c2fb11d294dec..7c5e29b03b6ed 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
-import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.engine.FrozenEngine;
@@ -55,14 +54,6 @@ public List<Setting<?>> getSettings() {
         return Arrays.asList(FrozenEngine.INDEX_FROZEN);
     }
 
-    @Override
-    public void onIndexModule(IndexModule indexModule) {
-        if (FrozenEngine.INDEX_FROZEN.get(indexModule.getSettings())) {
-            indexModule.addSearchOperationListener(new FrozenEngine.ReacquireEngineSearcherListener());
-        }
-        super.onIndexModule(indexModule);
-    }
-
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> actions = new ArrayList<>();
diff --git a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java
index 9048983447781..0d505880575a9 100644
--- a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java
+++ b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java
@@ -11,7 +11,6 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
@@ -32,6 +31,7 @@
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -50,22 +50,25 @@ public void testAcquireReleaseReset() throws IOException {
                 listener.reset();
                 try (FrozenEngine frozenEngine = new FrozenEngine(engine.engineConfig, true)) {
                     assertFalse(frozenEngine.isReaderOpen());
-                    Engine.Searcher searcher = frozenEngine.acquireSearcher("test");
-                    assertEquals(config.getShardId(), ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(searcher
-                        .getDirectoryReader()).shardId());
-                    assertTrue(frozenEngine.isReaderOpen());
-                    TopDocs search = searcher.search(new MatchAllDocsQuery(), numDocs);
-                    assertEquals(search.scoreDocs.length, numDocs);
-                    assertEquals(1, listener.afterRefresh.get());
-                    FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).release();
-                    assertFalse(frozenEngine.isReaderOpen());
-                    assertEquals(1, listener.afterRefresh.get());
-                    expectThrows(AlreadyClosedException.class, () -> searcher.search(new MatchAllDocsQuery(), numDocs));
-                    FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).reset();
-                    assertEquals(2, listener.afterRefresh.get());
-                    search = searcher.search(new MatchAllDocsQuery(), numDocs);
-                    assertEquals(search.scoreDocs.length, numDocs);
-                    searcher.close();
+                    try (Engine.SearcherSupplier reader = frozenEngine.acquireSearcherSupplier(Function.identity())) {
+                        assertFalse(frozenEngine.isReaderOpen());
+                        try (Engine.Searcher searcher = reader.acquireSearcher("frozen")) {
+                            assertEquals(config.getShardId(), ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(searcher
+                                .getDirectoryReader()).shardId());
+                            assertTrue(frozenEngine.isReaderOpen());
+                            TopDocs search = searcher.search(new MatchAllDocsQuery(), numDocs);
+                            assertEquals(search.scoreDocs.length, numDocs);
+                            assertEquals(1, listener.afterRefresh.get());
+                        }
+                        assertFalse(frozenEngine.isReaderOpen());
+                        assertEquals(1, listener.afterRefresh.get());
+
+                        try (Engine.Searcher searcher = reader.acquireSearcher("frozen")) {
+                            assertTrue(frozenEngine.isReaderOpen());
+                            TopDocs search = searcher.search(new MatchAllDocsQuery(), numDocs);
+                            assertEquals(search.scoreDocs.length, numDocs);
+                        }
+                    }
                 }
             }
         }
@@ -84,24 +87,31 @@ public void testAcquireReleaseResetTwoSearchers() throws IOException {
                 listener.reset();
                 try (FrozenEngine frozenEngine = new FrozenEngine(engine.engineConfig, true)) {
                     assertFalse(frozenEngine.isReaderOpen());
-                    Engine.Searcher searcher1 = frozenEngine.acquireSearcher("test");
-                    assertTrue(frozenEngine.isReaderOpen());
-                    TopDocs search = searcher1.search(new MatchAllDocsQuery(), numDocs);
-                    assertEquals(search.scoreDocs.length, numDocs);
-                    assertEquals(1, listener.afterRefresh.get());
-                    FrozenEngine.unwrapLazyReader(searcher1.getDirectoryReader()).release();
-                    Engine.Searcher searcher2 = frozenEngine.acquireSearcher("test");
-                    search = searcher2.search(new MatchAllDocsQuery(), numDocs);
-                    assertEquals(search.scoreDocs.length, numDocs);
-                    assertTrue(frozenEngine.isReaderOpen());
-                    assertEquals(2, listener.afterRefresh.get());
-                    expectThrows(AlreadyClosedException.class, () -> searcher1.search(new MatchAllDocsQuery(), numDocs));
-                    FrozenEngine.unwrapLazyReader(searcher1.getDirectoryReader()).reset();
+                    Engine.SearcherSupplier reader1 = frozenEngine.acquireSearcherSupplier(Function.identity());
+                    try (Engine.Searcher searcher1 = reader1.acquireSearcher("test")) {
+                        assertTrue(frozenEngine.isReaderOpen());
+                        TopDocs search = searcher1.search(new MatchAllDocsQuery(), numDocs);
+                        assertEquals(search.scoreDocs.length, numDocs);
+                        assertEquals(1, listener.afterRefresh.get());
+                    }
+                    assertFalse(frozenEngine.isReaderOpen());
+                    Engine.SearcherSupplier reader2 = frozenEngine.acquireSearcherSupplier(Function.identity());
+                    try (Engine.Searcher searcher2 = reader2.acquireSearcher("test")) {
+                        TopDocs search = searcher2.search(new MatchAllDocsQuery(), numDocs);
+                        assertEquals(search.scoreDocs.length, numDocs);
+                        assertTrue(frozenEngine.isReaderOpen());
+                        assertEquals(2, listener.afterRefresh.get());
+                    }
+                    assertFalse(frozenEngine.isReaderOpen());
                     assertEquals(2, listener.afterRefresh.get());
-                    search = searcher1.search(new MatchAllDocsQuery(), numDocs);
-                    assertEquals(search.scoreDocs.length, numDocs);
-                    searcher1.close();
-                    searcher2.close();
+                    reader2.close();
+                    try (Engine.Searcher searcher1 = reader1.acquireSearcher("test")) {
+                        TopDocs search = searcher1.search(new MatchAllDocsQuery(), numDocs);
+                        assertEquals(search.scoreDocs.length, numDocs);
+                        assertTrue(frozenEngine.isReaderOpen());
+                    }
+                    reader1.close();
+                    assertFalse(frozenEngine.isReaderOpen());
                 }
             }
         }
@@ -119,21 +129,24 @@ public void testSegmentStats() throws IOException {
                 engine.flushAndClose();
                 listener.reset();
                 try (FrozenEngine frozenEngine = new FrozenEngine(engine.engineConfig, true)) {
-                    Engine.Searcher searcher = frozenEngine.acquireSearcher("test");
-                    SegmentsStats segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false);
-                    assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount());
-                    FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).release();
-                    assertEquals(1, listener.afterRefresh.get());
-                    segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false);
-                    assertEquals(0, segmentsStats.getCount());
-                    segmentsStats = frozenEngine.segmentsStats(randomBoolean(), true);
-                    assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount());
-                    assertEquals(1, listener.afterRefresh.get());
-                    assertFalse(frozenEngine.isReaderOpen());
-                    FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).reset();
-                    segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false);
-                    assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount());
-                    searcher.close();
+                    try (Engine.SearcherSupplier reader = frozenEngine.acquireSearcherSupplier(Function.identity())) {
+                        SegmentsStats segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false);
+                        try (Engine.Searcher searcher = reader.acquireSearcher("test")) {
+                            segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false);
+                            assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount());
+                            assertEquals(1, listener.afterRefresh.get());
+                        }
+                        segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false);
+                        assertEquals(0, segmentsStats.getCount());
+                        try (Engine.Searcher searcher = reader.acquireSearcher("test")) {
+                            segmentsStats = frozenEngine.segmentsStats(randomBoolean(), true);
+                            assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount());
+                            assertEquals(2, listener.afterRefresh.get());
+                        }
+                        assertFalse(frozenEngine.isReaderOpen());
+                        segmentsStats = frozenEngine.segmentsStats(randomBoolean(), true);
+                        assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount());
+                    }
                 }
             }
         }
@@ -165,16 +178,18 @@ public void testCircuitBreakerAccounting() throws IOException {
             assertEquals(0, breaker.getUsed());
             listener.reset();
             try (FrozenEngine frozenEngine = new FrozenEngine(config, true)) {
-                Engine.Searcher searcher = frozenEngine.acquireSearcher("test");
-                assertEquals(expectedUse, breaker.getUsed());
-                FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).release();
-                assertEquals(1, listener.afterRefresh.get());
-                assertEquals(0, breaker.getUsed());
-                assertFalse(frozenEngine.isReaderOpen());
-                FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).reset();
-                assertEquals(expectedUse, breaker.getUsed());
-                searcher.close();
-                assertEquals(0, breaker.getUsed());
+                try (Engine.SearcherSupplier reader = frozenEngine.acquireSearcherSupplier(Function.identity())) {
+                    try (Engine.Searcher searcher = reader.acquireSearcher("test")) {
+                        assertEquals(expectedUse, breaker.getUsed());
+                    }
+                    assertEquals(1, listener.afterRefresh.get());
+                    assertEquals(0, breaker.getUsed());
+                    assertFalse(frozenEngine.isReaderOpen());
+                    try (Engine.Searcher searcher = reader.acquireSearcher("test")) {
+                        assertEquals(expectedUse, breaker.getUsed());
+                    }
+                    assertEquals(0, breaker.getUsed());
+                }
             }
         }
     }
@@ -216,18 +231,17 @@ public void testSearchConcurrently() throws IOException, InterruptedException {
                     CountDownLatch latch = new CountDownLatch(numThreads);
                     for (int i = 0; i < numThreads; i++) {
                         threads[i] = new Thread(() -> {
-                            try (Engine.Searcher searcher = frozenEngine.acquireSearcher("test")) {
+                            try (Engine.SearcherSupplier reader = frozenEngine.acquireSearcherSupplier(Function.identity())) {
                                 barrier.await();
-                                FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).release();
                                 for (int j = 0; j < numIters; j++) {
-                                    FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).reset();
-                                    assertTrue(frozenEngine.isReaderOpen());
-                                    TopDocs search = searcher.search(new MatchAllDocsQuery(), Math.min(10, numDocsAdded));
-                                    assertEquals(search.scoreDocs.length, Math.min(10, numDocsAdded));
-                                    FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).release();
+                                    try (Engine.Searcher searcher = reader.acquireSearcher("test")) {
+                                        assertTrue(frozenEngine.isReaderOpen());
+                                        TopDocs search = searcher.search(new MatchAllDocsQuery(), Math.min(10, numDocsAdded));
+                                        assertEquals(search.scoreDocs.length, Math.min(10, numDocsAdded));
+                                    }
                                 }
                                 if (randomBoolean()) {
-                                    FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).reset();
+                                    reader.acquireSearcher("test").close();
                                 }
                             } catch (Exception e) {
                                 throw new AssertionError(e);
@@ -267,12 +281,6 @@ private static void checkOverrideMethods(Class<?> clazz) throws NoSuchMethodExce
         }
     }
 
-    // here we make sure we catch any change to their super classes FilterLeafReader / FilterDirectoryReader
-    public void testOverrideMethods() throws Exception {
-        checkOverrideMethods(FrozenEngine.LazyDirectoryReader.class);
-        checkOverrideMethods(FrozenEngine.LazyLeafReader.class);
-    }
-
     private class CountingRefreshListener implements ReferenceManager.RefreshListener {
 
         final AtomicInteger afterRefresh = new AtomicInteger(0);
@@ -306,25 +314,27 @@ public void testCanMatch() throws IOException {
                 engine.flushAndClose();
                 listener.reset();
                 try (FrozenEngine frozenEngine = new FrozenEngine(engine.engineConfig, true)) {
-                    DirectoryReader reader;
-                    try (Engine.Searcher searcher = frozenEngine.acquireSearcher("can_match")) {
-                        assertNotNull(ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(searcher.getDirectoryReader()));
-                        assertEquals(config.getShardId(), ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(searcher
-                            .getDirectoryReader()).shardId());
-                        reader = searcher.getDirectoryReader();
-                        assertNotEquals(reader, Matchers.instanceOf(FrozenEngine.LazyDirectoryReader.class));
-                        assertEquals(0, listener.afterRefresh.get());
-                        DirectoryReader unwrap = FilterDirectoryReader.unwrap(searcher.getDirectoryReader());
-                        assertThat(unwrap, Matchers.instanceOf(RewriteCachingDirectoryReader.class));
-                        assertNotNull(ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(searcher.getDirectoryReader()));
+                    DirectoryReader dirReader;
+                    try (Engine.SearcherSupplier reader = frozenEngine.acquireSearcherSupplier(Function.identity())) {
+                        try (Engine.Searcher searcher = reader.acquireSearcher("can_match")) {
+                            dirReader = searcher.getDirectoryReader();
+                            assertNotNull(ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(searcher.getDirectoryReader()));
+                            assertEquals(config.getShardId(), ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(searcher
+                                .getDirectoryReader()).shardId());
+                            assertEquals(0, listener.afterRefresh.get());
+                            DirectoryReader unwrap = FilterDirectoryReader.unwrap(searcher.getDirectoryReader());
+                            assertThat(unwrap, Matchers.instanceOf(RewriteCachingDirectoryReader.class));
+                            assertNotNull(ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(searcher.getDirectoryReader()));
+                        }
                     }
 
-                    try (Engine.Searcher searcher = frozenEngine.acquireSearcher("can_match")) {
-                        assertSame(reader, searcher.getDirectoryReader());
-                        assertNotEquals(reader, Matchers.instanceOf(FrozenEngine.LazyDirectoryReader.class));
-                        assertEquals(0, listener.afterRefresh.get());
-                        DirectoryReader unwrap = FilterDirectoryReader.unwrap(searcher.getDirectoryReader());
-                        assertThat(unwrap, Matchers.instanceOf(RewriteCachingDirectoryReader.class));
+                    try (Engine.SearcherSupplier reader = frozenEngine.acquireSearcherSupplier(Function.identity())) {
+                        try (Engine.Searcher searcher = reader.acquireSearcher("can_match")) {
+                            assertSame(dirReader, searcher.getDirectoryReader());
+                            assertEquals(0, listener.afterRefresh.get());
+                            DirectoryReader unwrap = FilterDirectoryReader.unwrap(searcher.getDirectoryReader());
+                            assertThat(unwrap, Matchers.instanceOf(RewriteCachingDirectoryReader.class));
+                        }
                     }
                 }
             }
@@ -346,14 +356,18 @@ public void testSearchers() throws Exception {
                 // See TransportVerifyShardBeforeCloseAction#executeShardOperation
                 engine.flush(true, true);
                 engine.refresh("test");
-                try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
-                    totalDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE).scoreDocs.length;
+                try (Engine.SearcherSupplier reader = engine.acquireSearcherSupplier(Function.identity())) {
+                    try (Engine.Searcher searcher = reader.acquireSearcher("test")) {
+                        totalDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE).scoreDocs.length;
+                    }
                 }
             }
             try (FrozenEngine frozenEngine = new FrozenEngine(config, true)) {
-                try (Engine.Searcher searcher = frozenEngine.acquireSearcher("test")) {
-                    TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE);
-                    assertThat(topDocs.scoreDocs.length, equalTo(totalDocs));
+                try (Engine.SearcherSupplier reader = frozenEngine.acquireSearcherSupplier(Function.identity())) {
+                    try (Engine.Searcher searcher = reader.acquireSearcher("test")) {
+                        TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE);
+                        assertThat(topDocs.scoreDocs.length, equalTo(totalDocs));
+                    }
                 }
             }
         }
diff --git a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
index 27cd14576f8af..dd6d22342990c 100644
--- a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
+++ b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
@@ -11,9 +11,14 @@
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.ClearReaderAction;
+import org.elasticsearch.action.search.ClearReaderRequest;
+import org.elasticsearch.action.search.OpenReaderRequest;
+import org.elasticsearch.action.search.OpenReaderResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.search.TransportOpenReaderAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -65,7 +70,15 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(FrozenIndices.class);
     }
 
-    public void testCloseFreezeAndOpen() {
+    String openReaders(TimeValue keepAlive, String... indices) {
+        OpenReaderRequest request = new OpenReaderRequest(indices, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED,
+            keepAlive, null, null);
+        final OpenReaderResponse response = client()
+            .execute(TransportOpenReaderAction.INSTANCE, request).actionGet();
+        return response.getReaderId();
+    }
+
+    public void testCloseFreezeAndOpen() throws Exception {
         createIndex("index", Settings.builder().put("index.number_of_shards", 2).build());
         client().prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         client().prepareIndex("index").setId("2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
@@ -86,9 +99,7 @@ public void testCloseFreezeAndOpen() {
         assertEquals(useDFS ? 3 : 2, shard.refreshStats().getTotal());
         assertFalse(((FrozenEngine)engine).isReaderOpen());
         assertTrue(indexService.getIndexSettings().isSearchThrottled());
-        try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
-            assertNotNull(FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()));
-        }
+
         // now scroll
         SearchResponse searchResponse = client().prepareSearch().setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED)
             .setScroll(TimeValue.timeValueMinutes(1)).setSize(1).get();
@@ -100,13 +111,39 @@ public void testCloseFreezeAndOpen() {
             for (int i = 0; i < 2; i++) {
                 shard = indexService.getShard(i);
                 engine = IndexShardTestCase.getEngine(shard);
-                assertFalse(((FrozenEngine) engine).isReaderOpen());
+                // scrolls keep the reader open
+                assertTrue(((FrozenEngine) engine).isReaderOpen());
             }
             searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(1)).get();
         } while (searchResponse.getHits().getHits().length > 0);
+        client().prepareClearScroll().addScrollId(searchResponse.getScrollId()).get();
+
+        String readerId = openReaders( TimeValue.timeValueMinutes(1), "index");
+        try {
+            // now readerId
+            for (int from = 0; from < 3; from++) {
+                searchResponse = client().prepareSearch()
+                    .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED)
+                    .setReader(readerId, TimeValue.timeValueMinutes(1))
+                    .setSize(1)
+                    .setFrom(from)
+                    .get();
+                assertHitCount(searchResponse, 3);
+                assertEquals(1, searchResponse.getHits().getHits().length);
+                SearchService searchService = getInstanceFromNode(SearchService.class);
+                assertThat(searchService.getActiveContexts(), Matchers.greaterThanOrEqualTo(1));
+                for (int i = 0; i < 2; i++) {
+                    shard = indexService.getShard(i);
+                    engine = IndexShardTestCase.getEngine(shard);
+                    assertFalse(((FrozenEngine) engine).isReaderOpen());
+                }
+            }
+        } finally {
+            client().execute(ClearReaderAction.INSTANCE, new ClearReaderRequest(searchResponse.getReaderId())).get();
+        }
     }
 
-    public void testSearchAndGetAPIsAreThrottled() throws InterruptedException, IOException {
+    public void testSearchAndGetAPIsAreThrottled() throws IOException {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_doc")
             .startObject("properties").startObject("field").field("type", "text").field("term_vector", "with_positions_offsets_payloads")
             .endObject().endObject()
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
index 1cf5098b8bdd8..5c591bdeb4166 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.search.SearchContextMissingException;
-import org.elasticsearch.search.internal.ScrollContext;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.transport.TransportRequest;
@@ -24,13 +24,12 @@
 import static org.elasticsearch.xpack.security.authz.AuthorizationService.ORIGINATING_ACTION_KEY;
 
 /**
- * A {@link SearchOperationListener} that is used to provide authorization for scroll requests.
+ * A {@link SearchOperationListener} that is used to provide authorization for search requests.
  *
- * In order to identify the user associated with a scroll request, we replace the {@link ScrollContext}
- * on creation with a custom implementation that holds the {@link Authentication} object. When
- * this context is accessed again in {@link SearchOperationListener#onPreQueryPhase(SearchContext)}
- * the ScrollContext is inspected for the authentication, which is compared to the currently
- * authentication.
+ * In order to identify the user associated with a search request, we put the {@link Authentication}
+ * object in the {@link ReaderContext} on creation. When this context is accessed again in
+ * {@link SearchOperationListener#validateSearchContext} the ReaderContext is inspected for
+ * the authentication, which is compared to the currently authentication.
  */
 public final class SecuritySearchOperationListener implements SearchOperationListener {
 
@@ -44,36 +43,31 @@ public SecuritySearchOperationListener(SecurityContext securityContext, XPackLic
         this.auditTrailService = auditTrail;
     }
 
-    /**
-     * Adds the {@link Authentication} to the {@link ScrollContext}
-     */
     @Override
-    public void onNewScrollContext(ScrollContext scrollContext) {
+    public void onNewReaderContext(ReaderContext readerContext) {
         if (licenseState.isSecurityEnabled()) {
-            scrollContext.putInContext(AuthenticationField.AUTHENTICATION_KEY, securityContext.getAuthentication());
+            readerContext.putInContext(AuthenticationField.AUTHENTICATION_KEY, securityContext.getAuthentication());
         }
     }
 
     /**
-     * Checks for the {@link ScrollContext} if it exists and compares the {@link Authentication}
-     * object from the scroll context with the current authentication context
+     * compares the {@link Authentication} object from the reader context with the current
+     * authentication context
      */
     @Override
-    public void validateSearchContext(SearchContext searchContext, TransportRequest request) {
+    public void validateSearchContext(ReaderContext readerContext, SearchContext searchContext, TransportRequest request) {
         if (licenseState.isSecurityEnabled()) {
-            if (searchContext.scrollContext() != null) {
-                final Authentication originalAuth = searchContext.scrollContext().getFromContext(AuthenticationField.AUTHENTICATION_KEY);
-                final Authentication current = securityContext.getAuthentication();
-                final ThreadContext threadContext = securityContext.getThreadContext();
-                final String action = threadContext.getTransient(ORIGINATING_ACTION_KEY);
-                ensureAuthenticatedUserIsSame(originalAuth, current, auditTrailService, searchContext.id(), action, request,
-                        AuditUtil.extractRequestId(threadContext), threadContext.getTransient(AUTHORIZATION_INFO_KEY));
-            }
+            final Authentication originalAuth = readerContext.getFromContext(AuthenticationField.AUTHENTICATION_KEY);
+            final Authentication current = securityContext.getAuthentication();
+            final ThreadContext threadContext = securityContext.getThreadContext();
+            final String action = threadContext.getTransient(ORIGINATING_ACTION_KEY);
+            ensureAuthenticatedUserIsSame(originalAuth, current, auditTrailService, searchContext.id(), action, request,
+                AuditUtil.extractRequestId(threadContext), threadContext.getTransient(AUTHORIZATION_INFO_KEY));
         }
     }
 
     /**
-     * Compares the {@link Authentication} that was stored in the {@link ScrollContext} with the
+     * Compares the {@link Authentication} that was stored in the {@link ReaderContext} with the
      * current authentication. We cannot guarantee that all of the details of the authentication will
      * be the same. Some things that could differ include the roles, the name of the authenticating
      * (or lookup) realm. To work around this we compare the username and the originating realm type.
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
index 72d09aec8a140..87b25c5cf3f50 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
@@ -12,6 +12,8 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.search.ClearReaderAction;
+import org.elasticsearch.action.search.ClearReaderRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
@@ -63,6 +65,7 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.integration.FieldLevelSecurityTests.openReaders;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -759,6 +762,46 @@ public void testScroll() throws Exception {
         }
     }
 
+    public void testReaderId() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("test")
+            .setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true))
+            .setMapping("field1", "type=text", "field2", "type=text", "field3", "type=text")
+        );
+        final int numVisible = scaledRandomIntBetween(2, 10);
+        final int numInvisible = scaledRandomIntBetween(2, 10);
+        int id = 1;
+        for (int i = 0; i < numVisible; i++) {
+            client().prepareIndex("test").setId(String.valueOf(id++)).setSource("field1", "value1").get();
+        }
+
+        for (int i = 0; i < numInvisible; i++) {
+            client().prepareIndex("test").setId(String.valueOf(id++)).setSource("field2", "value2").get();
+            client().prepareIndex("test").setId(String.valueOf(id++)).setSource("field3", "value3").get();
+        }
+        refresh();
+
+        String readerId = openReaders("user1", TimeValue.timeValueMinutes(1), "test");
+        SearchResponse response = null;
+        try {
+            for (int from = 0; from < numVisible; from++) {
+                response = client()
+                    .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                    .prepareSearch()
+                    .setSize(1)
+                    .setFrom(from)
+                    .setReader(readerId, TimeValue.timeValueMinutes(1))
+                    .setQuery(termQuery("field1", "value1"))
+                    .get();
+                assertNoFailures(response);
+                assertThat(response.getHits().getTotalHits().value, is((long) numVisible));
+                assertThat(response.getHits().getAt(0).getSourceAsMap().size(), is(1));
+                assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
+            }
+        } finally {
+            client().execute(ClearReaderAction.INSTANCE, new ClearReaderRequest(response.getReaderId())).actionGet();
+        }
+    }
+
     public void testRequestCache() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test")
                 .setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true))
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
index a4fe2e9a2cee0..f7b1e227cdd3b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
@@ -11,8 +11,13 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.search.ClearReaderAction;
+import org.elasticsearch.action.search.ClearReaderRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
+import org.elasticsearch.action.search.OpenReaderRequest;
+import org.elasticsearch.action.search.OpenReaderResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.TransportOpenReaderAction;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
@@ -672,7 +677,8 @@ public void testMSearchApi() throws Exception {
 
     public void testScroll() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test")
-                .setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true))
+                .setSettings(Settings.builder()
+                    .put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true))
                 .setMapping("field1", "type=text", "field2", "type=text", "field3", "type=text")
         );
 
@@ -722,6 +728,51 @@ public void testScroll() throws Exception {
         }
     }
 
+    static String openReaders(String userName, TimeValue keepAlive, String... indices) {
+        OpenReaderRequest request = new OpenReaderRequest(indices, OpenReaderRequest.DEFAULT_INDICES_OPTIONS, keepAlive, null, null);
+        final OpenReaderResponse response = client()
+            .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue(userName, USERS_PASSWD)))
+            .execute(TransportOpenReaderAction.INSTANCE, request).actionGet();
+        return response.getReaderId();
+    }
+
+    public void testReaderId() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("test")
+            .setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true))
+            .setMapping("field1", "type=text", "field2", "type=text", "field3", "type=text")
+        );
+
+        final int numDocs = scaledRandomIntBetween(2, 10);
+        for (int i = 0; i < numDocs; i++) {
+            client().prepareIndex("test").setId(String.valueOf(i))
+                .setSource("field1", "value1", "field2", "value2", "field3", "value3")
+                .get();
+        }
+        refresh("test");
+
+        String readerId = openReaders("user1", TimeValue.timeValueMinutes(1), "test");
+        SearchResponse response = null;
+        try {
+            for (int from = 0; from < numDocs; from++) {
+                response = client()
+                    .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                    .prepareSearch()
+                    .setReader(readerId, TimeValue.timeValueMinutes(1L))
+                    .setSize(1)
+                    .setFrom(from)
+                    .setQuery(constantScoreQuery(termQuery("field1", "value1")))
+                    .setFetchSource(true)
+                    .get();
+                assertThat(response.getHits().getTotalHits().value, is((long) numDocs));
+                assertThat(response.getHits().getHits().length, is(1));
+                assertThat(response.getHits().getAt(0).getSourceAsMap().size(), is(1));
+                assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
+            }
+        } finally {
+            client().execute(ClearReaderAction.INSTANCE, new ClearReaderRequest(readerId)).actionGet();
+        }
+    }
+
     public void testQueryCache() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test")
                         .setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true))
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index 1e4cc478a0421..b390ccbb85f81 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -10,15 +10,17 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
+import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.license.XPackLicenseState.Feature;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContextId;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequest.Empty;
@@ -30,6 +32,7 @@
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
+import org.junit.Before;
 import org.mockito.Mockito;
 
 import java.util.Collections;
@@ -47,134 +50,145 @@
 import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
-public class SecuritySearchOperationListenerTests extends ESTestCase {
+public class SecuritySearchOperationListenerTests extends ESSingleNodeTestCase {
+    private IndexShard shard;
+
+    @Before
+    private void setupShard() {
+        shard = createIndex("index").getShard(0);
+    }
 
     public void testUnlicensed() {
-        XPackLicenseState licenseState = mock(XPackLicenseState.class);
-        when(licenseState.isSecurityEnabled()).thenReturn(false);
-        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
-        AuditTrailService auditTrailService = mock(AuditTrailService.class);
-        SearchContext searchContext = mock(SearchContext.class);
-        ScrollContext scrollContext = new ScrollContext();
-        when(searchContext.scrollContext()).thenReturn(scrollContext);
-
-        SecuritySearchOperationListener listener = new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService);
-        listener.onNewScrollContext(scrollContext);
-        listener.validateSearchContext(searchContext, Empty.INSTANCE);
-        verify(licenseState, times(2)).isSecurityEnabled();
-        verifyZeroInteractions(auditTrailService, searchContext);
+        try (ReaderContext readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false)) {
+            XPackLicenseState licenseState = mock(XPackLicenseState.class);
+            when(licenseState.isSecurityEnabled()).thenReturn(false);
+            ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+            final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
+            AuditTrailService auditTrailService = mock(AuditTrailService.class);
+            SearchContext searchContext = mock(SearchContext.class);
+
+            SecuritySearchOperationListener listener =
+                new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService);
+            listener.onNewReaderContext(readerContext);
+            listener.validateSearchContext(readerContext, searchContext, Empty.INSTANCE);
+            verify(licenseState, times(2)).isSecurityEnabled();
+            verifyZeroInteractions(auditTrailService, searchContext);
+        }
     }
 
     public void testOnNewContextSetsAuthentication() throws Exception {
-        ScrollContext scrollContext = new ScrollContext();
-        TestSearchContext testSearchContext = new TestSearchContext(null, null, null, scrollContext);
-        final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(2L));
-        testSearchContext.scrollContext().scroll = scroll;
-        XPackLicenseState licenseState = mock(XPackLicenseState.class);
-        when(licenseState.isSecurityEnabled()).thenReturn(true);
-        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
-        AuditTrailService auditTrailService = mock(AuditTrailService.class);
-        Authentication authentication = new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
-        authentication.writeToContext(threadContext);
+        try (ReaderContext readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false)) {
+            XPackLicenseState licenseState = mock(XPackLicenseState.class);
+            when(licenseState.isSecurityEnabled()).thenReturn(true);
+            ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+            final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
+            AuditTrailService auditTrailService = mock(AuditTrailService.class);
+            Authentication authentication = new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
+            authentication.writeToContext(threadContext);
 
-        SecuritySearchOperationListener listener = new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService);
-        listener.onNewScrollContext(scrollContext);
+            SecuritySearchOperationListener listener =
+                new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService);
+            listener.onNewReaderContext(readerContext);
 
-        Authentication contextAuth = testSearchContext.scrollContext().getFromContext(AuthenticationField.AUTHENTICATION_KEY);
-        assertEquals(authentication, contextAuth);
-        assertEquals(scroll, testSearchContext.scrollContext().scroll);
+            Authentication contextAuth = readerContext.getFromContext(AuthenticationField.AUTHENTICATION_KEY);
+            assertEquals(authentication, contextAuth);
 
-        verify(licenseState).isSecurityEnabled();
-        verifyZeroInteractions(auditTrailService);
+            verify(licenseState).isSecurityEnabled();
+            verifyZeroInteractions(auditTrailService);
+        }
     }
 
     public void testValidateSearchContext() throws Exception {
-        ScrollContext scrollContext = new ScrollContext();
-        TestSearchContext testSearchContext = new TestSearchContext(null, null, null, scrollContext);
-        testSearchContext.scrollContext().putInContext(AuthenticationField.AUTHENTICATION_KEY,
+        try (ReaderContext readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false)) {
+            ScrollContext scrollContext = new ScrollContext();
+            TestSearchContext testSearchContext = new TestSearchContext(null, null, null, scrollContext);
+            readerContext.putInContext(AuthenticationField.AUTHENTICATION_KEY,
                 new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null));
-        testSearchContext.scrollContext().scroll = new Scroll(TimeValue.timeValueSeconds(2L));
-        XPackLicenseState licenseState = mock(XPackLicenseState.class);
-        when(licenseState.isSecurityEnabled()).thenReturn(true);
-        when(licenseState.isAllowed(Feature.SECURITY_AUDITING)).thenReturn(true);
-        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
-        AuditTrail auditTrail = mock(AuditTrail.class);
-        AuditTrailService auditTrailService = new AuditTrailService(Collections.singletonList(auditTrail), licenseState);
+            testSearchContext.scrollContext().scroll = new Scroll(TimeValue.timeValueSeconds(2L));
+            XPackLicenseState licenseState = mock(XPackLicenseState.class);
+            when(licenseState.isSecurityEnabled()).thenReturn(true);
+            when(licenseState.isAllowed(Feature.SECURITY_AUDITING)).thenReturn(true);
+            ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+            final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
+            AuditTrail auditTrail = mock(AuditTrail.class);
+            AuditTrailService auditTrailService =
+                new AuditTrailService(Collections.singletonList(auditTrail), licenseState);
 
-        SecuritySearchOperationListener listener = new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService);
-        try (StoredContext ignore = threadContext.newStoredContext(false)) {
-            Authentication authentication = new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
-            authentication.writeToContext(threadContext);
-            listener.validateSearchContext(testSearchContext, Empty.INSTANCE);
-            verify(licenseState).isSecurityEnabled();
-            verifyZeroInteractions(auditTrail);
-        }
+            SecuritySearchOperationListener listener =
+                new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService);
+            try (StoredContext ignore = threadContext.newStoredContext(false)) {
+                Authentication authentication = new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
+                authentication.writeToContext(threadContext);
+                listener.validateSearchContext(readerContext, testSearchContext, Empty.INSTANCE);
+                verify(licenseState).isSecurityEnabled();
+                verifyZeroInteractions(auditTrail);
+            }
 
-        try (StoredContext ignore = threadContext.newStoredContext(false)) {
-            final String nodeName = randomAlphaOfLengthBetween(1, 8);
-            final String realmName = randomAlphaOfLengthBetween(1, 16);
-            Authentication authentication = new Authentication(new User("test", "role"), new RealmRef(realmName, "file", nodeName), null);
-            authentication.writeToContext(threadContext);
-            listener.validateSearchContext(testSearchContext, Empty.INSTANCE);
-            verify(licenseState, times(2)).isSecurityEnabled();
-            verifyZeroInteractions(auditTrail);
-        }
+            try (StoredContext ignore = threadContext.newStoredContext(false)) {
+                final String nodeName = randomAlphaOfLengthBetween(1, 8);
+                final String realmName = randomAlphaOfLengthBetween(1, 16);
+                Authentication authentication =
+                    new Authentication(new User("test", "role"), new RealmRef(realmName, "file", nodeName), null);
+                authentication.writeToContext(threadContext);
+                listener.validateSearchContext(readerContext, testSearchContext, Empty.INSTANCE);
+                verify(licenseState, times(2)).isSecurityEnabled();
+                verifyZeroInteractions(auditTrail);
+            }
 
-        try (StoredContext ignore = threadContext.newStoredContext(false)) {
-            final String nodeName = randomBoolean() ? "node" : randomAlphaOfLengthBetween(1, 8);
-            final String realmName = randomBoolean() ? "realm" : randomAlphaOfLengthBetween(1, 16);
-            final String type = randomAlphaOfLengthBetween(5, 16);
-            Authentication authentication = new Authentication(new User("test", "role"), new RealmRef(realmName, type, nodeName), null);
-            authentication.writeToContext(threadContext);
-            threadContext.putTransient(ORIGINATING_ACTION_KEY, "action");
-            threadContext.putTransient(AUTHORIZATION_INFO_KEY,
-                (AuthorizationInfo) () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, authentication.getUser().roles()));
-            final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
-            SearchContextMissingException expected =
-                    expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request));
-            assertEquals(testSearchContext.id(), expected.contextId());
-            verify(licenseState, Mockito.atLeast(3)).isSecurityEnabled();
-            verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
-                authzInfoRoles(authentication.getUser().roles()));
-        }
+            try (StoredContext ignore = threadContext.newStoredContext(false)) {
+                final String nodeName = randomBoolean() ? "node" : randomAlphaOfLengthBetween(1, 8);
+                final String realmName = randomBoolean() ? "realm" : randomAlphaOfLengthBetween(1, 16);
+                final String type = randomAlphaOfLengthBetween(5, 16);
+                Authentication authentication =
+                    new Authentication(new User("test", "role"), new RealmRef(realmName, type, nodeName), null);
+                authentication.writeToContext(threadContext);
+                threadContext.putTransient(ORIGINATING_ACTION_KEY, "action");
+                threadContext.putTransient(AUTHORIZATION_INFO_KEY,
+                    (AuthorizationInfo) () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, authentication.getUser().roles()));
+                final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
+                SearchContextMissingException expected = expectThrows(SearchContextMissingException.class,
+                    () -> listener.validateSearchContext(readerContext, testSearchContext, request));
+                assertEquals(testSearchContext.id(), expected.contextId());
+                verify(licenseState, Mockito.atLeast(3)).isSecurityEnabled();
+                verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
+                    authzInfoRoles(authentication.getUser().roles()));
+            }
 
-        // another user running as the original user
-        try (StoredContext ignore = threadContext.newStoredContext(false)) {
-            final String nodeName = randomBoolean() ? "node" : randomAlphaOfLengthBetween(1, 8);
-            final String realmName = randomBoolean() ? "realm" : randomAlphaOfLengthBetween(1, 16);
-            final String type = randomAlphaOfLengthBetween(5, 16);
-            User user = new User(new User("test", "role"), new User("authenticated", "runas"));
-            Authentication authentication = new Authentication(user, new RealmRef(realmName, type, nodeName),
+            // another user running as the original user
+            try (StoredContext ignore = threadContext.newStoredContext(false)) {
+                final String nodeName = randomBoolean() ? "node" : randomAlphaOfLengthBetween(1, 8);
+                final String realmName = randomBoolean() ? "realm" : randomAlphaOfLengthBetween(1, 16);
+                final String type = randomAlphaOfLengthBetween(5, 16);
+                User user = new User(new User("test", "role"), new User("authenticated", "runas"));
+                Authentication authentication = new Authentication(user, new RealmRef(realmName, type, nodeName),
                     new RealmRef(randomAlphaOfLengthBetween(1, 16), "file", nodeName));
-            authentication.writeToContext(threadContext);
-            threadContext.putTransient(ORIGINATING_ACTION_KEY, "action");
-            final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
-            listener.validateSearchContext(testSearchContext, request);
-            verify(licenseState, Mockito.atLeast(4)).isSecurityEnabled();
-            verifyNoMoreInteractions(auditTrail);
-        }
+                authentication.writeToContext(threadContext);
+                threadContext.putTransient(ORIGINATING_ACTION_KEY, "action");
+                final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
+                listener.validateSearchContext(readerContext, testSearchContext, request);
+                verify(licenseState, Mockito.atLeast(4)).isSecurityEnabled();
+                verifyNoMoreInteractions(auditTrail);
+            }
 
-        // the user that authenticated for the run as request
-        try (StoredContext ignore = threadContext.newStoredContext(false)) {
-            final String nodeName = randomBoolean() ? "node" : randomAlphaOfLengthBetween(1, 8);
-            final String realmName = randomBoolean() ? "realm" : randomAlphaOfLengthBetween(1, 16);
-            final String type = randomAlphaOfLengthBetween(5, 16);
-            Authentication authentication =
+            // the user that authenticated for the run as request
+            try (StoredContext ignore = threadContext.newStoredContext(false)) {
+                final String nodeName = randomBoolean() ? "node" : randomAlphaOfLengthBetween(1, 8);
+                final String realmName = randomBoolean() ? "realm" : randomAlphaOfLengthBetween(1, 16);
+                final String type = randomAlphaOfLengthBetween(5, 16);
+                Authentication authentication =
                     new Authentication(new User("authenticated", "runas"), new RealmRef(realmName, type, nodeName), null);
-            authentication.writeToContext(threadContext);
-            threadContext.putTransient(ORIGINATING_ACTION_KEY, "action");
-            threadContext.putTransient(AUTHORIZATION_INFO_KEY,
-                (AuthorizationInfo) () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, authentication.getUser().roles()));
-            final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
-            SearchContextMissingException expected =
-                    expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request));
-            assertEquals(testSearchContext.id(), expected.contextId());
-            verify(licenseState, Mockito.atLeast(5)).isSecurityEnabled();
-            verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
-                authzInfoRoles(authentication.getUser().roles()));
+                authentication.writeToContext(threadContext);
+                threadContext.putTransient(ORIGINATING_ACTION_KEY, "action");
+                threadContext.putTransient(AUTHORIZATION_INFO_KEY,
+                    (AuthorizationInfo) () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, authentication.getUser().roles()));
+                final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
+                SearchContextMissingException expected = expectThrows(SearchContextMissingException.class,
+                    () -> listener.validateSearchContext(readerContext, testSearchContext, request));
+                assertEquals(testSearchContext.id(), expected.contextId());
+                verify(licenseState, Mockito.atLeast(5)).isSecurityEnabled();
+                verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
+                    authzInfoRoles(authentication.getUser().roles()));
+            }
         }
     }
 

From 5b3a41be0e50e9f5434f3fab5250d57f9b592ce8 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 28 Apr 2020 16:58:16 +0200
Subject: [PATCH 13/47] Mark the reader context as used (#55854)

This commit ensures that we protect the reader context
from being removed during the full lifecycle of a search phase.
---
 .../elasticsearch/search/SearchService.java   | 69 +++++++++++--------
 1 file changed, 39 insertions(+), 30 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index f2617f13f9c1f..bd7454824927a 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -339,13 +339,14 @@ public void onFailure(Exception exc) {
     private DfsSearchResult executeDfsPhase(ShardSearchRequest request,
                                             SearchShardTask task,
                                             boolean keepStatesInContext) throws IOException {
-        ReaderContext reader = createOrGetReaderContext(request, keepStatesInContext);
-        try (SearchContext context = createContext(reader, request, task, true)) {
+        ReaderContext readerContext = createOrGetReaderContext(request, keepStatesInContext);
+        try (Releasable ignored = readerContext.markAsUsed();
+                SearchContext context = createContext(readerContext, request, task, true)) {
             dfsPhase.execute(context);
             return context.dfsResult();
         } catch (Exception e) {
             logger.trace("Dfs phase failed", e);
-            processFailure(reader, e);
+            processFailure(readerContext, e);
             throw e;
         }
     }
@@ -373,6 +374,7 @@ public void executeQueryPhase(ShardSearchRequest request, boolean keepStatesInCo
             @Override
             public void onResponse(ShardSearchRequest orig) {
                 final ReaderContext readerContext = createOrGetReaderContext(orig, keepStatesInContext);
+                final Releasable releasable = readerContext.markAsUsed();
                 try {
                     if (orig.canReturnNullResponseIfMatchNoDocs()) {
                         assert orig.shouldCreatePersistentReader() == false;
@@ -389,7 +391,7 @@ public void onResponse(ShardSearchRequest orig) {
                         if (canRewriteToMatchNone(canMatchRequest.source())
                                 && canMatchRequest.source().query() instanceof MatchNoneQueryBuilder) {
                             if (orig.readerId() != null) {
-                                try (Releasable r = readerContext.markAsUsed()) {
+                                try (releasable) {
                                     listener.onResponse(QuerySearchResult.nullInstance());
                                 }
                             } else {
@@ -397,21 +399,25 @@ public void onResponse(ShardSearchRequest orig) {
                                     listener.onResponse(QuerySearchResult.nullInstance());
                                 } finally {
                                     // close and remove the ephemeral reader context
-                                    try (readerContext) {
-                                        removeReaderContext(readerContext.id().getId());
-                                    }
+                                    Releasables.close(releasable, readerContext);
+                                    removeReaderContext(readerContext.id().getId());
                                 }
                             }
                             return;
                         }
                     }
                     // fork the execution in the search thread pool
-                    runAsync(shard, () -> executeQueryPhase(orig, task, readerContext), listener);
+                    runAsync(shard, () -> {
+                        try (releasable) {
+                            return executeQueryPhase(orig, task, readerContext);
+                        }
+                    }, listener);
                 } catch (Exception exc) {
-                    try (readerContext) {
-                        removeReaderContext(readerContext.id().getId());
-                    } finally {
+                    try {
                         listener.onFailure(exc);
+                    } finally {
+                        Releasables.close(releasable, readerContext);
+                        removeReaderContext(readerContext.id().getId());
                     }
                 }
             }
@@ -443,18 +449,18 @@ private <T> void runAsync(SearchContextId id, CheckedSupplier<T, IOException> ex
 
     private SearchPhaseResult executeQueryPhase(ShardSearchRequest request,
                                                 SearchShardTask task,
-                                                ReaderContext reader) throws Exception {
-        try (SearchContext context = createContext(reader, request, task, true)) {
+                                                ReaderContext readerContext) throws Exception {
+        try (SearchContext context = createContext(readerContext, request, task, true)) {
             final long afterQueryTime;
             try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) {
                 loadOrExecuteQueryPhase(request, context);
-                if (context.queryResult().hasSearchContext() == false && reader.singleSession()) {
-                    freeReaderContext(reader.id());
+                if (context.queryResult().hasSearchContext() == false && readerContext.singleSession()) {
+                    freeReaderContext(readerContext.id());
                 }
                 afterQueryTime = executor.success();
             }
             if (request.numberOfShards() == 1) {
-                return executeFetchPhase(reader, context, afterQueryTime);
+                return executeFetchPhase(readerContext, context, afterQueryTime);
             }
             return context.queryResult();
         } catch (Exception e) {
@@ -464,7 +470,7 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request,
                     (Exception) e.getCause() : new ElasticsearchException(e.getCause());
             }
             logger.trace("Query phase failed", e);
-            processFailure(reader, e);
+            processFailure(readerContext, e);
             throw e;
         }
     }
@@ -486,8 +492,10 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
                                   ActionListener<ScrollQuerySearchResult> listener) {
         runAsync(request.contextId(), () -> {
             final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
-            try (SearchContext searchContext = createContext(readerContext, readerContext.getShardSearchRequest(null), task, false);
-                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
+            final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
+            try (Releasable ignored = readerContext.markAsUsed();
+                    SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
+                    SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
                 if (request.scroll() != null && request.scroll().keepAlive() != null) {
                     final long keepAlive = request.scroll().keepAlive().millis();
                     checkKeepAliveLimit(keepAlive);
@@ -513,10 +521,11 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
         runAsync(request.contextId(), () -> {
             final ReaderContext readerContext = findReaderContext(request.contextId());
+            final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.shardSearchRequest());
             readerContext.setAggregatedDfs(request.dfs());
-            try (SearchContext searchContext =
-                     createContext(readerContext, readerContext.getShardSearchRequest(request.shardSearchRequest()), task, true);
-                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
+            try (Releasable ignored = readerContext.markAsUsed();
+                    SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, true);
+                    SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
                 readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
                 searchContext.searcher().setAggregatedDfs(request.dfs());
                 queryPhase.execute(searchContext);
@@ -550,8 +559,10 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
         runAsync(request.contextId(), () -> {
             final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
-            try (SearchContext searchContext = createContext(readerContext, readerContext.getShardSearchRequest(null), task, false);
-                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
+            final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
+            try (Releasable ignored = readerContext.markAsUsed();
+                    SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
+                    SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
                 if (request.scroll() != null && request.scroll().keepAlive() != null) {
                     checkKeepAliveLimit(request.scroll().keepAlive().millis());
                     readerContext.keepAlive(request.scroll().keepAlive().millis());
@@ -576,7 +587,8 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
         runAsync(request.contextId(), () -> {
             final ReaderContext readerContext = findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest());
-            try (SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false)) {
+            try (Releasable ignored = readerContext.markAsUsed();
+                    SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false)) {
                 readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
                 if (request.lastEmittedDoc() != null) {
                     searchContext.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
@@ -742,8 +754,6 @@ final SearchContext createContext(ReaderContext reader,
             dfsPhase.preProcess(context);
             queryPhase.preProcess(context);
             fetchPhase.preProcess(context);
-            // Disable timeout while searching and mark the reader as accessed when releasing the context.
-            context.addReleasable(reader.markAsUsed());
             context.addReleasable(() -> reader.indexShard().getSearchOperationListener().onFreeSearchContext(context));
         } catch (Exception e) {
             context.close();
@@ -759,6 +769,7 @@ public DefaultSearchContext createSearchContext(ShardSearchRequest request, Time
         try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, reader, -1L, true)) {
             reader = null; // transfer ownership to readerContext
             DefaultSearchContext searchContext = createSearchContext(readerContext, request, timeout);
+            searchContext.addReleasable(readerContext.markAsUsed());
             searchContext.addReleasable(readerContext);
             return searchContext;
         } finally {
@@ -808,7 +819,6 @@ private void freeAllContextForIndex(Index index) {
         }
     }
 
-
     public boolean freeReaderContext(SearchContextId contextId) {
         if (getReaderContext(contextId) != null) {
             try (ReaderContext context = removeReaderContext(contextId.getId())) {
@@ -817,8 +827,7 @@ public boolean freeReaderContext(SearchContextId contextId) {
         }
         return false;
     }
-
-
+    
     public void freeAllScrollContexts() {
         for (ReaderContext readerContext : activeReaders.values()) {
             if (readerContext.scrollContext() != null) {

From 3be26ede688fe8450ee119e0aff3fe3a05185756 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 4 May 2020 09:07:40 -0400
Subject: [PATCH 14/47] Enable can match for search with reader contexts
 (#56032)

Enabling the can-match phase for search requests with reader contexts
---
 .../action/search/TransportSearchAction.java  |  4 +-
 .../search/DefaultSearchContext.java          |  1 -
 .../elasticsearch/search/SearchService.java   | 70 +++++++++++--------
 .../search/internal/LegacyReaderContext.java  |  6 +-
 .../search/internal/ReaderContext.java        | 30 ++++----
 .../SearchWithReaderContextIT.java            | 49 +++++++++++++
 6 files changed, 111 insertions(+), 49 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 87e0711bbdc54..d031cf9134234 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -517,8 +517,8 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
             indexRoutings = Map.of();
             localShardIterators = searchShardsFromReaderContexts(
                 clusterState, localIndices, searchRequest.getLocalClusterAlias(), readerContexts, searchRequest.reader().getKeepAlive());
-            // TODO: support can_match with reader contexts after https://github.com/elastic/elasticsearch/pull/54966
-            preFilterSearchShards = false;
+            preFilterSearchShards = shouldPreFilterSearchShards(clusterState, searchRequest, localIndices.indices(),
+                localShardIterators.size() + remoteShardIterators.size());
         } else {
             final Index[] indices = resolveLocalIndices(localIndices, searchRequest.indicesOptions(), clusterState, timeProvider);
             Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(),
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index c368590929a36..945e5e4673bd9 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -186,7 +186,6 @@ final class DefaultSearchContext extends SearchContext {
     @Override
     public void doClose() {
         engineSearcher.close();
-        readerContext.decRef();
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 6dcefe0f52def..e281b766fd2e9 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -341,7 +341,7 @@ private DfsSearchResult executeDfsPhase(ShardSearchRequest request,
                                             boolean keepStatesInContext) throws IOException {
         ReaderContext readerContext = createOrGetReaderContext(request, keepStatesInContext);
         try (Releasable ignored = readerContext.markAsUsed();
-                SearchContext context = createContext(readerContext, request, task, true)) {
+             SearchContext context = createContext(readerContext, request, task, true)) {
             dfsPhase.execute(context);
             return context.dfsResult();
         } catch (Exception e) {
@@ -494,8 +494,8 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
             final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
             try (Releasable ignored = readerContext.markAsUsed();
-                    SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
-                    SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
+                 SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
                 if (request.scroll() != null && request.scroll().keepAlive() != null) {
                     final long keepAlive = request.scroll().keepAlive().millis();
                     checkKeepAliveLimit(keepAlive);
@@ -524,8 +524,8 @@ public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task,
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.shardSearchRequest());
             readerContext.setAggregatedDfs(request.dfs());
             try (Releasable ignored = readerContext.markAsUsed();
-                    SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, true);
-                    SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
+                 SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, true);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
                 readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
                 searchContext.searcher().setAggregatedDfs(request.dfs());
                 queryPhase.execute(searchContext);
@@ -561,8 +561,8 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
             final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
             try (Releasable ignored = readerContext.markAsUsed();
-                    SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
-                    SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
+                 SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
+                 SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
                 if (request.scroll() != null && request.scroll().keepAlive() != null) {
                     checkKeepAliveLimit(request.scroll().keepAlive().millis());
                     readerContext.keepAlive(request.scroll().keepAlive().millis());
@@ -588,7 +588,7 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
             final ReaderContext readerContext = findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest());
             try (Releasable ignored = readerContext.markAsUsed();
-                    SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false)) {
+                 SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false)) {
                 readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
                 if (request.lastEmittedDoc() != null) {
                     searchContext.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
@@ -767,13 +767,9 @@ public DefaultSearchContext createSearchContext(ShardSearchRequest request, Time
         IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().getId());
         Engine.SearcherSupplier reader = indexShard.acquireSearcherSupplier();
         try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, reader, -1L, true)) {
-            reader = null; // transfer ownership to readerContext
             DefaultSearchContext searchContext = createSearchContext(readerContext, request, timeout);
             searchContext.addReleasable(readerContext.markAsUsed());
-            searchContext.addReleasable(readerContext);
             return searchContext;
-        } finally {
-            Releasables.close(reader);
         }
     }
 
@@ -781,7 +777,6 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
         throws IOException {
         boolean success = false;
         DefaultSearchContext searchContext = null;
-        reader.incRef();
         try {
             IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
             IndexShard indexShard = indexService.getShard(request.shardId().getId());
@@ -802,9 +797,6 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
                 // constructor throws an exception since we would otherwise leak a searcher and this can have severe implications
                 // (unable to obtain shard lock exceptions).
                 IOUtils.closeWhileHandlingException(searchContext);
-                if (searchContext == null) {
-                    IOUtils.closeWhileHandlingException(reader::decRef);
-                }
             }
         }
         return searchContext;
@@ -1119,7 +1111,7 @@ class Reaper implements Runnable {
         @Override
         public void run() {
             for (ReaderContext context : activeReaders.values()) {
-                if (context.isKeepAliveLapsed()) {
+                if (context.isExpired()) {
                     logger.debug("freeing search context [{}]", context.id());
                     freeReaderContext(context.id());
                 }
@@ -1138,17 +1130,33 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Set<String
      */
     public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException {
         assert request.searchType() == SearchType.QUERY_THEN_FETCH : "unexpected search type: " + request.searchType();
-        // TODO: support can_match with reader contexts after https://github.com/elastic/elasticsearch/pull/54966
-        assert request.readerId() == null : "request with reader_id bypass can_match phase";
         IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-        IndexShard indexShard = indexService.getShard(request.shardId().getId());
-        // we don't want to use the reader wrapper since it could run costly operations
-        // and we can afford false positives.
-        final boolean hasRefreshPending = indexShard.hasRefreshPending();
-        try (Engine.Searcher searcher = indexShard.acquireSearcher("can_match")) {
+        final boolean hasRefreshPending;
+        final Engine.Searcher canMatchSearcher;
+        if (request.readerId() != null) {
+            final ReaderContext readerContext = findReaderContext(request.readerId());
+            checkKeepAliveLimit(request.keepAlive().millis());
+            readerContext.keepAlive(request.keepAlive().millis());
+            Releasable releasable = readerContext.markAsUsed();
+            try {
+                final Engine.Searcher searcher = readerContext.acquireSearcher("can_match");
+                canMatchSearcher = new Engine.Searcher(searcher.source(), searcher.getDirectoryReader(),
+                    searcher.getSimilarity(), searcher.getQueryCache(), searcher.getQueryCachingPolicy(),
+                    Releasables.wrap(searcher, releasable));
+                releasable = null;
+            } finally {
+                Releasables.close(releasable);
+            }
+            hasRefreshPending = false;
+        } else {
+            IndexShard indexShard = indexService.getShard(request.shardId().getId());
+            hasRefreshPending = indexShard.hasRefreshPending();
+            canMatchSearcher = indexShard.acquireSearcher("can_match");
+        }
+        try (Releasable ignored = canMatchSearcher) {
             final boolean aliasFilterCanMatch = request.getAliasFilter()
                 .getQueryBuilder() instanceof MatchNoneQueryBuilder == false;
-            QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), searcher,
+            QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher,
                 request::nowInMillis, request.getClusterAlias());
             Rewriteable.rewrite(request.getRewriteable(), context, false);
             FieldSortBuilder sortBuilder = FieldSortBuilder.getPrimaryFieldSortOrNull(request.source());
@@ -1187,10 +1195,14 @@ public static boolean canRewriteToMatchNone(SearchSourceBuilder source) {
     }
 
     private void rewriteAndFetchShardRequest(IndexShard shard, ShardSearchRequest request, ActionListener<ShardSearchRequest> listener) {
-        ActionListener<Rewriteable> actionListener = ActionListener.wrap(r ->
-            // now we need to check if there is a pending refresh and register
-            shard.awaitShardSearchActive(b -> listener.onResponse(request)),
-            listener::onFailure);
+        ActionListener<Rewriteable> actionListener = ActionListener.wrap(r -> {
+            if (request.readerId() != null) {
+                listener.onResponse(request);
+            } else {
+                // now we need to check if there is a pending refresh and register
+                shard.awaitShardSearchActive(b -> listener.onResponse(request));
+            }
+        }, listener::onFailure);
         // we also do rewrite on the coordinating node (TransportSearchService) but we also need to do it here for BWC as well as
         // AliasFilters that might need to be rewritten. These are edge-cases but we are every efficient doing the rewrite here so it's not
         // adding a lot of overhead
diff --git a/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
index 5f719836a4a3c..5be1aeea9be73 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
@@ -68,10 +68,10 @@ public Engine.Searcher acquireSearcher(String source) {
         return super.acquireSearcher(source);
     }
 
+
     @Override
-    protected void closeInternal() {
-        Releasables.close(onClose);
-        super.closeInternal();
+    void doClose() {
+        Releasables.close(onClose, super::doClose);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
index 37593e7c172f3..1caea59c7e608 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
@@ -34,7 +34,6 @@
 import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 /**
@@ -43,10 +42,8 @@
  * in {@link org.elasticsearch.search.SearchService} a SearchContext can be closed concurrently due to independent events
  * ie. when an index gets removed. To prevent accessing closed IndexReader / IndexSearcher instances the SearchContext
  * can be guarded by a reference count and fail if it's been closed by an external event.
- *
- * For reference why we use RefCounted here see https://github.com/elastic/elasticsearch/pull/20095.
  */
-public class ReaderContext extends AbstractRefCounted implements Releasable {
+public class ReaderContext implements Releasable {
     private final SearchContextId id;
     private final IndexShard indexShard;
     protected final Engine.SearcherSupplier searcherSupplier;
@@ -55,7 +52,8 @@ public class ReaderContext extends AbstractRefCounted implements Releasable {
 
     private final AtomicLong keepAlive;
     private final AtomicLong lastAccessTime;
-    private final AtomicInteger accessors = new AtomicInteger();
+    // For reference why we use RefCounted here see https://github.com/elastic/elasticsearch/pull/20095.
+    private final AbstractRefCounted refCounted;
 
     private final List<Releasable> onCloses = new CopyOnWriteArrayList<>();
 
@@ -68,13 +66,18 @@ public ReaderContext(long id,
                          Engine.SearcherSupplier searcherSupplier,
                          long keepAliveInMillis,
                          boolean singleSession) {
-        super("reader_context");
         this.id = new SearchContextId(UUIDs.base64UUID(), id);
         this.indexShard = indexShard;
         this.searcherSupplier = searcherSupplier;
         this.singleSession = singleSession;
         this.keepAlive = new AtomicLong(keepAliveInMillis);
         this.lastAccessTime = new AtomicLong(nowInMillis());
+        this.refCounted = new AbstractRefCounted("reader_context") {
+            @Override
+            protected void closeInternal() {
+                doClose();
+            }
+        };
     }
 
     private long nowInMillis() {
@@ -84,12 +87,11 @@ private long nowInMillis() {
     @Override
     public final void close() {
         if (closed.compareAndSet(false, true)) {
-            decRef();
+            refCounted.decRef();
         }
     }
 
-    @Override
-    protected void closeInternal() {
+    void doClose() {
         Releasables.close(Releasables.wrap(onCloses), searcherSupplier);
     }
 
@@ -120,16 +122,16 @@ public void keepAlive(long keepAlive) {
      * @return a releasable to indicate the caller has stopped using this reader
      */
     public Releasable markAsUsed() {
-        accessors.incrementAndGet();
+        refCounted.incRef();
         return Releasables.releaseOnce(() -> {
             this.lastAccessTime.updateAndGet(curr -> Math.max(curr, nowInMillis()));
-            accessors.decrementAndGet();
+            refCounted.decRef();
         });
     }
 
-    public boolean isKeepAliveLapsed() {
-        if (accessors.get() > 0) {
-            return false; // being used
+    public boolean isExpired() {
+        if (refCounted.refCount() > 1) {
+            return false; // being used by markAsUsed
         }
         final long elapsed = nowInMillis() - lastAccessTime.get();
         return elapsed > keepAlive.get();
diff --git a/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java b/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
index ffc1d83200e85..ac166298a69e4 100644
--- a/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
+++ b/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
@@ -27,20 +27,29 @@
 import org.elasticsearch.action.search.OpenReaderResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.search.TransportOpenReaderAction;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -189,6 +198,46 @@ public void testIndexNotFound() {
         clearReaderId(resp1.getReaderId());
     }
 
+    public void testCanMatch() throws Exception {
+        final Settings.Builder settings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(5, 10))
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.timeValueMillis(randomIntBetween(50, 100)));
+        assertAcked(prepareCreate("test").setSettings(settings)
+            .setMapping("{\"properties\":{\"created_date\":{\"type\": \"date\", \"format\": \"yyyy-MM-dd\"}}}"));
+        ensureGreen("test");
+        String readerId = openReaders(new String[]{"test*"}, TimeValue.timeValueMinutes(2));
+        try {
+            for (String node : internalCluster().nodesInclude("test")) {
+                for (IndexService indexService : internalCluster().getInstance(IndicesService.class, node)) {
+                    for (IndexShard indexShard : indexService) {
+                        assertBusy(() -> assertTrue(indexShard.isSearchIdle()));
+                    }
+                }
+            }
+            client().prepareIndex("test").setId("1").setSource("created_date", "2020-01-01").get();
+            SearchResponse resp = client().prepareSearch()
+                .setQuery(new RangeQueryBuilder("created_date").gte("2020-01-02").lte("2020-01-03"))
+                .setSearchType(SearchType.QUERY_THEN_FETCH)
+                .setPreference(null)
+                .setPreFilterShardSize(randomIntBetween(2, 3))
+                .setMaxConcurrentShardRequests(randomIntBetween(1, 2))
+                .setReader(readerId, TimeValue.timeValueMinutes(2))
+                .get();
+            assertThat(resp.getHits().getHits(), arrayWithSize(0));
+            for (String node : internalCluster().nodesInclude("test")) {
+                for (IndexService indexService : internalCluster().getInstance(IndicesService.class, node)) {
+                    for (IndexShard indexShard : indexService) {
+                        // all shards are still search-idle as we did not acquire new searchers
+                        assertTrue(indexShard.isSearchIdle());
+                    }
+                }
+            }
+        } finally {
+            clearReaderId(readerId);
+        }
+    }
+
     private String openReaders(String[] indices, TimeValue keepAlive) {
         OpenReaderRequest request = new OpenReaderRequest(indices, OpenReaderRequest.DEFAULT_INDICES_OPTIONS, keepAlive, null, null);
         final OpenReaderResponse response = client().execute(TransportOpenReaderAction.INSTANCE, request).actionGet();

From c2231fe5d9954e0c333e111bfcfb04709d933251 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Wed, 6 May 2020 23:06:07 -0400
Subject: [PATCH 15/47] Move SearchWithReaderContextIT to internalClusterTest

---
 .../search/searchafter/SearchWithReaderContextIT.java             | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename server/src/{test => internalClusterTest}/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java (100%)

diff --git a/server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
similarity index 100%
rename from server/src/test/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
rename to server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java

From c6dac42d3f94d8ed24466d41954848a4a8950678 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 7 May 2020 21:34:58 -0400
Subject: [PATCH 16/47] Rename reader context to search context (#56351)

This change renames reader context to search context in the rest layer.
---
 .../client/RestHighLevelClientTests.java      |  4 +-
 ...ntext.asciidoc => search_context.asciidoc} | 68 +++++++++----------
 ..._reader.json => close_search_context.json} | 10 +--
 ...n_reader.json => open_search_context.json} | 14 ++--
 ...der_context.yml => 330_search_context.yml} | 32 ++++-----
 .../SearchWithReaderContextIT.java            | 55 +++++++--------
 .../elasticsearch/action/ActionModule.java    | 21 +++---
 .../search/AbstractSearchAsyncAction.java     |  8 +--
 .../action/search/ClearContextController.java | 12 ++--
 ...ion.java => CloseSearchContextAction.java} | 10 +--
 ...st.java => CloseSearchContextRequest.java} | 12 ++--
 ...e.java => CloseSearchContextResponse.java} |  6 +-
 .../action/search/FetchSearchPhase.java       |  4 +-
 ...est.java => OpenSearchContextRequest.java} |  9 +--
 ...se.java => OpenSearchContextResponse.java} | 22 +++---
 .../action/search/SearchRequest.java          |  8 +--
 .../action/search/SearchRequestBuilder.java   |  2 +-
 .../action/search/SearchResponse.java         | 27 ++++----
 .../action/search/SearchTransportService.java | 12 ++--
 ...=> TransportCloseSearchContextAction.java} | 10 +--
 ... => TransportOpenSearchContextAction.java} | 42 ++++++------
 .../action/search/TransportSearchAction.java  | 14 ++--
 ...java => RestCloseSearchContextAction.java} | 16 ++---
 ....java => RestOpenSearchContextAction.java} | 16 ++---
 .../search/builder/SearchSourceBuilder.java   | 49 ++++++-------
 .../action/search/MockSearchPhaseContext.java |  5 +-
 .../action/search/SearchRequestTests.java     | 12 ++--
 .../index/engine/FrozenIndexTests.java        | 20 +++---
 .../xpack/security/authz/RBACEngine.java      |  4 +-
 .../DocumentLevelSecurityTests.java           | 10 +--
 .../integration/FieldLevelSecurityTests.java  | 25 +++----
 31 files changed, 282 insertions(+), 277 deletions(-)
 rename docs/reference/search/{request/reader_context.asciidoc => search_context.asciidoc} (54%)
 rename rest-api-spec/src/main/resources/rest-api-spec/api/{clear_reader.json => close_search_context.json} (57%)
 rename rest-api-spec/src/main/resources/rest-api-spec/api/{open_reader.json => open_search_context.json} (76%)
 rename rest-api-spec/src/main/resources/rest-api-spec/test/search/{330_reader_context.yml => 330_search_context.yml} (81%)
 rename server/src/main/java/org/elasticsearch/action/search/{ClearReaderAction.java => CloseSearchContextAction.java} (71%)
 rename server/src/main/java/org/elasticsearch/action/search/{ClearReaderRequest.java => CloseSearchContextRequest.java} (87%)
 rename server/src/main/java/org/elasticsearch/action/search/{ClearReaderResponse.java => CloseSearchContextResponse.java} (81%)
 rename server/src/main/java/org/elasticsearch/action/search/{OpenReaderRequest.java => OpenSearchContextRequest.java} (89%)
 rename server/src/main/java/org/elasticsearch/action/search/{OpenReaderResponse.java => OpenSearchContextResponse.java} (71%)
 rename server/src/main/java/org/elasticsearch/action/search/{TransportClearReaderAction.java => TransportCloseSearchContextAction.java} (72%)
 rename server/src/main/java/org/elasticsearch/action/search/{TransportOpenReaderAction.java => TransportOpenSearchContextAction.java} (81%)
 rename server/src/main/java/org/elasticsearch/rest/action/search/{RestClearReaderAction.java => RestCloseSearchContextAction.java} (73%)
 rename server/src/main/java/org/elasticsearch/rest/action/search/{RestOpenReaderAction.java => RestOpenSearchContextAction.java} (74%)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index d52ae7b35538b..6b265564b3509 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -861,8 +861,8 @@ public void testApiNamingConventions() throws Exception {
             "indices.create_data_stream",
             "indices.get_data_streams",
             "indices.delete_data_stream",
-            "clear_reader",
-            "open_reader"
+            "open_search_context",
+            "close_search_context",
         };
         //These API are not required for high-level client feature completeness
         String[] notRequiredApi = new String[] {
diff --git a/docs/reference/search/request/reader_context.asciidoc b/docs/reference/search/search_context.asciidoc
similarity index 54%
rename from docs/reference/search/request/reader_context.asciidoc
rename to docs/reference/search/search_context.asciidoc
index 6fb397fceceb2..7c3da5cedc865 100644
--- a/docs/reference/search/request/reader_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -1,24 +1,24 @@
-[[reader-context]]
-==== Reader Context
+[[search-context]]
+==== Search Context
 
 By default, a search request executes against the latest point in time readers of the
 target indices. Sometimes it's preferred to execute multiple search requests using
 the same point in time readers. For example, the combined result of the initial and
 subsequent search_after requests is more consistent if they use the same point in time
-readers.
+readers. This can be done by using a single search context for all search requests.
 
-Point in time readers must be opened in a separate step before being used in subsequent
+A search context must be opened in a separate step before being used in subsequent
 search requests. The keep_alive parameter tells Elasticsearch how long it should keep
-the ``reader_context'' alive, e.g. `?keep_alive=5m`.
+the seach context alive, e.g. `?keep_alive=5m`.
 
 [source,console]
 --------------------------------------------------
-POST /twitter/_open_reader?keep_alive=1m
+POST /twitter/_search_context?keep_alive=1m
 --------------------------------------------------
 // TEST[setup:twitter]
 
-The result from the above request includes a `reader_id`, which should
-be passed to the `id` of the `reader` parameter of a search request.
+The result from the above request includes a `id`, which should
+be passed to the `id` of the `search_context` parameter of a search request.
 
 POST /_search <1>
 {
@@ -28,46 +28,46 @@ POST /_search <1>
             "title" : "elasticsearch"
         }
     },
-    "reader": {
+    "search_context": {
 	    "id":  "46ToAwEHdHdpdHRlchZSa1dRZDFELVNLbTZkQVJudUtGMFFnAAAWNVAwS09JWTdTRUdibWE1ZC1id0tjQRRURnhPWjNFQmI3bmVOeVZReS1tRAAAAAAAAAAB", <2>
 	    "keep_alive": "1m"  <3>
     }
 }
 
-<1> A search request with `reader.id` must not specify `index`, `routing`,
-and `preferences` as these parameters are copied from the `reader.id`.
-<2> The `reader.id` parameter tells Elasticsearch to execute the request using
-the point-in-time readers from this id.
-<3> The `reader.keep_alive` parameter tells Elasticsearch how long it should extend
-the time to live of these readers.
-
-IMPORTANT: The open_reader request and each subsequent search request can return
-different `reader_id`; thus always use the most recently received `reader_id`
-for the next search request.
-
-[[reader-keep-alive]]
-===== Keeping the reader context alive
-The `keep_alive` parameter (passed to a `open_reader` request and search
-request) extends the time to live of open reader contexts. The value
+<1> A search request with `search_context` must not specify `index`, `routing`,
+and `preferences` as these parameters are copied from the `search_context`.
+<2> The `id` parameter tells Elasticsearch to execute the request using
+the point-in-time readers from this search context id.
+<3> The `keep_alive` parameter tells Elasticsearch how long it should extend
+the time to live of the search context
+
+IMPORTANT: The open search context request and each subsequent search request can
+return different `id`; thus always use the most recently received `id` for the
+next search request.
+
+[[search-context-keep-alive]]
+===== Keeping the search context alive
+The `keep_alive` parameter, which is passed to a open search context request and
+search request, extends the time to live of the search context. The value
 (e.g. `1m`, see <<time-units>>) does not need to be long enough to
 process all data -- it just needs to be long enough for the next request.
 
 Normally, the background merge process optimizes the index by merging together
 smaller segments to create new, bigger segments. Once the smaller segments are
-no longer needed they are deleted. However, open reader contexts prevents the
+no longer needed they are deleted. However, open search contexts prevents the
 old segments from being deleted since they are still in use.
 
 TIP: Keeping older segments alive means that more disk space and file handles
 are needed. Ensure that you have configured your nodes to have ample free file
 handles. See <<file-descriptors>>.
 
-Additionally, if a segment contains deleted or updated documents then the reader
+Additionally, if a segment contains deleted or updated documents then the search
 context must keep track of whether each document in the segment was live at the
 time of the initial search request. Ensure that your nodes have sufficient heap
-space if you have many open reader contexts on an index that is subject to ongoing
+space if you have many open search contexts on an index that is subject to ongoing
 deletes or updates.
 
-You can check how many reader contexts are open with the
+You can check how many search contexts are open with the
 <<cluster-nodes-stats,nodes stats API>>:
 
 [source,console]
@@ -75,16 +75,16 @@ You can check how many reader contexts are open with the
 GET /_nodes/stats/indices/search
 ---------------------------------------
 
-===== Clear reader API
+===== Close search context API
 
-Reader contexts are automatically closed when the `reader.keep_alive` has
-been elapsed. However keeping readers open has a cost, as discussed in the
-<<reader-keep-alive,previous section>> so readers should be explicitly
-closed as soon as they are no longer used in search requests.
+Search contexts are automatically closed when the `keep_alive` has
+been elapsed. However keeping search contexts as a cost, as discussed in the
+<<search-context-keep-alive,previous section>> so search context should be
+explicitly closed as soon as they are no longer used in search requests.
 
 [source,console]
 ---------------------------------------
-DELETE /_search/reader
+DELETE /_search_context
 {
     "id" : "46ToAwEHdHdpdHRlchZSa1dRZDFELVNLbTZkQVJudUtGMFFnAAAWNVAwS09JWTdTRUdibWE1ZC1id0tjQRRURnhPWjNFQmI3bmVOeVZReS1tRAAAAAAAAAAB"
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json b/rest-api-spec/src/main/resources/rest-api-spec/api/close_search_context.json
similarity index 57%
rename from rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json
rename to rest-api-spec/src/main/resources/rest-api-spec/api/close_search_context.json
index 0799d22a81e0d..87c97127fa616 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_reader.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/close_search_context.json
@@ -1,14 +1,14 @@
 {
-  "clear_reader":{
+  "close_search_context":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/reader-context.html",
-      "description":"Explicitly close the point in time readers"
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-context.html",
+      "description":"Close a search context"
     },
     "stability":"beta",
     "url":{
       "paths":[
         {
-          "path":"/_search/reader",
+          "path":"/_search_context",
           "methods":[
             "DELETE"
           ]
@@ -17,7 +17,7 @@
     },
     "params":{},
     "body":{
-      "description": "point in time readers to clear"
+      "description": "a search context to close"
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/open_reader.json b/rest-api-spec/src/main/resources/rest-api-spec/api/open_search_context.json
similarity index 76%
rename from rest-api-spec/src/main/resources/rest-api-spec/api/open_reader.json
rename to rest-api-spec/src/main/resources/rest-api-spec/api/open_search_context.json
index 435e40e8d9281..4d3fb8c5f632b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/open_reader.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/open_search_context.json
@@ -1,27 +1,27 @@
 {
-  "open_reader":{
+  "open_search_context":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/reader-context.html",
-      "description":"Open and keep point-in-time readers that can be used in subsequent search requests"
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-context.html",
+      "description":"Open a search context that can be used in subsequent searches"
     },
     "stability":"beta",
     "url":{
       "paths":[
         {
-          "path":"/_open_reader",
+          "path":"/_search_context",
           "methods":[
             "POST"
           ]
         },
         {
-          "path":"/{index}/_open_reader",
+          "path":"/{index}/_search_context",
           "methods":[
             "POST"
           ],
           "parts":{
             "index":{
               "type":"list",
-              "description":"A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices"
+              "description":"A comma-separated list of index names to open search context; use `_all` or empty string to perform the operation on all indices"
             }
           }
         }
@@ -54,7 +54,7 @@
       },
       "keep_alive": {
         "type": "string",
-        "description": "Specific the time to live for the point-in-time readers"
+        "description": "Specific the time to live for the search context"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml
similarity index 81%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml
rename to rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml
index 6da2ea1152ad9..253678dda9db2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_reader_context.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml
@@ -25,15 +25,15 @@ setup:
         index: test
 
 ---
-"search with reader context":
+"search context":
   - skip:
       version: " - 7.9.99"
-      reason: "reader context is introduced in 8.0"
+      reason: "search context is introduced in 8.0"
   - do:
-      open_reader:
+      open_search_context:
         index: test
-        keep_alive: 1m
-  - set: {reader_id: reader_id}
+        keep_alive: 5m
+  - set: {id: search_context_id}
 
   - do:
       search:
@@ -44,8 +44,8 @@ setup:
             match:
               foo: bar
           sort: [{ age: desc }, { id: desc }]
-          reader:
-            id: "$reader_id"
+          search_context:
+            id: "$search_context_id"
             keep_alive: 1m
 
   - match: {hits.total: 3 }
@@ -63,7 +63,7 @@ setup:
       indices.refresh:
         index: test
 
-  # search with the retaining readers
+  # search with the retaining search context
   - do:
       search:
         rest_total_hits_as_int: true
@@ -74,8 +74,8 @@ setup:
               foo: bar
           sort: [{ age: desc }, { id: desc }]
           search_after: [24, 172]
-          reader:
-            id: "$reader_id"
+          search_context:
+            id: "$search_context_id"
             keep_alive: 1m
 
   - match: {hits.total: 3 }
@@ -94,8 +94,8 @@ setup:
               foo: bar
           sort: [ { age: desc }, { id: desc } ]
           search_after: [18, 42]
-          reader:
-            id: "$reader_id"
+          search_context:
+            id: "$search_context_id"
             keep_alive: 1m
 
   - match: {hits.total: 3}
@@ -114,14 +114,14 @@ setup:
               foo: bar
           sort: [{ age: desc }, { id: desc } ]
           search_after: [18, 1]
-          reader:
-            id: "$reader_id"
+          search_context:
+            id: "$search_context_id"
             keep_alive: 1m
 
   - match: {hits.total: 3}
   - length: {hits.hits: 0 }
 
   - do:
-      clear_reader:
+      close_search_context:
         body:
-          id: "$reader_id"
+          id: "$search_context_id"
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
index ac166298a69e4..e597ea23e5847 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
@@ -21,15 +21,15 @@
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
-import org.elasticsearch.action.search.ClearReaderAction;
-import org.elasticsearch.action.search.ClearReaderRequest;
-import org.elasticsearch.action.search.OpenReaderRequest;
-import org.elasticsearch.action.search.OpenReaderResponse;
+import org.elasticsearch.action.search.CloseSearchContextAction;
+import org.elasticsearch.action.search.CloseSearchContextRequest;
+import org.elasticsearch.action.search.OpenSearchContextRequest;
+import org.elasticsearch.action.search.OpenSearchContextResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.action.search.TransportOpenReaderAction;
+import org.elasticsearch.action.search.TransportOpenSearchContextAction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -70,9 +70,9 @@ public void testBasic() {
             client().prepareIndex("test").setId(id).setSource("value", i).get();
         }
         refresh("test");
-        String readerId = openReaders(new String[]{"test"}, TimeValue.timeValueMinutes(2));
+        String readerId = openSearchContext(new String[]{"test"}, TimeValue.timeValueMinutes(2));
         SearchResponse resp1 = client().prepareSearch().setPreference(null).setReader(readerId, TimeValue.timeValueMinutes(2)).get();
-        assertThat(resp1.getReaderId(), equalTo(readerId));
+        assertThat(resp1.searchContextId(), equalTo(readerId));
         assertHitCount(resp1, numDocs);
         int deletedDocs = 0;
         for (int i = 0; i < numDocs; i++) {
@@ -90,13 +90,13 @@ public void testBasic() {
         }
         try {
             SearchResponse resp3 = client().prepareSearch().setPreference(null).setQuery(new MatchAllQueryBuilder())
-                .setReader(resp1.getReaderId(), TimeValue.timeValueMinutes(2))
+                .setReader(resp1.searchContextId(), TimeValue.timeValueMinutes(2))
                 .get();
             assertNoFailures(resp3);
             assertHitCount(resp3, numDocs);
-            assertThat(resp3.getReaderId(), equalTo(readerId));
+            assertThat(resp3.searchContextId(), equalTo(readerId));
         } finally {
-            clearReaderId(readerId);
+            closeSearchContext(readerId);
         }
     }
 
@@ -112,7 +112,7 @@ public void testMultipleIndices() {
             client().prepareIndex(index).setId(id).setSource("value", i).get();
         }
         refresh();
-        String readerId = openReaders(new String[]{"*"}, TimeValue.timeValueMinutes(2));
+        String readerId = openSearchContext(new String[]{"*"}, TimeValue.timeValueMinutes(2));
         SearchResponse resp1 = client().prepareSearch().setPreference(null).setReader(readerId, TimeValue.timeValueMinutes(2)).get();
         assertNoFailures(resp1);
         assertHitCount(resp1, numDocs);
@@ -129,11 +129,11 @@ public void testMultipleIndices() {
             assertHitCount(resp2, numDocs + moreDocs);
 
             SearchResponse resp3 = client().prepareSearch().setPreference(null)
-                .setReader(resp1.getReaderId(), TimeValue.timeValueMinutes(1)).get();
+                .setReader(resp1.searchContextId(), TimeValue.timeValueMinutes(1)).get();
             assertNoFailures(resp3);
             assertHitCount(resp3, numDocs);
         } finally {
-            clearReaderId(resp1.getReaderId());
+            closeSearchContext(resp1.searchContextId());
         }
     }
 
@@ -145,7 +145,7 @@ public void testReaderIdNotFound() throws Exception {
             client().prepareIndex("index").setId(id).setSource("value", i).get();
         }
         refresh();
-        String readerId = openReaders(new String[]{"index"}, TimeValue.timeValueSeconds(5));
+        String readerId = openSearchContext(new String[]{"index"}, TimeValue.timeValueSeconds(5));
         SearchResponse resp1 = client().prepareSearch().setPreference(null)
             .setReader(readerId, TimeValue.timeValueMillis(randomIntBetween(0, 10))).get();
         assertNoFailures(resp1);
@@ -156,10 +156,10 @@ public void testReaderIdNotFound() throws Exception {
                 assertThat(stats.search.getOpenContexts(), equalTo(0L));
             }, 60, TimeUnit.SECONDS);
         } else {
-            clearReaderId(resp1.getReaderId());
+            closeSearchContext(resp1.searchContextId());
         }
         SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class,
-            () -> client().prepareSearch().setPreference(null).setReader(resp1.getReaderId(), TimeValue.timeValueMinutes(1)).get());
+            () -> client().prepareSearch().setPreference(null).setReader(resp1.searchContextId(), TimeValue.timeValueMinutes(1)).get());
         for (ShardSearchFailure failure : e.shardFailures()) {
             assertThat(ExceptionsHelper.unwrapCause(failure.getCause()), instanceOf(SearchContextMissingException.class));
         }
@@ -181,7 +181,7 @@ public void testIndexNotFound() {
             client().prepareIndex("index-2").setId(id).setSource("value", i).get();
         }
         refresh();
-        String readerId = openReaders(new String[]{"index-*"}, TimeValue.timeValueMinutes(2));
+        String readerId = openSearchContext(new String[]{"index-*"}, TimeValue.timeValueMinutes(2));
         SearchResponse resp1 = client().prepareSearch().setPreference(null).setReader(readerId, TimeValue.timeValueMinutes(2)).get();
         assertNoFailures(resp1);
         assertHitCount(resp1, index1 + index2);
@@ -194,8 +194,8 @@ public void testIndexNotFound() {
         }
         expectThrows(IndexNotFoundException.class, () -> client().prepareSearch()
             .setPreference(null)
-            .setReader(resp1.getReaderId(), TimeValue.timeValueMinutes(1)).get());
-        clearReaderId(resp1.getReaderId());
+            .setReader(resp1.searchContextId(), TimeValue.timeValueMinutes(1)).get());
+        closeSearchContext(resp1.searchContextId());
     }
 
     public void testCanMatch() throws Exception {
@@ -206,7 +206,7 @@ public void testCanMatch() throws Exception {
         assertAcked(prepareCreate("test").setSettings(settings)
             .setMapping("{\"properties\":{\"created_date\":{\"type\": \"date\", \"format\": \"yyyy-MM-dd\"}}}"));
         ensureGreen("test");
-        String readerId = openReaders(new String[]{"test*"}, TimeValue.timeValueMinutes(2));
+        String readerId = openSearchContext(new String[]{"test*"}, TimeValue.timeValueMinutes(2));
         try {
             for (String node : internalCluster().nodesInclude("test")) {
                 for (IndexService indexService : internalCluster().getInstance(IndicesService.class, node)) {
@@ -234,17 +234,18 @@ public void testCanMatch() throws Exception {
                 }
             }
         } finally {
-            clearReaderId(readerId);
+            closeSearchContext(readerId);
         }
     }
 
-    private String openReaders(String[] indices, TimeValue keepAlive) {
-        OpenReaderRequest request = new OpenReaderRequest(indices, OpenReaderRequest.DEFAULT_INDICES_OPTIONS, keepAlive, null, null);
-        final OpenReaderResponse response = client().execute(TransportOpenReaderAction.INSTANCE, request).actionGet();
-        return response.getReaderId();
+    private String openSearchContext(String[] indices, TimeValue keepAlive) {
+        OpenSearchContextRequest request =
+            new OpenSearchContextRequest(indices, OpenSearchContextRequest.DEFAULT_INDICES_OPTIONS, keepAlive, null, null);
+        final OpenSearchContextResponse response = client().execute(TransportOpenSearchContextAction.INSTANCE, request).actionGet();
+        return response.getSearchContextId();
     }
 
-    private void clearReaderId(String readerId) {
-        client().execute(ClearReaderAction.INSTANCE, new ClearReaderRequest(readerId)).actionGet();
+    private void closeSearchContext(String readerId) {
+        client().execute(CloseSearchContextAction.INSTANCE, new CloseSearchContextRequest(readerId)).actionGet();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 9dc7c09d5decb..9867ddc237cb3 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -109,9 +109,6 @@
 import org.elasticsearch.action.admin.indices.close.TransportVerifyShardBeforeCloseAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
-import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
-import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
-import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
 import org.elasticsearch.action.admin.indices.flush.FlushAction;
@@ -205,15 +202,15 @@
 import org.elasticsearch.action.ingest.SimulatePipelineTransportAction;
 import org.elasticsearch.action.main.MainAction;
 import org.elasticsearch.action.main.TransportMainAction;
-import org.elasticsearch.action.search.ClearReaderAction;
+import org.elasticsearch.action.search.CloseSearchContextAction;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.MultiSearchAction;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchScrollAction;
-import org.elasticsearch.action.search.TransportClearReaderAction;
+import org.elasticsearch.action.search.TransportCloseSearchContextAction;
 import org.elasticsearch.action.search.TransportClearScrollAction;
 import org.elasticsearch.action.search.TransportMultiSearchAction;
-import org.elasticsearch.action.search.TransportOpenReaderAction;
+import org.elasticsearch.action.search.TransportOpenSearchContextAction;
 import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.search.TransportSearchScrollAction;
 import org.elasticsearch.action.support.ActionFilters;
@@ -367,12 +364,12 @@
 import org.elasticsearch.rest.action.ingest.RestGetPipelineAction;
 import org.elasticsearch.rest.action.ingest.RestPutPipelineAction;
 import org.elasticsearch.rest.action.ingest.RestSimulatePipelineAction;
-import org.elasticsearch.rest.action.search.RestClearReaderAction;
+import org.elasticsearch.rest.action.search.RestCloseSearchContextAction;
 import org.elasticsearch.rest.action.search.RestClearScrollAction;
 import org.elasticsearch.rest.action.search.RestCountAction;
 import org.elasticsearch.rest.action.search.RestExplainAction;
 import org.elasticsearch.rest.action.search.RestMultiSearchAction;
-import org.elasticsearch.rest.action.search.RestOpenReaderAction;
+import org.elasticsearch.rest.action.search.RestOpenSearchContextAction;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.rest.action.search.RestSearchScrollAction;
 import org.elasticsearch.tasks.Task;
@@ -575,8 +572,8 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(MultiSearchAction.INSTANCE, TransportMultiSearchAction.class);
         actions.register(ExplainAction.INSTANCE, TransportExplainAction.class);
         actions.register(ClearScrollAction.INSTANCE, TransportClearScrollAction.class);
-        actions.register(TransportOpenReaderAction.INSTANCE, TransportOpenReaderAction.class);
-        actions.register(ClearReaderAction.INSTANCE, TransportClearReaderAction.class);
+        actions.register(TransportOpenSearchContextAction.INSTANCE, TransportOpenSearchContextAction.class);
+        actions.register(CloseSearchContextAction.INSTANCE, TransportCloseSearchContextAction.class);
         actions.register(RecoveryAction.INSTANCE, TransportRecoveryAction.class);
         actions.register(NodesReloadSecureSettingsAction.INSTANCE, TransportNodesReloadSecureSettingsAction.class);
         actions.register(AutoCreateAction.INSTANCE, AutoCreateAction.TransportAction.class);
@@ -729,8 +726,8 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestSearchAction());
         registerHandler.accept(new RestSearchScrollAction());
         registerHandler.accept(new RestClearScrollAction());
-        registerHandler.accept(new RestOpenReaderAction());
-        registerHandler.accept(new RestClearReaderAction());
+        registerHandler.accept(new RestOpenSearchContextAction());
+        registerHandler.accept(new RestCloseSearchContextAction());
         registerHandler.accept(new RestMultiSearchAction(settings));
 
         registerHandler.accept(new RestValidateQueryAction());
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 77deb13df71cb..579ae8d362e23 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -533,13 +533,13 @@ protected final SearchResponse buildSearchResponse(InternalSearchResponse intern
                                                        ShardSearchFailure[] failures) {
         final Version minNodeVersion = clusterState.nodes().getMinNodeVersion();
         String scrollId = request.scroll() != null ? TransportSearchHelper.buildScrollId(queryResults, minNodeVersion) : null;
-        String readerId = includeReaderIdInResponse() ? TransportSearchHelper.encodeReaderIds(queryResults, minNodeVersion) : null;
+        String readerId = includeSearchContextInResponse() ? TransportSearchHelper.encodeReaderIds(queryResults, minNodeVersion) : null;
         return new SearchResponse(internalSearchResponse, scrollId, getNumShards(), successfulOps.get(),
             skippedOps.get(), buildTookInMillis(), failures, clusters, readerId);
     }
 
-    boolean includeReaderIdInResponse() {
-        return request.reader() != null;
+    boolean includeSearchContextInResponse() {
+        return request.searchContextBuilder() != null;
     }
 
     @Override
@@ -547,7 +547,7 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At
         ShardSearchFailure[] failures = buildShardFailures();
         Boolean allowPartialResults = request.allowPartialSearchResults();
         assert allowPartialResults != null : "SearchRequest missing setting for allowPartialSearchResults";
-        if (request.reader() == null && allowPartialResults == false && failures.length > 0) {
+        if (request.searchContextBuilder() == null && allowPartialResults == false && failures.length > 0) {
             raisePhaseFailure(new SearchPhaseExecutionException("", "Shard failures", null, failures));
         } else {
             listener.onResponse(buildSearchResponse(internalSearchResponse, queryResults, failures));
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearContextController.java b/server/src/main/java/org/elasticsearch/action/search/ClearContextController.java
index a9c0da94a8bc8..59a53eab6e50c 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearContextController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearContextController.java
@@ -40,7 +40,7 @@ final class ClearContextController implements Runnable {
     private final DiscoveryNodes nodes;
     private final SearchTransportService searchTransportService;
     private final CountDown expectedOps;
-    private final ActionListener<? super ClearReaderResponse> listener;
+    private final ActionListener<? super CloseSearchContextResponse> listener;
     private final AtomicBoolean hasFailed = new AtomicBoolean(false);
     private final AtomicInteger freedSearchContexts = new AtomicInteger(0);
     private final Logger logger;
@@ -68,14 +68,14 @@ final class ClearContextController implements Runnable {
         this.expectedOps = new CountDown(expectedOps);
     }
 
-    ClearContextController(ClearReaderRequest clearReaderRequest, ActionListener<ClearReaderResponse> listener,
+    ClearContextController(CloseSearchContextRequest closeSearchContextRequest, ActionListener<CloseSearchContextResponse> listener,
                            DiscoveryNodes nodes, Logger logger, SearchTransportService searchTransportService) {
         this.nodes = nodes;
         this.logger = logger;
         this.searchTransportService = searchTransportService;
         this.listener = listener;
         final Collection<ReaderIdForNode> contexts =
-            TransportSearchHelper.decodeReaderIds(clearReaderRequest.getId()).values();
+            TransportSearchHelper.decodeReaderIds(closeSearchContextRequest.getId()).values();
         expectedOps = new CountDown(contexts.size());
         runner = () -> cleanReaderIds(contexts);
     }
@@ -108,7 +108,7 @@ public void onFailure(Exception e) {
 
     void cleanReaderIds(Collection<ReaderIdForNode> readerIds) {
         if (readerIds.isEmpty()) {
-            listener.onResponse(new ClearReaderResponse(true, 0));
+            listener.onResponse(new CloseSearchContextResponse(true, 0));
             return;
         }
         SearchScrollAsyncAction.collectNodesAndRun(readerIds, nodes, searchTransportService, ActionListener.wrap(
@@ -136,7 +136,7 @@ private void onFreedContext(boolean freed) {
         }
         if (expectedOps.countDown()) {
             boolean succeeded = hasFailed.get() == false;
-            listener.onResponse(new ClearReaderResponse(succeeded, freedSearchContexts.get()));
+            listener.onResponse(new CloseSearchContextResponse(succeeded, freedSearchContexts.get()));
         }
     }
 
@@ -148,7 +148,7 @@ private void onFailedFreedContext(Throwable e, DiscoveryNode node) {
          */
         hasFailed.set(true);
         if (expectedOps.countDown()) {
-            listener.onResponse(new ClearReaderResponse(false, freedSearchContexts.get()));
+            listener.onResponse(new CloseSearchContextResponse(false, freedSearchContexts.get()));
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearReaderAction.java b/server/src/main/java/org/elasticsearch/action/search/CloseSearchContextAction.java
similarity index 71%
rename from server/src/main/java/org/elasticsearch/action/search/ClearReaderAction.java
rename to server/src/main/java/org/elasticsearch/action/search/CloseSearchContextAction.java
index d6215365e4aa9..3869d10019bbc 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearReaderAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CloseSearchContextAction.java
@@ -21,12 +21,12 @@
 
 import org.elasticsearch.action.ActionType;
 
-public class ClearReaderAction extends ActionType<ClearReaderResponse> {
+public class CloseSearchContextAction extends ActionType<CloseSearchContextResponse> {
 
-    public static final ClearReaderAction INSTANCE = new ClearReaderAction();
-    public static final String NAME = "indices:data/read/reader/clear";
+    public static final CloseSearchContextAction INSTANCE = new CloseSearchContextAction();
+    public static final String NAME = "indices:data/read/close_search_context";
 
-    private ClearReaderAction() {
-        super(NAME, ClearReaderResponse::new);
+    private CloseSearchContextAction() {
+        super(NAME, CloseSearchContextResponse::new);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearReaderRequest.java b/server/src/main/java/org/elasticsearch/action/search/CloseSearchContextRequest.java
similarity index 87%
rename from server/src/main/java/org/elasticsearch/action/search/ClearReaderRequest.java
rename to server/src/main/java/org/elasticsearch/action/search/CloseSearchContextRequest.java
index 75f85c3024d13..0d8a24f8a708d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearReaderRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CloseSearchContextRequest.java
@@ -31,17 +31,17 @@
 
 import java.io.IOException;
 
-public class ClearReaderRequest extends ActionRequest implements ToXContentObject {
+public class CloseSearchContextRequest extends ActionRequest implements ToXContentObject {
     private static final ParseField ID = new ParseField("id");
 
     private final String id;
 
-    public ClearReaderRequest(StreamInput in) throws IOException {
+    public CloseSearchContextRequest(StreamInput in) throws IOException {
         super(in);
         this.id = in.readString();
     }
 
-    public ClearReaderRequest(String id) {
+    public CloseSearchContextRequest(String id) {
         this.id = id;
     }
 
@@ -71,7 +71,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static ClearReaderRequest fromXContent(XContentParser parser) throws IOException {
+    public static CloseSearchContextRequest fromXContent(XContentParser parser) throws IOException {
         if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
             throw new IllegalArgumentException("Malformed content, must start with an object");
         } else {
@@ -90,9 +90,9 @@ public static ClearReaderRequest fromXContent(XContentParser parser) throws IOEx
                 }
             }
             if (Strings.isNullOrEmpty(id)) {
-                throw new IllegalArgumentException("reader context is is not provided");
+                throw new IllegalArgumentException("search context id is is not provided");
             }
-            return new ClearReaderRequest(id);
+            return new CloseSearchContextRequest(id);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearReaderResponse.java b/server/src/main/java/org/elasticsearch/action/search/CloseSearchContextResponse.java
similarity index 81%
rename from server/src/main/java/org/elasticsearch/action/search/ClearReaderResponse.java
rename to server/src/main/java/org/elasticsearch/action/search/CloseSearchContextResponse.java
index 6efdc11bdb0d2..a015c00331fd8 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearReaderResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CloseSearchContextResponse.java
@@ -23,12 +23,12 @@
 
 import java.io.IOException;
 
-public class ClearReaderResponse extends ClearScrollResponse {
-    public ClearReaderResponse(boolean succeeded, int numFreed) {
+public class CloseSearchContextResponse extends ClearScrollResponse {
+    public CloseSearchContextResponse(boolean succeeded, int numFreed) {
         super(succeeded, numFreed);
     }
 
-    public ClearReaderResponse(StreamInput in) throws IOException {
+    public CloseSearchContextResponse(StreamInput in) throws IOException {
         super(in);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index a9c4587823862..2f1f2cee1ccc4 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -208,7 +208,9 @@ public void onFailure(Exception e) {
     private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) {
         // we only release search context that we did not fetch from if we are not scrolling
         // and if it has at lease one hit that didn't make it to the global topDocs
-        if (context.getRequest().scroll() == null &&  context.getRequest().reader() == null && queryResult.hasSearchContext()) {
+        if (context.getRequest().scroll() == null &&
+            context.getRequest().searchContextBuilder() == null &&
+            queryResult.hasSearchContext()) {
             try {
                 SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget();
                 Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenReaderRequest.java b/server/src/main/java/org/elasticsearch/action/search/OpenSearchContextRequest.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/action/search/OpenReaderRequest.java
rename to server/src/main/java/org/elasticsearch/action/search/OpenSearchContextRequest.java
index c911b3124af66..5134568c8158a 100644
--- a/server/src/main/java/org/elasticsearch/action/search/OpenReaderRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/OpenSearchContextRequest.java
@@ -36,7 +36,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public final class OpenReaderRequest extends ActionRequest implements IndicesRequest {
+public final class OpenSearchContextRequest extends ActionRequest implements IndicesRequest {
     private final String[] indices;
     private final IndicesOptions indicesOptions;
     private final TimeValue keepAlive;
@@ -48,7 +48,8 @@ public final class OpenReaderRequest extends ActionRequest implements IndicesReq
 
     public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed();
 
-    public OpenReaderRequest(String[] indices, IndicesOptions indicesOptions, TimeValue keepAlive, String routing, String preference) {
+    public OpenSearchContextRequest(String[] indices, IndicesOptions indicesOptions,
+                                    TimeValue keepAlive, String routing, String preference) {
         this.indices = Objects.requireNonNull(indices);
         this.indicesOptions = Objects.requireNonNull(indicesOptions);
         this.keepAlive = keepAlive;
@@ -56,7 +57,7 @@ public OpenReaderRequest(String[] indices, IndicesOptions indicesOptions, TimeVa
         this.preference = preference;
     }
 
-    public OpenReaderRequest(StreamInput in) throws IOException {
+    public OpenSearchContextRequest(StreamInput in) throws IOException {
         super(in);
         this.indices = in.readStringArray();
         this.indicesOptions = IndicesOptions.readIndicesOptions(in);
@@ -114,7 +115,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
         return new SearchTask(id, type, action, null, parentTaskId, headers) {
             @Override
             public String getDescription() {
-                return "open reader: indices [" + String.join(",", indices) + "] keep_alive [" + keepAlive + "]";
+                return "open search context: indices [" + String.join(",", indices) + "] keep_alive [" + keepAlive + "]";
             }
         };
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenReaderResponse.java b/server/src/main/java/org/elasticsearch/action/search/OpenSearchContextResponse.java
similarity index 71%
rename from server/src/main/java/org/elasticsearch/action/search/OpenReaderResponse.java
rename to server/src/main/java/org/elasticsearch/action/search/OpenSearchContextResponse.java
index 43114f7d14f19..7ce3061b4f858 100644
--- a/server/src/main/java/org/elasticsearch/action/search/OpenReaderResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/OpenSearchContextResponse.java
@@ -28,34 +28,34 @@
 
 import java.io.IOException;
 
-public final class OpenReaderResponse extends ActionResponse implements ToXContentObject {
-    private static final ParseField READER_ID = new ParseField("reader_id");
+public final class OpenSearchContextResponse extends ActionResponse implements ToXContentObject {
+    private static final ParseField ID = new ParseField("id");
 
-    private final String readerId;
+    private final String searchContextId;
 
-    public OpenReaderResponse(String readerId) {
-        this.readerId = readerId;
+    public OpenSearchContextResponse(String searchContextId) {
+        this.searchContextId = searchContextId;
     }
 
-    public OpenReaderResponse(StreamInput in) throws IOException {
+    public OpenSearchContextResponse(StreamInput in) throws IOException {
         super(in);
-        readerId = in.readString();
+        searchContextId = in.readString();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(readerId);
+        out.writeString(searchContextId);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(READER_ID.getPreferredName(), readerId);
+        builder.field(ID.getPreferredName(), searchContextId);
         builder.endObject();
         return builder;
     }
 
-    public String getReaderId() {
-        return readerId;
+    public String getSearchContextId() {
+        return searchContextId;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index a6a2a47bbf31b..816d9efbf8a79 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -277,7 +277,7 @@ public ActionRequestValidationException validate() {
                 validationException =
                     addValidationError("[request_cache] cannot be used in a scroll context", validationException);
             }
-            if (reader() != null) {
+            if (searchContextBuilder() != null) {
                 validationException = addValidationError("using [reader] is not allowed in a scroll context", validationException);
             }
         }
@@ -286,7 +286,7 @@ public ActionRequestValidationException validate() {
                 validationException = source.aggregations().validate(validationException);
             }
         }
-        if (reader() != null) {
+        if (searchContextBuilder() != null) {
             if (indices.length > 0) {
                 validationException = addValidationError("[index] cannot be used with reader contexts", validationException);
             }
@@ -446,9 +446,9 @@ public SearchSourceBuilder source() {
         return source;
     }
 
-    public SearchSourceBuilder.ReaderBuilder reader() {
+    public SearchSourceBuilder.SearchContextBuilder searchContextBuilder() {
         if (source != null) {
-            return source.reader();
+            return source.searchContextBuilder();
         }
         return null;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 6a9f1e0c1e389..05f1cab42f902 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -528,7 +528,7 @@ public SearchRequestBuilder setCollapse(CollapseBuilder collapse) {
      * @param keepAlive the extended keep alive for the readers used in this request
      */
     public SearchRequestBuilder setReader(@Nullable String readerId, @Nullable TimeValue keepAlive) {
-        sourceBuilder().reader(new SearchSourceBuilder.ReaderBuilder(readerId, keepAlive));
+        sourceBuilder().searchContextBuilder(new SearchSourceBuilder.SearchContextBuilder(readerId, keepAlive));
         return this;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 8db0abf3fa574..482c846dbed55 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -70,7 +70,7 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
 
     private final SearchResponseSections internalResponse;
     private final String scrollId;
-    private final String readerId;
+    private final String searchContextId;
     private final int totalShards;
     private final int successfulShards;
     private final int skippedShards;
@@ -97,17 +97,18 @@ public SearchResponse(StreamInput in) throws IOException {
         tookInMillis = in.readVLong();
         skippedShards = in.readVInt();
         if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
-            readerId = in.readOptionalString();
+            searchContextId = in.readOptionalString();
         } else {
-            readerId = null;
+            searchContextId = null;
         }
     }
 
     public SearchResponse(SearchResponseSections internalResponse, String scrollId, int totalShards, int successfulShards,
-                          int skippedShards, long tookInMillis, ShardSearchFailure[] shardFailures, Clusters clusters, String readerId) {
+                          int skippedShards, long tookInMillis, ShardSearchFailure[] shardFailures, Clusters clusters,
+                          String searchContextId) {
         this.internalResponse = internalResponse;
         this.scrollId = scrollId;
-        this.readerId = readerId;
+        this.searchContextId = searchContextId;
         this.clusters = clusters;
         this.totalShards = totalShards;
         this.successfulShards = successfulShards;
@@ -115,8 +116,8 @@ public SearchResponse(SearchResponseSections internalResponse, String scrollId,
         this.tookInMillis = tookInMillis;
         this.shardFailures = shardFailures;
         assert skippedShards <= totalShards : "skipped: " + skippedShards + " total: " + totalShards;
-        assert scrollId == null || readerId == null :
-            "SearchResponse can't have both scrollId [" + scrollId + "] and readerId [" + readerId + "]";
+        assert scrollId == null || searchContextId == null :
+            "SearchResponse can't have both scrollId [" + scrollId + "] and searchContextId [" + searchContextId + "]";
     }
 
     @Override
@@ -219,10 +220,10 @@ public String getScrollId() {
     }
 
     /**
-     * Returns the encoded string representing reader ids that ared used to executed the request.
+     * Returns the encoded string of the search context that the search request is used to executed
      */
-    public String getReaderId() {
-        return readerId;
+    public String searchContextId() {
+        return searchContextId;
     }
 
     /**
@@ -257,8 +258,8 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         if (scrollId != null) {
             builder.field(SCROLL_ID.getPreferredName(), scrollId);
         }
-        if (readerId != null) {
-            builder.field(READER_ID.getPreferredName(), readerId);
+        if (searchContextId != null) {
+            builder.field(READER_ID.getPreferredName(), searchContextId);
         }
         builder.field(TOOK.getPreferredName(), tookInMillis);
         builder.field(TIMED_OUT.getPreferredName(), isTimedOut());
@@ -403,7 +404,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(tookInMillis);
         out.writeVInt(skippedShards);
         if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
-            out.writeOptionalString(readerId);
+            out.writeOptionalString(searchContextId);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 38a0288c5e2f0..7156e6ee840c3 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -170,10 +170,10 @@ public void sendExecuteFetchScroll(Transport.Connection connection, final ShardF
     }
 
     void sendShardOpenReader(Transport.Connection connection, SearchTask task,
-                             TransportOpenReaderAction.ShardOpenReaderRequest request,
-                             ActionListener<TransportOpenReaderAction.ShardOpenReaderResponse> listener) {
+                             TransportOpenSearchContextAction.ShardOpenReaderRequest request,
+                             ActionListener<TransportOpenSearchContextAction.ShardOpenReaderResponse> listener) {
         transportService.sendChildRequest(connection, SHARD_OPEN_READER_NAME, request, task,
-            new ActionListenerResponseHandler<>(listener, TransportOpenReaderAction.ShardOpenReaderResponse::new));
+            new ActionListenerResponseHandler<>(listener, TransportOpenSearchContextAction.ShardOpenReaderResponse::new));
     }
 
     private void sendExecuteFetch(Transport.Connection connection, String action, final ShardFetchRequest request, SearchTask task,
@@ -372,14 +372,14 @@ public static void registerRequestHandler(TransportService transportService, Sea
         TransportActionProxy.registerProxyAction(transportService, QUERY_CAN_MATCH_NAME, SearchService.CanMatchResponse::new);
 
         transportService.registerRequestHandler(SHARD_OPEN_READER_NAME, ThreadPool.Names.SAME,
-            TransportOpenReaderAction.ShardOpenReaderRequest::new,
+            TransportOpenSearchContextAction.ShardOpenReaderRequest::new,
             (request, channel, task) -> {
                 searchService.openReaderContext(request.getShardId(), request.keepAlive,
                     ActionListener.map(new ChannelActionListener<>(channel, SHARD_OPEN_READER_NAME, request),
-                        contextId -> new TransportOpenReaderAction.ShardOpenReaderResponse(contextId)));
+                        contextId -> new TransportOpenSearchContextAction.ShardOpenReaderResponse(contextId)));
             });
         TransportActionProxy.registerProxyAction(
-            transportService, SHARD_OPEN_READER_NAME, TransportOpenReaderAction.ShardOpenReaderResponse::new);
+            transportService, SHARD_OPEN_READER_NAME, TransportOpenSearchContextAction.ShardOpenReaderResponse::new);
     }
 
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportClearReaderAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java
similarity index 72%
rename from server/src/main/java/org/elasticsearch/action/search/TransportClearReaderAction.java
rename to server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java
index eb0e586a1c774..23311624961bf 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportClearReaderAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java
@@ -27,21 +27,21 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
-public class TransportClearReaderAction extends HandledTransportAction<ClearReaderRequest, ClearReaderResponse> {
+public class TransportCloseSearchContextAction extends HandledTransportAction<CloseSearchContextRequest, CloseSearchContextResponse> {
 
     private final ClusterService clusterService;
     private final SearchTransportService searchTransportService;
 
     @Inject
-    public TransportClearReaderAction(TransportService transportService, ClusterService clusterService,
-                                      ActionFilters actionFilters, SearchTransportService searchTransportService) {
-        super(ClearReaderAction.NAME, transportService, actionFilters, ClearReaderRequest::new);
+    public TransportCloseSearchContextAction(TransportService transportService, ClusterService clusterService,
+                                             ActionFilters actionFilters, SearchTransportService searchTransportService) {
+        super(CloseSearchContextAction.NAME, transportService, actionFilters, CloseSearchContextRequest::new);
         this.clusterService = clusterService;
         this.searchTransportService = searchTransportService;
     }
 
     @Override
-    protected void doExecute(Task task, ClearReaderRequest request, ActionListener<ClearReaderResponse> listener) {
+    protected void doExecute(Task task, CloseSearchContextRequest request, ActionListener<CloseSearchContextResponse> listener) {
         Runnable runnable = new ClearContextController(
             request, listener, clusterService.state().nodes(), logger, searchTransportService);
         runnable.run();
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java
similarity index 81%
rename from server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java
rename to server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java
index b827610893617..bc004968e0213 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenReaderAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java
@@ -53,23 +53,23 @@
 import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
 
-public class TransportOpenReaderAction extends HandledTransportAction<OpenReaderRequest, OpenReaderResponse> {
-    public static final String NAME = "indices:data/read/open_reader";
-    public static final ActionType<OpenReaderResponse> INSTANCE = new ActionType<>(NAME, OpenReaderResponse::new);
+public class TransportOpenSearchContextAction extends HandledTransportAction<OpenSearchContextRequest, OpenSearchContextResponse> {
+    public static final String NAME = "indices:data/read/open_search_context";
+    public static final ActionType<OpenSearchContextResponse> INSTANCE = new ActionType<>(NAME, OpenSearchContextResponse::new);
 
     private final TransportSearchAction transportSearchAction;
     private final SearchTransportService searchTransportService;
 
     @Inject
-    public TransportOpenReaderAction(TransportService transportService, SearchTransportService searchTransportService,
-                                     ActionFilters actionFilters, TransportSearchAction transportSearchAction) {
-        super(NAME, transportService, actionFilters, OpenReaderRequest::new);
+    public TransportOpenSearchContextAction(TransportService transportService, SearchTransportService searchTransportService,
+                                            ActionFilters actionFilters, TransportSearchAction transportSearchAction) {
+        super(NAME, transportService, actionFilters, OpenSearchContextRequest::new);
         this.transportSearchAction = transportSearchAction;
         this.searchTransportService = searchTransportService;
     }
 
     @Override
-    protected void doExecute(Task task, OpenReaderRequest openReaderRequest, ActionListener<OpenReaderResponse> listener) {
+    protected void doExecute(Task task, OpenSearchContextRequest request, ActionListener<OpenSearchContextResponse> listener) {
         final TransportSearchAction.SearchAsyncActionProvider actionProvider = new TransportSearchAction.SearchAsyncActionProvider() {
             @Override
             public AbstractSearchAsyncAction<? extends SearchPhaseResult> asyncSearchAction(
@@ -79,19 +79,19 @@ public AbstractSearchAsyncAction<? extends SearchPhaseResult> asyncSearchAction(
                 Map<String, Set<String>> indexRoutings, ActionListener<SearchResponse> listener, boolean preFilter,
                 ThreadPool threadPool, SearchResponse.Clusters clusters) {
                 final Executor executor = threadPool.executor(ThreadPool.Names.SEARCH);
-                return new OpenReaderSearchPhase(openReaderRequest, logger, searchTransportService, connectionLookup,
+                return new OpenReaderSearchPhase(request, logger, searchTransportService, connectionLookup,
                     aliasFilter, concreteIndexBoosts, indexRoutings, executor, searchRequest, listener, shardIterators,
                     timeProvider, clusterState, task, clusters);
             }
         };
         final SearchRequest searchRequest = new SearchRequest()
-            .indices(openReaderRequest.indices())
-            .indicesOptions(openReaderRequest.indicesOptions())
-            .preference(openReaderRequest.preference())
-            .routing(openReaderRequest.routing())
+            .indices(request.indices())
+            .indicesOptions(request.indicesOptions())
+            .preference(request.preference())
+            .routing(request.routing())
             .allowPartialSearchResults(false);
         transportSearchAction.executeRequest(task, searchRequest, actionProvider,
-            ActionListener.map(listener, r -> new OpenReaderResponse(r.getReaderId())));
+            ActionListener.map(listener, r -> new OpenSearchContextResponse(r.searchContextId())));
     }
 
     static final class ShardOpenReaderRequest extends TransportRequest implements IndicesRequest {
@@ -152,19 +152,19 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     static final class OpenReaderSearchPhase extends AbstractSearchAsyncAction<SearchPhaseResult> {
-        final OpenReaderRequest openReaderRequest;
+        final OpenSearchContextRequest request;
 
-        OpenReaderSearchPhase(OpenReaderRequest openReaderRequest, Logger logger, SearchTransportService searchTransportService,
+        OpenReaderSearchPhase(OpenSearchContextRequest request, Logger logger, SearchTransportService searchTransportService,
                               BiFunction<String, String, Transport.Connection> nodeIdToConnection,
                               Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
-                              Map<String, Set<String>> indexRoutings, Executor executor, SearchRequest request,
+                              Map<String, Set<String>> indexRoutings, Executor executor, SearchRequest searchRequest,
                               ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts,
                               TransportSearchAction.SearchTimeProvider timeProvider, ClusterState clusterState,
                               SearchTask task, SearchResponse.Clusters clusters) {
-            super("open_reader", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings,
-                executor, request, listener, shardsIts, timeProvider, clusterState, task,
+            super("open_search_context", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts,
+                indexRoutings, executor, searchRequest, listener, shardsIts, timeProvider, clusterState, task,
                 new ArraySearchPhaseResults<>(shardsIts.size()), shardsIts.size(), clusters);
-            this.openReaderRequest = openReaderRequest;
+            this.request = request;
         }
 
         @Override
@@ -173,7 +173,7 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
             final Transport.Connection connection = getConnection(shardIt.getClusterAlias(), shard.currentNodeId());
             final SearchShardTarget searchShardTarget = shardIt.newSearchShardTarget(shard.currentNodeId());
             final ShardOpenReaderRequest shardRequest = new ShardOpenReaderRequest(searchShardTarget.getShardId(),
-                searchShardTarget.getOriginalIndices(), openReaderRequest.keepAlive());
+                searchShardTarget.getOriginalIndices(), request.keepAlive());
             getSearchTransport().sendShardOpenReader(connection, getTask(), shardRequest, ActionListener.map(listener, r -> r));
         }
 
@@ -189,7 +189,7 @@ public void run() {
         }
 
         @Override
-        boolean includeReaderIdInResponse() {
+        boolean includeSearchContextInResponse() {
             return true;
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index edb0f361c6afc..9d683131d311f 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -216,8 +216,8 @@ void executeRequest(Task task, SearchRequest searchRequest,
             }
             final Map<ShardId, ReaderIdForNode> readerContexts;
             final Map<String, OriginalIndices> remoteClusterIndices;
-            if (searchRequest.reader() != null) {
-                readerContexts = TransportSearchHelper.decodeReaderIds(searchRequest.reader().getId());
+            if (searchRequest.searchContextBuilder() != null) {
+                readerContexts = TransportSearchHelper.decodeReaderIds(searchRequest.searchContextBuilder().getId());
                 remoteClusterIndices = indicesFromReaderContexts(readerContexts, searchRequest.indicesOptions());
             } else {
                 readerContexts = null;
@@ -272,7 +272,7 @@ static boolean shouldMinimizeRoundtrips(SearchRequest searchRequest) {
         if (searchRequest.scroll() != null) {
             return false;
         }
-        if (searchRequest.reader() != null) {
+        if (searchRequest.searchContextBuilder() != null) {
             // TODO: support reader contexts
             return false;
         }
@@ -311,7 +311,7 @@ public void onResponse(SearchResponse searchResponse) {
                     listener.onResponse(new SearchResponse(internalSearchResponse, searchResponse.getScrollId(),
                         searchResponse.getTotalShards(), searchResponse.getSuccessfulShards(), searchResponse.getSkippedShards(),
                         timeProvider.buildTookInMillis(), searchResponse.getShardFailures(), new SearchResponse.Clusters(1, 1, 0),
-                        searchResponse.getReaderId()));
+                        searchResponse.searchContextId()));
                 }
 
                 @Override
@@ -509,14 +509,14 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
 
         boolean preFilterSearchShards;
         if (readerContexts != null) {
-            assert searchRequest.reader() != null;
+            assert searchRequest.searchContextBuilder() != null;
             aliasFilter = new HashMap<>();
             for (ShardId shardId : readerContexts.keySet()) {
                 aliasFilter.put(shardId.getIndex().getUUID(), AliasFilter.EMPTY);
             }
             indexRoutings = Map.of();
-            localShardIterators = searchShardsFromReaderContexts(
-                clusterState, localIndices, searchRequest.getLocalClusterAlias(), readerContexts, searchRequest.reader().getKeepAlive());
+            localShardIterators = searchShardsFromReaderContexts(clusterState, localIndices, searchRequest.getLocalClusterAlias(),
+                readerContexts, searchRequest.searchContextBuilder().getKeepAlive());
             preFilterSearchShards = shouldPreFilterSearchShards(clusterState, searchRequest, localIndices.indices(),
                 localShardIterators.size() + remoteShardIterators.size());
         } else {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestClearReaderAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestCloseSearchContextAction.java
similarity index 73%
rename from server/src/main/java/org/elasticsearch/rest/action/search/RestClearReaderAction.java
rename to server/src/main/java/org/elasticsearch/rest/action/search/RestCloseSearchContextAction.java
index c672ad821778f..6a76559789fae 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestClearReaderAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestCloseSearchContextAction.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.rest.action.search;
 
-import org.elasticsearch.action.search.ClearReaderAction;
-import org.elasticsearch.action.search.ClearReaderRequest;
+import org.elasticsearch.action.search.CloseSearchContextAction;
+import org.elasticsearch.action.search.CloseSearchContextRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -32,24 +32,24 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
-public class RestClearReaderAction extends BaseRestHandler {
+public class RestCloseSearchContextAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return List.of(new Route(DELETE, "/_search/reader"));
+        return List.of(new Route(DELETE, "/_search_context"));
     }
 
     @Override
     public String getName() {
-        return "clear_reader_action";
+        return "close_search_context";
     }
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        final ClearReaderRequest clearRequest;
+        final CloseSearchContextRequest clearRequest;
         try (XContentParser parser = request.contentOrSourceParamParser()) {
-            clearRequest = ClearReaderRequest.fromXContent(parser);
+            clearRequest = CloseSearchContextRequest.fromXContent(parser);
         }
-        return channel -> client.execute(ClearReaderAction.INSTANCE, clearRequest, new RestStatusToXContentListener<>(channel));
+        return channel -> client.execute(CloseSearchContextAction.INSTANCE, clearRequest, new RestStatusToXContentListener<>(channel));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestOpenReaderAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestOpenSearchContextAction.java
similarity index 74%
rename from server/src/main/java/org/elasticsearch/rest/action/search/RestOpenReaderAction.java
rename to server/src/main/java/org/elasticsearch/rest/action/search/RestOpenSearchContextAction.java
index 7294347fc1e1e..e992efe8eb639 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestOpenReaderAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestOpenSearchContextAction.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.rest.action.search;
 
-import org.elasticsearch.action.search.OpenReaderRequest;
-import org.elasticsearch.action.search.TransportOpenReaderAction;
+import org.elasticsearch.action.search.OpenSearchContextRequest;
+import org.elasticsearch.action.search.TransportOpenSearchContextAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
@@ -34,26 +34,26 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-public class RestOpenReaderAction extends BaseRestHandler {
+public class RestOpenSearchContextAction extends BaseRestHandler {
 
     @Override
     public String getName() {
-        return "open_reader_action";
+        return "open_search_context";
     }
 
     @Override
     public List<Route> routes() {
-        return List.of(new Route(POST, "/{index}/_open_reader"));
+        return List.of(new Route(POST, "/{index}/_search_context"));
     }
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        final IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, OpenReaderRequest.DEFAULT_INDICES_OPTIONS);
+        final IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, OpenSearchContextRequest.DEFAULT_INDICES_OPTIONS);
         final String routing = request.param("routing");
         final String preference = request.param("preference");
         final TimeValue keepAlive = TimeValue.parseTimeValue(request.param("keep_alive"), null, "keep_alive");
-        final OpenReaderRequest openReaderRequest = new OpenReaderRequest(indices, indicesOptions, keepAlive, routing, preference);
-        return channel -> client.execute(TransportOpenReaderAction.INSTANCE, openReaderRequest, new RestToXContentListener<>(channel));
+        final OpenSearchContextRequest openRequest = new OpenSearchContextRequest(indices, indicesOptions, keepAlive, routing, preference);
+        return channel -> client.execute(TransportOpenSearchContextAction.INSTANCE, openRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 1b20917e785f3..4fa78400cc1bc 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -114,7 +114,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R
     public static final ParseField SEARCH_AFTER = new ParseField("search_after");
     public static final ParseField COLLAPSE = new ParseField("collapse");
     public static final ParseField SLICE = new ParseField("slice");
-    public static final ParseField READER_CONTEXT = new ParseField("reader");
+    public static final ParseField SEARCH_CONTEXT = new ParseField("search_context");
 
     public static SearchSourceBuilder fromXContent(XContentParser parser) throws IOException {
         return fromXContent(parser, true);
@@ -193,7 +193,7 @@ public static HighlightBuilder highlight() {
 
     private CollapseBuilder collapse = null;
 
-    private ReaderBuilder reader = null;
+    private SearchContextBuilder searchContextBuilder = null;
 
     /**
      * Constructs a new search source builder.
@@ -250,7 +250,7 @@ public SearchSourceBuilder(StreamInput in) throws IOException {
         collapse = in.readOptionalWriteable(CollapseBuilder::new);
         trackTotalHitsUpTo = in.readOptionalInt();
         if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
-            reader = in.readOptionalWriteable(ReaderBuilder::new);
+            searchContextBuilder = in.readOptionalWriteable(SearchContextBuilder::new);
         }
     }
 
@@ -307,7 +307,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(collapse);
         out.writeOptionalInt(trackTotalHitsUpTo);
         if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
-            out.writeOptionalWriteable(reader);
+            out.writeOptionalWriteable(searchContextBuilder);
         }
     }
 
@@ -928,17 +928,17 @@ public boolean isSuggestOnly() {
     }
 
     /**
-     * Returns the reader context that is configured with this query
+     * Returns the search context that is configured with this query
      */
-    public ReaderBuilder reader() {
-        return reader;
+    public SearchContextBuilder searchContextBuilder() {
+        return searchContextBuilder;
     }
 
     /**
-     * Specify the reader context that this query should use to execute.
+     * Specify a search context that this query should execute against.
      */
-    public SearchSourceBuilder reader(ReaderBuilder reader) {
-        this.reader = reader;
+    public SearchSourceBuilder searchContextBuilder(SearchContextBuilder reader) {
+        this.searchContextBuilder = reader;
         return this;
     }
 
@@ -1026,7 +1026,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder
         rewrittenBuilder.version = version;
         rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm;
         rewrittenBuilder.collapse = collapse;
-        rewrittenBuilder.reader = reader;
+        rewrittenBuilder.searchContextBuilder = searchContextBuilder;
         return rewrittenBuilder;
     }
 
@@ -1135,8 +1135,9 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th
                     sliceBuilder = SliceBuilder.fromXContent(parser);
                 } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) {
                     collapse = CollapseBuilder.fromXContent(parser);
-                } else if (READER_CONTEXT.match(currentFieldName, parser.getDeprecationHandler())) {
-                    reader = ReaderBuilder.fromXContent(parser);
+                } else if (SEARCH_CONTEXT.match(currentFieldName, parser.getDeprecationHandler())) {
+                    searchContextBuilder = SearchContextBuilder.
+                        fromXContent(parser);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -1333,8 +1334,8 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         if (collapse != null) {
             builder.field(COLLAPSE.getPreferredName(), collapse);
         }
-        if (reader != null) {
-            builder.field(READER_CONTEXT.getPreferredName(), reader);
+        if (searchContextBuilder != null) {
+            builder.field(SEARCH_CONTEXT.getPreferredName(), searchContextBuilder);
         }
         return builder;
     }
@@ -1548,7 +1549,7 @@ public int hashCode() {
         return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder,
                 indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size,
                 sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version,
-                seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, reader);
+                seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, searchContextBuilder);
     }
 
     @Override
@@ -1588,7 +1589,7 @@ public boolean equals(Object obj) {
                 && Objects.equals(extBuilders, other.extBuilders)
                 && Objects.equals(collapse, other.collapse)
                 && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo)
-                && Objects.equals(reader, other.reader);
+                && Objects.equals(searchContextBuilder, other.searchContextBuilder);
     }
 
     @Override
@@ -1607,13 +1608,13 @@ public String toString(Params params) {
     /**
      * Specify whether this search should use specific reader contexts instead of the latest ones.
      */
-    public static final class ReaderBuilder implements Writeable, ToXContentObject {
+    public static final class SearchContextBuilder implements Writeable, ToXContentObject {
         private static final ParseField ID_FIELD = new ParseField("id");
         private static final ParseField KEEP_ALIVE_FIELD = new ParseField("keep_alive");
         private static final ObjectParser<XContentParams, Void> PARSER;
 
         static {
-            PARSER = new ObjectParser<>(READER_CONTEXT.getPreferredName(), XContentParams::new);
+            PARSER = new ObjectParser<>(SEARCH_CONTEXT.getPreferredName(), XContentParams::new);
             PARSER.declareString((params, id) -> params.id = id, ID_FIELD);
             PARSER.declareField((params, keepAlive) -> params.keepAlive = keepAlive,
                 (p, c) -> TimeValue.parseTimeValue(p.text(), KEEP_ALIVE_FIELD.getPreferredName()),
@@ -1628,12 +1629,12 @@ private static final class XContentParams {
         private final String id;
         private final TimeValue keepAlive;
 
-        public ReaderBuilder(String id, TimeValue keepAlive) {
+        public SearchContextBuilder(String id, TimeValue keepAlive) {
             this.id = Objects.requireNonNull(id);
             this.keepAlive = Objects.requireNonNull(keepAlive);
         }
 
-        public ReaderBuilder(StreamInput in) throws IOException {
+        public SearchContextBuilder(StreamInput in) throws IOException {
             id = in.readString();
             keepAlive = in.readTimeValue();
         }
@@ -1651,12 +1652,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             return builder;
         }
 
-        public static ReaderBuilder fromXContent(XContentParser parser) throws IOException {
+        public static SearchContextBuilder fromXContent(XContentParser parser) throws IOException {
             final XContentParams params = PARSER.parse(parser, null);
             if (params.id == null || params.keepAlive == null) {
                 throw new IllegalArgumentException("id and keep_alive must be specified");
             }
-            return new ReaderBuilder(params.id, params.keepAlive);
+            return new SearchContextBuilder(params.id, params.keepAlive);
         }
 
         public TimeValue getKeepAlive() {
@@ -1671,7 +1672,7 @@ public String getId() {
         public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
-            final ReaderBuilder that = (ReaderBuilder) o;
+            final SearchContextBuilder that = (SearchContextBuilder) o;
             return Objects.equals(id, that.id) && Objects.equals(keepAlive, that.keepAlive);
         }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
index 21b6363c8015d..b0ddabe498153 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
@@ -88,9 +88,10 @@ public SearchRequest getRequest() {
     @Override
     public void sendSearchResponse(InternalSearchResponse internalSearchResponse, AtomicArray<SearchPhaseResult> queryResults) {
         String scrollId = getRequest().scroll() != null ? TransportSearchHelper.buildScrollId(queryResults, Version.CURRENT) : null;
-        String readerContextId = getRequest().reader() != null ? TransportSearchHelper.buildScrollId(queryResults, Version.CURRENT) : null;
+        String searchContextId =
+            getRequest().searchContextBuilder() != null ? TransportSearchHelper.buildScrollId(queryResults, Version.CURRENT) : null;
         searchResponse.set(new SearchResponse(internalSearchResponse, scrollId, numShards, numSuccess.get(), 0, 0,
-            failures.toArray(ShardSearchFailure.EMPTY_ARRAY), SearchResponse.Clusters.EMPTY, readerContextId));
+            failures.toArray(ShardSearchFailure.EMPTY_ARRAY), SearchResponse.Clusters.EMPTY, searchContextId));
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
index 6f99c7c981fa7..e6e5f2af93f20 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
@@ -167,8 +167,8 @@ public void testValidate() throws IOException {
         {
             // Reader context with scroll
             SearchRequest searchRequest = new SearchRequest()
-                .source(new SearchSourceBuilder()
-                    .reader(new SearchSourceBuilder.ReaderBuilder("id", TimeValue.timeValueMillis(randomIntBetween(1, 10)))))
+                .source(new SearchSourceBuilder().searchContextBuilder(
+                    new SearchSourceBuilder.SearchContextBuilder("id", TimeValue.timeValueMillis(randomIntBetween(1, 10)))))
                 .scroll(TimeValue.timeValueMillis(randomIntBetween(1, 100)));
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNotNull(validationErrors);
@@ -179,7 +179,7 @@ public void testValidate() throws IOException {
             // Reader context with indices
             SearchRequest searchRequest = new SearchRequest()
                 .source(new SearchSourceBuilder()
-                    .reader(new SearchSourceBuilder.ReaderBuilder("id", TimeValue.timeValueMillis(randomIntBetween(1, 10)))))
+                    .searchContextBuilder(new SearchSourceBuilder.SearchContextBuilder("id", TimeValue.timeValueMillis(between(1, 10)))))
                 .indices("test");
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNotNull(validationErrors);
@@ -189,8 +189,8 @@ public void testValidate() throws IOException {
         {
             // Reader context with preference
             SearchRequest searchRequest = new SearchRequest()
-                .source(new SearchSourceBuilder()
-                    .reader(new SearchSourceBuilder.ReaderBuilder("id", TimeValue.timeValueMillis(randomIntBetween(1, 10)))))
+                .source(new SearchSourceBuilder().
+                    searchContextBuilder(new SearchSourceBuilder.SearchContextBuilder("id", TimeValue.timeValueMillis(between(1, 10)))))
                 .preference("test");
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNotNull(validationErrors);
@@ -201,7 +201,7 @@ public void testValidate() throws IOException {
             // Reader context with routing
             SearchRequest searchRequest = new SearchRequest()
                 .source(new SearchSourceBuilder()
-                    .reader(new SearchSourceBuilder.ReaderBuilder("id", TimeValue.timeValueMillis(randomIntBetween(1, 10)))))
+                    .searchContextBuilder(new SearchSourceBuilder.SearchContextBuilder("id", TimeValue.timeValueMillis(between(1, 10)))))
                 .routing("test");
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNotNull(validationErrors);
diff --git a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
index dd6d22342990c..bf86bb90631fb 100644
--- a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
+++ b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
@@ -11,14 +11,14 @@
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.search.ClearReaderAction;
-import org.elasticsearch.action.search.ClearReaderRequest;
-import org.elasticsearch.action.search.OpenReaderRequest;
-import org.elasticsearch.action.search.OpenReaderResponse;
+import org.elasticsearch.action.search.CloseSearchContextAction;
+import org.elasticsearch.action.search.CloseSearchContextRequest;
+import org.elasticsearch.action.search.OpenSearchContextRequest;
+import org.elasticsearch.action.search.OpenSearchContextResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.action.search.TransportOpenReaderAction;
+import org.elasticsearch.action.search.TransportOpenSearchContextAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -71,11 +71,11 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     }
 
     String openReaders(TimeValue keepAlive, String... indices) {
-        OpenReaderRequest request = new OpenReaderRequest(indices, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED,
+        OpenSearchContextRequest request = new OpenSearchContextRequest(indices, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED,
             keepAlive, null, null);
-        final OpenReaderResponse response = client()
-            .execute(TransportOpenReaderAction.INSTANCE, request).actionGet();
-        return response.getReaderId();
+        final OpenSearchContextResponse response = client()
+            .execute(TransportOpenSearchContextAction.INSTANCE, request).actionGet();
+        return response.getSearchContextId();
     }
 
     public void testCloseFreezeAndOpen() throws Exception {
@@ -139,7 +139,7 @@ public void testCloseFreezeAndOpen() throws Exception {
                 }
             }
         } finally {
-            client().execute(ClearReaderAction.INSTANCE, new ClearReaderRequest(searchResponse.getReaderId())).get();
+            client().execute(CloseSearchContextAction.INSTANCE, new CloseSearchContextRequest(searchResponse.searchContextId())).get();
         }
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
index 3cfb060d20808..e105b23b401b0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.action.delete.DeleteAction;
 import org.elasticsearch.action.get.MultiGetAction;
 import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.search.ClearReaderAction;
+import org.elasticsearch.action.search.CloseSearchContextAction;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.MultiSearchAction;
 import org.elasticsearch.action.search.SearchScrollAction;
@@ -278,7 +278,7 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth
                     // the same as the user that submitted the original request so we can skip security here.
                     listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
                 }
-            } else if (action.equals(ClearReaderAction.NAME)) {
+            } else if (action.equals(CloseSearchContextAction.NAME)) {
                 // NORELEASE: ensure security works correctly
                 listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
             } else {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
index 87b25c5cf3f50..9389deccb792a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.action.search.ClearReaderAction;
-import org.elasticsearch.action.search.ClearReaderRequest;
+import org.elasticsearch.action.search.CloseSearchContextAction;
+import org.elasticsearch.action.search.CloseSearchContextRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
@@ -65,7 +65,7 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.integration.FieldLevelSecurityTests.openReaders;
+import static org.elasticsearch.integration.FieldLevelSecurityTests.openSearchContext;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -780,7 +780,7 @@ public void testReaderId() throws Exception {
         }
         refresh();
 
-        String readerId = openReaders("user1", TimeValue.timeValueMinutes(1), "test");
+        String readerId = openSearchContext("user1", TimeValue.timeValueMinutes(1), "test");
         SearchResponse response = null;
         try {
             for (int from = 0; from < numVisible; from++) {
@@ -798,7 +798,7 @@ public void testReaderId() throws Exception {
                 assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
             }
         } finally {
-            client().execute(ClearReaderAction.INSTANCE, new ClearReaderRequest(response.getReaderId())).actionGet();
+            client().execute(CloseSearchContextAction.INSTANCE, new CloseSearchContextRequest(response.searchContextId())).actionGet();
         }
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
index f7b1e227cdd3b..17754ae429432 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
@@ -11,13 +11,13 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.action.search.ClearReaderAction;
-import org.elasticsearch.action.search.ClearReaderRequest;
+import org.elasticsearch.action.search.CloseSearchContextAction;
+import org.elasticsearch.action.search.CloseSearchContextRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
-import org.elasticsearch.action.search.OpenReaderRequest;
-import org.elasticsearch.action.search.OpenReaderResponse;
+import org.elasticsearch.action.search.OpenSearchContextRequest;
+import org.elasticsearch.action.search.OpenSearchContextResponse;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.TransportOpenReaderAction;
+import org.elasticsearch.action.search.TransportOpenSearchContextAction;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
@@ -728,12 +728,13 @@ public void testScroll() throws Exception {
         }
     }
 
-    static String openReaders(String userName, TimeValue keepAlive, String... indices) {
-        OpenReaderRequest request = new OpenReaderRequest(indices, OpenReaderRequest.DEFAULT_INDICES_OPTIONS, keepAlive, null, null);
-        final OpenReaderResponse response = client()
+    static String openSearchContext(String userName, TimeValue keepAlive, String... indices) {
+        OpenSearchContextRequest request = new OpenSearchContextRequest(
+            indices, OpenSearchContextRequest.DEFAULT_INDICES_OPTIONS, keepAlive, null, null);
+        final OpenSearchContextResponse response = client()
             .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue(userName, USERS_PASSWD)))
-            .execute(TransportOpenReaderAction.INSTANCE, request).actionGet();
-        return response.getReaderId();
+            .execute(TransportOpenSearchContextAction.INSTANCE, request).actionGet();
+        return response.getSearchContextId();
     }
 
     public void testReaderId() throws Exception {
@@ -750,7 +751,7 @@ public void testReaderId() throws Exception {
         }
         refresh("test");
 
-        String readerId = openReaders("user1", TimeValue.timeValueMinutes(1), "test");
+        String readerId = openSearchContext("user1", TimeValue.timeValueMinutes(1), "test");
         SearchResponse response = null;
         try {
             for (int from = 0; from < numDocs; from++) {
@@ -769,7 +770,7 @@ public void testReaderId() throws Exception {
                 assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), is("value1"));
             }
         } finally {
-            client().execute(ClearReaderAction.INSTANCE, new ClearReaderRequest(readerId)).actionGet();
+            client().execute(CloseSearchContextAction.INSTANCE, new CloseSearchContextRequest(readerId)).actionGet();
         }
     }
 

From de53080d8d6a1730cf89ef812ac13d4e26848c04 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Fri, 8 May 2020 11:25:56 -0400
Subject: [PATCH 17/47] Add total hits info to test assertion

---
 .../elasticsearch/search/basic/SearchWhileRelocatingIT.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
index 320f4f8c780e3..0769eca51fa21 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
@@ -88,8 +88,8 @@ public void run() {
                                 }
 
                                 final SearchHits sh = sr.getHits();
-                                assertThat("Expected hits to be the same size the actual hits array", sh.getTotalHits().value,
-                                        equalTo((long) (sh.getHits().length)));
+                                assertThat("Expected hits to be the same size the actual hits array [" + sh.getTotalHits() + "]",
+                                    sh.getTotalHits().value, equalTo((long) (sh.getHits().length)));
                                 // this is the more critical but that we hit the actual hit array has a different size than the
                                 // actual number of hits.
                             }

From 1c36fefd286c94260b7f85531bc7c307cb3dc656 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Fri, 8 May 2020 14:06:16 -0400
Subject: [PATCH 18/47] more on renaming

---
 docs/reference/search/search_context.asciidoc |  6 +-
 .../SearchWithReaderContextIT.java            | 21 ++---
 .../search/AbstractSearchAsyncAction.java     | 20 ++---
 ...java => ClearSearchContextController.java} | 22 +++---
 .../action/search/ParsedScrollId.java         |  6 +-
 ...rNode.java => SearchContextIdForNode.java} | 16 ++--
 .../action/search/SearchRequest.java          |  6 +-
 .../action/search/SearchRequestBuilder.java   | 10 +--
 .../action/search/SearchResponse.java         | 12 +--
 .../search/SearchScrollAsyncAction.java       | 16 ++--
 .../action/search/SearchShardIterator.java    | 25 +++---
 .../action/search/SearchTransportService.java | 16 ++--
 .../search/TransportClearScrollAction.java    |  2 +-
 .../TransportCloseSearchContextAction.java    |  2 +-
 .../TransportOpenSearchContextAction.java     |  2 +-
 .../action/search/TransportSearchAction.java  | 51 ++++++------
 .../action/search/TransportSearchHelper.java  | 14 ++--
 .../index/shard/SearchOperationListener.java  | 31 ++------
 .../elasticsearch/search/SearchService.java   | 11 +--
 .../AbstractSearchAsyncActionTests.java       |  6 +-
 ...=> ClearSearchContextControllerTests.java} |  8 +-
 .../action/search/SearchAsyncActionTests.java |  2 +-
 .../action/search/SearchRequestTests.java     |  6 +-
 .../search/SearchScrollAsyncActionTests.java  | 78 +++++++++----------
 .../search/TransportSearchHelperTests.java    | 40 +++++-----
 .../shard/SearchOperationListenerTests.java   |  7 +-
 .../index/engine/FrozenIndexTests.java        |  2 +-
 .../SecuritySearchOperationListener.java      |  5 +-
 .../DocumentLevelSecurityTests.java           |  2 +-
 .../integration/FieldLevelSecurityTests.java  |  2 +-
 .../SecuritySearchOperationListenerTests.java | 12 +--
 .../sql/qa/security/with-ssl/build.gradle     |  9 ++-
 32 files changed, 220 insertions(+), 248 deletions(-)
 rename server/src/main/java/org/elasticsearch/action/search/{ClearContextController.java => ClearSearchContextController.java} (85%)
 rename server/src/main/java/org/elasticsearch/action/search/{ReaderIdForNode.java => SearchContextIdForNode.java} (76%)
 rename server/src/test/java/org/elasticsearch/action/search/{ClearContextControllerTests.java => ClearSearchContextControllerTests.java} (97%)

diff --git a/docs/reference/search/search_context.asciidoc b/docs/reference/search/search_context.asciidoc
index 7c3da5cedc865..e9566efe42b21 100644
--- a/docs/reference/search/search_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -2,10 +2,10 @@
 ==== Search Context
 
 By default, a search request executes against the latest point in time readers of the
-target indices. Sometimes it's preferred to execute multiple search requests using
+target indices. Sometimes it's more preferred to execute multiple search requests using
 the same point in time readers. For example, the combined result of the initial and
-subsequent search_after requests is more consistent if they use the same point in time
-readers. This can be done by using a single search context for all search requests.
+subsequent search_after requests would be more consistent if they use the same point
+in time readers. This can be done by using a single search context for all search requests.
 
 A search context must be opened in a separate step before being used in subsequent
 search requests. The keep_alive parameter tells Elasticsearch how long it should keep
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
index e597ea23e5847..5e007eed0fdd8 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchWithReaderContextIT.java
@@ -71,7 +71,7 @@ public void testBasic() {
         }
         refresh("test");
         String readerId = openSearchContext(new String[]{"test"}, TimeValue.timeValueMinutes(2));
-        SearchResponse resp1 = client().prepareSearch().setPreference(null).setReader(readerId, TimeValue.timeValueMinutes(2)).get();
+        SearchResponse resp1 = client().prepareSearch().setPreference(null).setSearchContext(readerId, TimeValue.timeValueMinutes(2)).get();
         assertThat(resp1.searchContextId(), equalTo(readerId));
         assertHitCount(resp1, numDocs);
         int deletedDocs = 0;
@@ -90,7 +90,7 @@ public void testBasic() {
         }
         try {
             SearchResponse resp3 = client().prepareSearch().setPreference(null).setQuery(new MatchAllQueryBuilder())
-                .setReader(resp1.searchContextId(), TimeValue.timeValueMinutes(2))
+                .setSearchContext(resp1.searchContextId(), TimeValue.timeValueMinutes(2))
                 .get();
             assertNoFailures(resp3);
             assertHitCount(resp3, numDocs);
@@ -113,7 +113,7 @@ public void testMultipleIndices() {
         }
         refresh();
         String readerId = openSearchContext(new String[]{"*"}, TimeValue.timeValueMinutes(2));
-        SearchResponse resp1 = client().prepareSearch().setPreference(null).setReader(readerId, TimeValue.timeValueMinutes(2)).get();
+        SearchResponse resp1 = client().prepareSearch().setPreference(null).setSearchContext(readerId, TimeValue.timeValueMinutes(2)).get();
         assertNoFailures(resp1);
         assertHitCount(resp1, numDocs);
         int moreDocs = randomIntBetween(10, 50);
@@ -129,7 +129,7 @@ public void testMultipleIndices() {
             assertHitCount(resp2, numDocs + moreDocs);
 
             SearchResponse resp3 = client().prepareSearch().setPreference(null)
-                .setReader(resp1.searchContextId(), TimeValue.timeValueMinutes(1)).get();
+                .setSearchContext(resp1.searchContextId(), TimeValue.timeValueMinutes(1)).get();
             assertNoFailures(resp3);
             assertHitCount(resp3, numDocs);
         } finally {
@@ -147,7 +147,7 @@ public void testReaderIdNotFound() throws Exception {
         refresh();
         String readerId = openSearchContext(new String[]{"index"}, TimeValue.timeValueSeconds(5));
         SearchResponse resp1 = client().prepareSearch().setPreference(null)
-            .setReader(readerId, TimeValue.timeValueMillis(randomIntBetween(0, 10))).get();
+            .setSearchContext(readerId, TimeValue.timeValueMillis(randomIntBetween(0, 10))).get();
         assertNoFailures(resp1);
         assertHitCount(resp1, index1);
         if (rarely()) {
@@ -158,8 +158,9 @@ public void testReaderIdNotFound() throws Exception {
         } else {
             closeSearchContext(resp1.searchContextId());
         }
-        SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class,
-            () -> client().prepareSearch().setPreference(null).setReader(resp1.searchContextId(), TimeValue.timeValueMinutes(1)).get());
+        SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () ->
+            client().prepareSearch().setPreference(null)
+                .setSearchContext(resp1.searchContextId(), TimeValue.timeValueMinutes(1)).get());
         for (ShardSearchFailure failure : e.shardFailures()) {
             assertThat(ExceptionsHelper.unwrapCause(failure.getCause()), instanceOf(SearchContextMissingException.class));
         }
@@ -182,7 +183,7 @@ public void testIndexNotFound() {
         }
         refresh();
         String readerId = openSearchContext(new String[]{"index-*"}, TimeValue.timeValueMinutes(2));
-        SearchResponse resp1 = client().prepareSearch().setPreference(null).setReader(readerId, TimeValue.timeValueMinutes(2)).get();
+        SearchResponse resp1 = client().prepareSearch().setPreference(null).setSearchContext(readerId, TimeValue.timeValueMinutes(2)).get();
         assertNoFailures(resp1);
         assertHitCount(resp1, index1 + index2);
         client().admin().indices().prepareDelete("index-1").get();
@@ -194,7 +195,7 @@ public void testIndexNotFound() {
         }
         expectThrows(IndexNotFoundException.class, () -> client().prepareSearch()
             .setPreference(null)
-            .setReader(resp1.searchContextId(), TimeValue.timeValueMinutes(1)).get());
+            .setSearchContext(resp1.searchContextId(), TimeValue.timeValueMinutes(1)).get());
         closeSearchContext(resp1.searchContextId());
     }
 
@@ -222,7 +223,7 @@ public void testCanMatch() throws Exception {
                 .setPreference(null)
                 .setPreFilterShardSize(randomIntBetween(2, 3))
                 .setMaxConcurrentShardRequests(randomIntBetween(1, 2))
-                .setReader(readerId, TimeValue.timeValueMinutes(2))
+                .setSearchContext(readerId, TimeValue.timeValueMinutes(2))
                 .get();
             assertThat(resp.getHits().getHits(), arrayWithSize(0));
             for (String node : internalCluster().nodesInclude("test")) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 579ae8d362e23..9911adf079f7e 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -528,14 +528,10 @@ public final SearchRequest getRequest() {
         return request;
     }
 
-    protected final SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse,
-                                                       AtomicArray<SearchPhaseResult> queryResults,
-                                                       ShardSearchFailure[] failures) {
-        final Version minNodeVersion = clusterState.nodes().getMinNodeVersion();
-        String scrollId = request.scroll() != null ? TransportSearchHelper.buildScrollId(queryResults, minNodeVersion) : null;
-        String readerId = includeSearchContextInResponse() ? TransportSearchHelper.encodeReaderIds(queryResults, minNodeVersion) : null;
+    protected final SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse, ShardSearchFailure[] failures,
+                                                       String scrollId, String searchContextId) {
         return new SearchResponse(internalSearchResponse, scrollId, getNumShards(), successfulOps.get(),
-            skippedOps.get(), buildTookInMillis(), failures, clusters, readerId);
+            skippedOps.get(), buildTookInMillis(), failures, clusters, searchContextId);
     }
 
     boolean includeSearchContextInResponse() {
@@ -550,7 +546,11 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At
         if (request.searchContextBuilder() == null && allowPartialResults == false && failures.length > 0) {
             raisePhaseFailure(new SearchPhaseExecutionException("", "Shard failures", null, failures));
         } else {
-            listener.onResponse(buildSearchResponse(internalSearchResponse, queryResults, failures));
+            final Version minNodeVersion = clusterState.nodes().getMinNodeVersion();
+            final String scrollId = request.scroll() != null ? TransportSearchHelper.buildScrollId(queryResults, minNodeVersion) : null;
+            final String searchContextId =
+                includeSearchContextInResponse() ? TransportSearchHelper.encodeSearchContextId(queryResults, minNodeVersion) : null;
+            listener.onResponse(buildSearchResponse(internalSearchResponse, failures, scrollId, searchContextId));
         }
     }
 
@@ -618,11 +618,11 @@ public final ShardSearchRequest buildShardSearchRequest(SearchShardIterator shar
             .toArray(new String[0]);
         ShardSearchRequest shardRequest = new ShardSearchRequest(shardIt.getOriginalIndices(), request, shardIt.shardId(), getNumShards(),
             filter, indexBoost, timeProvider.getAbsoluteStartMillis(), shardIt.getClusterAlias(), routings,
-            shardIt.getReaderId(), shardIt.getReaderKeepAlive());
+            shardIt.getSearchContextId(), shardIt.getSearchContextKeepAlive());
         // if we already received a search result we can inform the shard that it
         // can return a null response if the request rewrites to match none rather
         // than creating an empty response in the search thread pool.
-        // Note that, we have to disable this shortcut for queries that create a context (scroll and reader_id).
+        // Note that, we have to disable this shortcut for queries that create a context (scroll and search context).
         shardRequest.canReturnNullResponseIfMatchNoDocs(hasShardResponse.get() && shardRequest.shouldCreatePersistentReader() == false);
         return shardRequest;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearContextController.java b/server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java
similarity index 85%
rename from server/src/main/java/org/elasticsearch/action/search/ClearContextController.java
rename to server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java
index 59a53eab6e50c..571be1c6f6c98 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearContextController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java
@@ -36,7 +36,7 @@
 
 import static org.elasticsearch.action.search.TransportSearchHelper.parseScrollId;
 
-final class ClearContextController implements Runnable {
+final class ClearSearchContextController implements Runnable {
     private final DiscoveryNodes nodes;
     private final SearchTransportService searchTransportService;
     private final CountDown expectedOps;
@@ -46,8 +46,8 @@ final class ClearContextController implements Runnable {
     private final Logger logger;
     private final Runnable runner;
 
-    ClearContextController(ClearScrollRequest clearScrollRequest, ActionListener<ClearScrollResponse> listener,
-                           DiscoveryNodes nodes, Logger logger, SearchTransportService searchTransportService) {
+    ClearSearchContextController(ClearScrollRequest clearScrollRequest, ActionListener<ClearScrollResponse> listener,
+                                 DiscoveryNodes nodes, Logger logger, SearchTransportService searchTransportService) {
         this.nodes = nodes;
         this.logger = logger;
         this.searchTransportService = searchTransportService;
@@ -58,7 +58,7 @@ final class ClearContextController implements Runnable {
             expectedOps = nodes.getSize();
             runner = this::cleanAllScrolls;
         } else {
-            List<ReaderIdForNode> contexts = new ArrayList<>();
+            List<SearchContextIdForNode> contexts = new ArrayList<>();
             for (String scrollId : scrollIds) {
                 contexts.addAll(Arrays.asList(parseScrollId(scrollId).getContext()));
             }
@@ -68,14 +68,14 @@ final class ClearContextController implements Runnable {
         this.expectedOps = new CountDown(expectedOps);
     }
 
-    ClearContextController(CloseSearchContextRequest closeSearchContextRequest, ActionListener<CloseSearchContextResponse> listener,
-                           DiscoveryNodes nodes, Logger logger, SearchTransportService searchTransportService) {
+    ClearSearchContextController(CloseSearchContextRequest closeSearchContextRequest, ActionListener<CloseSearchContextResponse> listener,
+                                 DiscoveryNodes nodes, Logger logger, SearchTransportService searchTransportService) {
         this.nodes = nodes;
         this.logger = logger;
         this.searchTransportService = searchTransportService;
         this.listener = listener;
-        final Collection<ReaderIdForNode> contexts =
-            TransportSearchHelper.decodeReaderIds(closeSearchContextRequest.getId()).values();
+        final Collection<SearchContextIdForNode> contexts =
+            TransportSearchHelper.decodeSearchContextId(closeSearchContextRequest.getId()).values();
         expectedOps = new CountDown(contexts.size());
         runner = () -> cleanReaderIds(contexts);
     }
@@ -106,21 +106,21 @@ public void onFailure(Exception e) {
         }
     }
 
-    void cleanReaderIds(Collection<ReaderIdForNode> readerIds) {
+    void cleanReaderIds(Collection<SearchContextIdForNode> readerIds) {
         if (readerIds.isEmpty()) {
             listener.onResponse(new CloseSearchContextResponse(true, 0));
             return;
         }
         SearchScrollAsyncAction.collectNodesAndRun(readerIds, nodes, searchTransportService, ActionListener.wrap(
             lookup -> {
-                for (ReaderIdForNode target : readerIds) {
+                for (SearchContextIdForNode target : readerIds) {
                     final DiscoveryNode node = lookup.apply(target.getClusterAlias(), target.getNode());
                     if (node == null) {
                         onFreedContext(false);
                     } else {
                         try {
                             Transport.Connection connection = searchTransportService.getConnection(target.getClusterAlias(), node);
-                            searchTransportService.sendFreeContext(connection, target.getContextId(),
+                            searchTransportService.sendFreeContext(connection, target.getSearchContextId(),
                                 ActionListener.wrap(freed -> onFreedContext(freed.isFreed()), e -> onFailedFreedContext(e, node)));
                         } catch (Exception e) {
                             onFailedFreedContext(e, node);
diff --git a/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java b/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
index a638e6e3cf1e0..43ae39669606b 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
@@ -29,9 +29,9 @@ class ParsedScrollId {
 
     private final String type;
 
-    private final ReaderIdForNode[] context;
+    private final SearchContextIdForNode[] context;
 
-    ParsedScrollId(String source, String type, ReaderIdForNode[] context) {
+    ParsedScrollId(String source, String type, SearchContextIdForNode[] context) {
         this.source = source;
         this.type = type;
         this.context = context;
@@ -45,7 +45,7 @@ public String getType() {
         return type;
     }
 
-    public ReaderIdForNode[] getContext() {
+    public SearchContextIdForNode[] getContext() {
         return context;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ReaderIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
similarity index 76%
rename from server/src/main/java/org/elasticsearch/action/search/ReaderIdForNode.java
rename to server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
index 74cec46416fe8..798087481a39a 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ReaderIdForNode.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
@@ -22,15 +22,15 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.search.internal.SearchContextId;
 
-final class ReaderIdForNode {
+final class SearchContextIdForNode {
     private final String node;
-    private final SearchContextId contextId;
+    private final SearchContextId searchContextId;
     private final String clusterAlias;
 
-    ReaderIdForNode(@Nullable String clusterAlias, String node, SearchContextId contextId) {
+    SearchContextIdForNode(@Nullable String clusterAlias, String node, SearchContextId searchContextId) {
         this.node = node;
         this.clusterAlias = clusterAlias;
-        this.contextId = contextId;
+        this.searchContextId = searchContextId;
     }
 
     public String getNode() {
@@ -42,15 +42,15 @@ public String getClusterAlias() {
         return clusterAlias;
     }
 
-    public SearchContextId getContextId() {
-        return contextId;
+    public SearchContextId getSearchContextId() {
+        return searchContextId;
     }
 
     @Override
     public String toString() {
-        return "ReaderIdForNode{" +
+        return "SearchContextIdForNode{" +
             "node='" + node + '\'' +
-            ", scrollId=" + contextId +
+            ", seachContextId=" + searchContextId +
             ", clusterAlias='" + clusterAlias + '\'' +
             '}';
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 816d9efbf8a79..f9d3b0a0d7ab4 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -288,13 +288,13 @@ public ActionRequestValidationException validate() {
         }
         if (searchContextBuilder() != null) {
             if (indices.length > 0) {
-                validationException = addValidationError("[index] cannot be used with reader contexts", validationException);
+                validationException = addValidationError("[index] cannot be used with search context", validationException);
             }
             if (routing() != null) {
-                validationException = addValidationError("[routing] cannot be used with reader contexts", validationException);
+                validationException = addValidationError("[routing] cannot be used with search context", validationException);
             }
             if (preference() != null) {
-                validationException = addValidationError("[preference] cannot be used with reader contexts", validationException);
+                validationException = addValidationError("[preference] cannot be used with search context", validationException);
             }
         }
         return validationException;
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 05f1cab42f902..7007592139319 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -522,13 +522,13 @@ public SearchRequestBuilder setCollapse(CollapseBuilder collapse) {
     }
 
     /**
-     * Specifies the point in time reader contexts that this request should execute against
+     * Specifies the search context that Elasticsearch should use to perform the query
      *
-     * @param readerId  the base64 string of the set of reader ids.
-     * @param keepAlive the extended keep alive for the readers used in this request
+     * @param searchContextId the base64 encoded string of the search context id
+     * @param keepAlive       the extended time to live for the search context
      */
-    public SearchRequestBuilder setReader(@Nullable String readerId, @Nullable TimeValue keepAlive) {
-        sourceBuilder().searchContextBuilder(new SearchSourceBuilder.SearchContextBuilder(readerId, keepAlive));
+    public SearchRequestBuilder setSearchContext(String searchContextId, TimeValue keepAlive) {
+        sourceBuilder().searchContextBuilder(new SearchSourceBuilder.SearchContextBuilder(searchContextId, keepAlive));
         return this;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 482c846dbed55..2d3b4d5d34d63 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -62,7 +62,7 @@
 public class SearchResponse extends ActionResponse implements StatusToXContentObject {
 
     private static final ParseField SCROLL_ID = new ParseField("_scroll_id");
-    private static final ParseField READER_ID = new ParseField("reader_id");
+    private static final ParseField SEARCH_CONTEXT_ID = new ParseField("search_context_id");
     private static final ParseField TOOK = new ParseField("took");
     private static final ParseField TIMED_OUT = new ParseField("timed_out");
     private static final ParseField TERMINATED_EARLY = new ParseField("terminated_early");
@@ -259,7 +259,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
             builder.field(SCROLL_ID.getPreferredName(), scrollId);
         }
         if (searchContextId != null) {
-            builder.field(READER_ID.getPreferredName(), searchContextId);
+            builder.field(SEARCH_CONTEXT_ID.getPreferredName(), searchContextId);
         }
         builder.field(TOOK.getPreferredName(), tookInMillis);
         builder.field(TIMED_OUT.getPreferredName(), isTimedOut());
@@ -297,7 +297,7 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE
         int totalShards = -1;
         int skippedShards = 0; // 0 for BWC
         String scrollId = null;
-        String readerContextId = null;
+        String searchContextId = null;
         List<ShardSearchFailure> failures = new ArrayList<>();
         Clusters clusters = Clusters.EMPTY;
         for (Token token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) {
@@ -306,8 +306,8 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE
             } else if (token.isValue()) {
                 if (SCROLL_ID.match(currentFieldName, parser.getDeprecationHandler())) {
                     scrollId = parser.text();
-                } else if (READER_ID.match(currentFieldName, parser.getDeprecationHandler())) {
-                    readerContextId = parser.text();
+                } else if (SEARCH_CONTEXT_ID.match(currentFieldName, parser.getDeprecationHandler())) {
+                    searchContextId = parser.text();
                 } else if (TOOK.match(currentFieldName, parser.getDeprecationHandler())) {
                     tookInMillis = parser.longValue();
                 } else if (TIMED_OUT.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -386,7 +386,7 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE
         SearchResponseSections searchResponseSections = new SearchResponseSections(hits, aggs, suggest, timedOut, terminatedEarly,
                 profile, numReducePhases);
         return new SearchResponse(searchResponseSections, scrollId, totalShards, successfulShards, skippedShards, tookInMillis,
-                failures.toArray(ShardSearchFailure.EMPTY_ARRAY), clusters, readerContextId);
+                failures.toArray(ShardSearchFailure.EMPTY_ARRAY), clusters, searchContextId);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
index 6e8bf47cbbc61..e8d3387b9ca18 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
@@ -104,7 +104,7 @@ private long buildTookInMillis() {
     }
 
     public final void run() {
-        final ReaderIdForNode[] context = scrollId.getContext();
+        final SearchContextIdForNode[] context = scrollId.getContext();
         if (context.length == 0) {
             listener.onFailure(new SearchPhaseExecutionException("query", "no nodes to search on", ShardSearchFailure.EMPTY_ARRAY));
         } else {
@@ -117,11 +117,11 @@ public final void run() {
      * This method collects nodes from the remote clusters asynchronously if any of the scroll IDs references a remote cluster.
      * Otherwise the action listener will be invoked immediately with a function based on the given discovery nodes.
      */
-    static void collectNodesAndRun(final Iterable<ReaderIdForNode> scrollIds, DiscoveryNodes nodes,
+    static void collectNodesAndRun(final Iterable<SearchContextIdForNode> scrollIds, DiscoveryNodes nodes,
                                    SearchTransportService searchTransportService,
                                    ActionListener<BiFunction<String, String, DiscoveryNode>> listener) {
         Set<String> clusters = new HashSet<>();
-        for (ReaderIdForNode target : scrollIds) {
+        for (SearchContextIdForNode target : scrollIds) {
             if (target.getClusterAlias() != null) {
                 clusters.add(target.getClusterAlias());
             }
@@ -135,10 +135,10 @@ static void collectNodesAndRun(final Iterable<ReaderIdForNode> scrollIds, Discov
         }
     }
 
-    private void run(BiFunction<String, String, DiscoveryNode> clusterNodeLookup, final ReaderIdForNode[] context) {
+    private void run(BiFunction<String, String, DiscoveryNode> clusterNodeLookup, final SearchContextIdForNode[] context) {
         final CountDown counter = new CountDown(scrollId.getContext().length);
         for (int i = 0; i < context.length; i++) {
-            ReaderIdForNode target = context[i];
+            SearchContextIdForNode target = context[i];
             final int shardIndex = i;
             final Transport.Connection connection;
             try {
@@ -148,11 +148,11 @@ private void run(BiFunction<String, String, DiscoveryNode> clusterNodeLookup, fi
                 }
                 connection = getConnection(target.getClusterAlias(), node);
             } catch (Exception ex) {
-                onShardFailure("query", counter, target.getContextId(),
+                onShardFailure("query", counter, target.getSearchContextId(),
                     ex, null, () -> SearchScrollAsyncAction.this.moveToNextPhase(clusterNodeLookup));
                 continue;
             }
-            final InternalScrollSearchRequest internalRequest = internalScrollSearchRequest(target.getContextId(), request);
+            final InternalScrollSearchRequest internalRequest = internalScrollSearchRequest(target.getSearchContextId(), request);
             // we can't create a SearchShardTarget here since we don't know the index and shard ID we are talking to
             // we only know the node and the search context ID. Yet, the response will contain the SearchShardTarget
             // from the target node instead...that's why we pass null here
@@ -192,7 +192,7 @@ protected void innerOnResponse(T result) {
 
                 @Override
                 public void onFailure(Exception t) {
-                    onShardFailure("query", counter, target.getContextId(), t, null,
+                    onShardFailure("query", counter, target.getSearchContextId(), t, null,
                         () -> SearchScrollAsyncAction.this.moveToNextPhase(clusterNodeLookup));
                 }
             };
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
index 974b062e0cc1d..d46bfb449e5de 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
@@ -44,8 +44,8 @@ public final class SearchShardIterator extends PlainShardIterator {
     private final String clusterAlias;
     private boolean skip = false;
 
-    private final SearchContextId readerId;
-    private final TimeValue readerKeepAlive;
+    private final SearchContextId searchContextId;
+    private final TimeValue searchContextKeepAlive;
 
     /**
      * Creates a {@link PlainShardIterator} instance that iterates over a subset of the given shards
@@ -62,12 +62,13 @@ public SearchShardIterator(@Nullable String clusterAlias, ShardId shardId, List<
 
     public SearchShardIterator(@Nullable String clusterAlias, ShardId shardId,
                                List<ShardRouting> shards, OriginalIndices originalIndices,
-                               SearchContextId readerId, TimeValue readerKeepAlive) {
+                               SearchContextId searchContextId, TimeValue searchContextKeepAlive) {
         super(shardId, shards);
         this.originalIndices = originalIndices;
         this.clusterAlias = clusterAlias;
-        this.readerId = readerId;
-        this.readerKeepAlive = readerKeepAlive;
+        this.searchContextId = searchContextId;
+        this.searchContextKeepAlive = searchContextKeepAlive;
+        assert (searchContextId == null) == (searchContextKeepAlive == null);
     }
 
     /**
@@ -94,18 +95,14 @@ SearchShardTarget newSearchShardTarget(String nodeId) {
     }
 
     /**
-     * Returns a non-null value if this request should execute using a specific point-in-time reader;
-     * otherwise, using the most up to date point-in-time reader.
+     * Returns a non-null value if this request should use a specific search context instead of the latest one.
      */
-    SearchContextId getReaderId() {
-        return readerId;
+    SearchContextId getSearchContextId() {
+        return searchContextId;
     }
 
-    /**
-     * Returns a non-null to specify the time to live of the point-in-time reader that is used to execute this request.
-     */
-    TimeValue getReaderKeepAlive() {
-        return readerKeepAlive;
+    TimeValue getSearchContextKeepAlive() {
+        return searchContextKeepAlive;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 7156e6ee840c3..dbb02ec3ad4ef 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -78,7 +78,7 @@ public class SearchTransportService {
     public static final String FETCH_ID_SCROLL_ACTION_NAME = "indices:data/read/search[phase/fetch/id/scroll]";
     public static final String FETCH_ID_ACTION_NAME = "indices:data/read/search[phase/fetch/id]";
     public static final String QUERY_CAN_MATCH_NAME = "indices:data/read/search[can_match]";
-    public static final String SHARD_OPEN_READER_NAME = "indices:data/read/shard_open_reader";
+    public static final String OPEN_SHARD_READER_CONTEXT_NAME = "indices:data/read/open_reader_context";
 
     private final TransportService transportService;
     private final BiFunction<Transport.Connection, SearchActionListener, ActionListener> responseWrapper;
@@ -169,10 +169,10 @@ public void sendExecuteFetchScroll(Transport.Connection connection, final ShardF
         sendExecuteFetch(connection, FETCH_ID_SCROLL_ACTION_NAME, request, task, listener);
     }
 
-    void sendShardOpenReader(Transport.Connection connection, SearchTask task,
-                             TransportOpenSearchContextAction.ShardOpenReaderRequest request,
-                             ActionListener<TransportOpenSearchContextAction.ShardOpenReaderResponse> listener) {
-        transportService.sendChildRequest(connection, SHARD_OPEN_READER_NAME, request, task,
+    void sendOpenShardReaderContext(Transport.Connection connection, SearchTask task,
+                                    TransportOpenSearchContextAction.ShardOpenReaderRequest request,
+                                    ActionListener<TransportOpenSearchContextAction.ShardOpenReaderResponse> listener) {
+        transportService.sendChildRequest(connection, OPEN_SHARD_READER_CONTEXT_NAME, request, task,
             new ActionListenerResponseHandler<>(listener, TransportOpenSearchContextAction.ShardOpenReaderResponse::new));
     }
 
@@ -371,15 +371,15 @@ public static void registerRequestHandler(TransportService transportService, Sea
             });
         TransportActionProxy.registerProxyAction(transportService, QUERY_CAN_MATCH_NAME, SearchService.CanMatchResponse::new);
 
-        transportService.registerRequestHandler(SHARD_OPEN_READER_NAME, ThreadPool.Names.SAME,
+        transportService.registerRequestHandler(OPEN_SHARD_READER_CONTEXT_NAME, ThreadPool.Names.SAME,
             TransportOpenSearchContextAction.ShardOpenReaderRequest::new,
             (request, channel, task) -> {
                 searchService.openReaderContext(request.getShardId(), request.keepAlive,
-                    ActionListener.map(new ChannelActionListener<>(channel, SHARD_OPEN_READER_NAME, request),
+                    ActionListener.map(new ChannelActionListener<>(channel, OPEN_SHARD_READER_CONTEXT_NAME, request),
                         contextId -> new TransportOpenSearchContextAction.ShardOpenReaderResponse(contextId)));
             });
         TransportActionProxy.registerProxyAction(
-            transportService, SHARD_OPEN_READER_NAME, TransportOpenSearchContextAction.ShardOpenReaderResponse::new);
+            transportService, OPEN_SHARD_READER_CONTEXT_NAME, TransportOpenSearchContextAction.ShardOpenReaderResponse::new);
     }
 
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
index 1a5580977e488..016f6bca0a828 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
@@ -42,7 +42,7 @@ public TransportClearScrollAction(TransportService transportService, ClusterServ
 
     @Override
     protected void doExecute(Task task, ClearScrollRequest request, final ActionListener<ClearScrollResponse> listener) {
-        Runnable runnable = new ClearContextController(
+        Runnable runnable = new ClearSearchContextController(
             request, listener, clusterService.state().nodes(), logger, searchTransportService);
         runnable.run();
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java
index 23311624961bf..a668cf61bb0ea 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java
@@ -42,7 +42,7 @@ public TransportCloseSearchContextAction(TransportService transportService, Clus
 
     @Override
     protected void doExecute(Task task, CloseSearchContextRequest request, ActionListener<CloseSearchContextResponse> listener) {
-        Runnable runnable = new ClearContextController(
+        Runnable runnable = new ClearSearchContextController(
             request, listener, clusterService.state().nodes(), logger, searchTransportService);
         runnable.run();
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java
index bc004968e0213..65c810b803aef 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java
@@ -174,7 +174,7 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
             final SearchShardTarget searchShardTarget = shardIt.newSearchShardTarget(shard.currentNodeId());
             final ShardOpenReaderRequest shardRequest = new ShardOpenReaderRequest(searchShardTarget.getShardId(),
                 searchShardTarget.getOriginalIndices(), request.keepAlive());
-            getSearchTransport().sendShardOpenReader(connection, getTask(), shardRequest, ActionListener.map(listener, r -> r));
+            getSearchTransport().sendOpenShardReaderContext(connection, getTask(), shardRequest, ActionListener.map(listener, r -> r));
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 9d683131d311f..c5082d107f6e5 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -214,27 +214,27 @@ void executeRequest(Task task, SearchRequest searchRequest,
                 // situations when source is rewritten to null due to a bug
                 searchRequest.source(source);
             }
-            final Map<ShardId, ReaderIdForNode> readerContexts;
+            final Map<ShardId, SearchContextIdForNode> searchContexts;
             final Map<String, OriginalIndices> remoteClusterIndices;
             if (searchRequest.searchContextBuilder() != null) {
-                readerContexts = TransportSearchHelper.decodeReaderIds(searchRequest.searchContextBuilder().getId());
-                remoteClusterIndices = indicesFromReaderContexts(readerContexts, searchRequest.indicesOptions());
+                searchContexts = TransportSearchHelper.decodeSearchContextId(searchRequest.searchContextBuilder().getId());
+                remoteClusterIndices = getIndicesFromSearchContexts(searchContexts, searchRequest.indicesOptions());
             } else {
-                readerContexts = null;
+                searchContexts = null;
                 remoteClusterIndices = remoteClusterService.groupIndices(searchRequest.indicesOptions(), searchRequest.indices());
             }
             OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
             final ClusterState clusterState = clusterService.state();
             if (remoteClusterIndices.isEmpty()) {
                 executeLocalSearch(
-                    task, timeProvider, searchRequest, localIndices, clusterState, listener, readerContexts, searchAsyncActionProvider);
+                    task, timeProvider, searchRequest, localIndices, clusterState, listener, searchContexts, searchAsyncActionProvider);
             } else {
                 if (shouldMinimizeRoundtrips(searchRequest)) {
                     ccsRemoteReduce(searchRequest, localIndices, remoteClusterIndices, timeProvider,
                         searchService.aggReduceContextBuilder(searchRequest),
                         remoteClusterService, threadPool, listener,
                         (r, l) -> executeLocalSearch(
-                            task, timeProvider, r, localIndices, clusterState, l, readerContexts, searchAsyncActionProvider));
+                            task, timeProvider, r, localIndices, clusterState, l, searchContexts, searchAsyncActionProvider));
                 } else {
                     AtomicInteger skippedClusters = new AtomicInteger(0);
                     collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(),
@@ -251,7 +251,7 @@ void executeRequest(Task task, SearchRequest searchRequest,
                                 executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices, remoteShardIterators,
                                     clusterNodeLookup, clusterState, remoteAliasFilters, listener,
                                     new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters.get()),
-                                    readerContexts, searchAsyncActionProvider);
+                                    searchContexts, searchAsyncActionProvider);
                             },
                             listener::onFailure));
                 }
@@ -273,7 +273,6 @@ static boolean shouldMinimizeRoundtrips(SearchRequest searchRequest) {
             return false;
         }
         if (searchRequest.searchContextBuilder() != null) {
-            // TODO: support reader contexts
             return false;
         }
         if (searchRequest.searchType() == DFS_QUERY_THEN_FETCH) {
@@ -424,10 +423,11 @@ SearchResponse createFinalResponse() {
 
     private void executeLocalSearch(Task task, SearchTimeProvider timeProvider, SearchRequest searchRequest, OriginalIndices localIndices,
                                     ClusterState clusterState, ActionListener<SearchResponse> listener,
-                                    Map<ShardId, ReaderIdForNode> readerContexts, SearchAsyncActionProvider searchAsyncActionProvider) {
+                                    Map<ShardId, SearchContextIdForNode> searchContexts,
+                                    SearchAsyncActionProvider searchAsyncActionProvider) {
         executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, Collections.emptyList(),
             (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener, SearchResponse.Clusters.EMPTY,
-            readerContexts, searchAsyncActionProvider);
+            searchContexts, searchAsyncActionProvider);
     }
 
     static BiFunction<String, String, DiscoveryNode> processRemoteShards(Map<String, ClusterSearchShardsResponse> searchShardsResponses,
@@ -491,7 +491,7 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
                                OriginalIndices localIndices, List<SearchShardIterator> remoteShardIterators,
                                BiFunction<String, String, DiscoveryNode> remoteConnections, ClusterState clusterState,
                                Map<String, AliasFilter> remoteAliasMap, ActionListener<SearchResponse> listener,
-                               SearchResponse.Clusters clusters, Map<ShardId, ReaderIdForNode> readerContexts,
+                               SearchResponse.Clusters clusters, Map<ShardId, SearchContextIdForNode> searchContexts,
                                SearchAsyncActionProvider searchAsyncActionProvider) {
 
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
@@ -508,15 +508,15 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
         final Map<String, Set<String>> indexRoutings;
 
         boolean preFilterSearchShards;
-        if (readerContexts != null) {
+        if (searchContexts != null) {
             assert searchRequest.searchContextBuilder() != null;
             aliasFilter = new HashMap<>();
-            for (ShardId shardId : readerContexts.keySet()) {
+            for (ShardId shardId : searchContexts.keySet()) {
                 aliasFilter.put(shardId.getIndex().getUUID(), AliasFilter.EMPTY);
             }
             indexRoutings = Map.of();
-            localShardIterators = searchShardsFromReaderContexts(clusterState, localIndices, searchRequest.getLocalClusterAlias(),
-                readerContexts, searchRequest.searchContextBuilder().getKeepAlive());
+            localShardIterators = getSearchShardsFromSearchContexts(clusterState, localIndices, searchRequest.getLocalClusterAlias(),
+                searchContexts, searchRequest.searchContextBuilder().getKeepAlive());
             preFilterSearchShards = shouldPreFilterSearchShards(clusterState, searchRequest, localIndices.indices(),
                 localShardIterators.size() + remoteShardIterators.size());
         } else {
@@ -778,10 +778,10 @@ private static RemoteTransportException wrapRemoteClusterFailure(String clusterA
         return new RemoteTransportException("error while communicating with remote cluster [" + clusterAlias + "]", e);
     }
 
-    static Map<String, OriginalIndices> indicesFromReaderContexts(Map<ShardId, ReaderIdForNode> readerContexts,
-                                                                  IndicesOptions indicesOptions) {
+    static Map<String, OriginalIndices> getIndicesFromSearchContexts(Map<ShardId, SearchContextIdForNode> searchContexts,
+                                                                     IndicesOptions indicesOptions) {
         final Map<String, List<String>> indices = new HashMap<>();
-        for (Map.Entry<ShardId, ReaderIdForNode> entry : readerContexts.entrySet()) {
+        for (Map.Entry<ShardId, SearchContextIdForNode> entry : searchContexts.entrySet()) {
             String clusterAlias = entry.getValue().getClusterAlias() == null ?
                 RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY : entry.getValue().getClusterAlias();
             indices.computeIfAbsent(clusterAlias, k -> new ArrayList<>()).add(entry.getKey().getIndexName());
@@ -790,13 +790,12 @@ static Map<String, OriginalIndices> indicesFromReaderContexts(Map<ShardId, Reade
             .collect(Collectors.toMap(Map.Entry::getKey, e -> new OriginalIndices(e.getValue().toArray(String[]::new), indicesOptions)));
     }
 
-    static List<SearchShardIterator> searchShardsFromReaderContexts(ClusterState clusterState,
-                                                                    OriginalIndices originalIndices,
-                                                                    String localClusterAlias,
-                                                                    Map<ShardId, ReaderIdForNode> readerContexts,
-                                                                    TimeValue keepAlive) {
-        final List<SearchShardIterator> iterators = new ArrayList<>(readerContexts.size());
-        for (Map.Entry<ShardId, ReaderIdForNode> entry : readerContexts.entrySet()) {
+    static List<SearchShardIterator> getSearchShardsFromSearchContexts(ClusterState clusterState, OriginalIndices originalIndices,
+                                                                       String localClusterAlias,
+                                                                       Map<ShardId, SearchContextIdForNode> searchContexts,
+                                                                       TimeValue keepAlive) {
+        final List<SearchShardIterator> iterators = new ArrayList<>(searchContexts.size());
+        for (Map.Entry<ShardId, SearchContextIdForNode> entry : searchContexts.entrySet()) {
             final ShardId shardId = entry.getKey();
             final ShardIterator shards = OperationRouting.getShards(clusterState, shardId);
             final List<ShardRouting> matchingNodeFirstRoutings = new ArrayList<>();
@@ -808,7 +807,7 @@ static List<SearchShardIterator> searchShardsFromReaderContexts(ClusterState clu
                 }
             }
             iterators.add(new SearchShardIterator(localClusterAlias, shardId, matchingNodeFirstRoutings, originalIndices,
-                entry.getValue().getContextId(), keepAlive));
+                entry.getValue().getSearchContextId(), keepAlive));
         }
         return iterators;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
index ab1051873a4f8..f545f1ca3c641 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
@@ -88,7 +88,7 @@ static ParsedScrollId parseScrollId(String scrollId) {
                 includeContextUUID = false;
                 type = firstChunk;
             }
-            ReaderIdForNode[] context = new ReaderIdForNode[in.readVInt()];
+            SearchContextIdForNode[] context = new SearchContextIdForNode[in.readVInt()];
             for (int i = 0; i < context.length; ++i) {
                 final String contextUUID = includeContextUUID ? in.readString() : "";
                 long id = in.readLong();
@@ -101,7 +101,7 @@ static ParsedScrollId parseScrollId(String scrollId) {
                     clusterAlias = target.substring(0, index);
                     target = target.substring(index+1);
                 }
-                context[i] = new ReaderIdForNode(clusterAlias, target, new SearchContextId(contextUUID, id));
+                context[i] = new SearchContextIdForNode(clusterAlias, target, new SearchContextId(contextUUID, id));
             }
             if (in.getPosition() != bytes.length) {
                 throw new IllegalArgumentException("Not all bytes were read");
@@ -112,7 +112,7 @@ static ParsedScrollId parseScrollId(String scrollId) {
         }
     }
 
-    static String encodeReaderIds(AtomicArray<? extends SearchPhaseResult> searchPhaseResults, Version version) {
+    static String encodeSearchContextId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults, Version version) {
         try (RAMOutputStream out = new RAMOutputStream()) {
             out.writeVInt(version.id);
             out.writeVInt(searchPhaseResults.asList().size());
@@ -143,13 +143,13 @@ static String encodeReaderIds(AtomicArray<? extends SearchPhaseResult> searchPha
         }
     }
 
-    static Map<ShardId, ReaderIdForNode> decodeReaderIds(String readerId) {
+    static Map<ShardId, SearchContextIdForNode> decodeSearchContextId(String searchContextId) {
         try {
-            final byte[] bytes = Base64.getUrlDecoder().decode(readerId);
+            final byte[] bytes = Base64.getUrlDecoder().decode(searchContextId);
             final ByteArrayDataInput in = new ByteArrayDataInput(bytes);
             Version.fromId(in.readVInt());
             final int numOfContexts = in.readVInt();
-            final Map<ShardId, ReaderIdForNode> contexts = new HashMap<>();
+            final Map<ShardId, SearchContextIdForNode> contexts = new HashMap<>();
             for (int i = 0; i < numOfContexts; i++) {
                 final ShardId shardId = new ShardId(new Index(in.readString(), in.readString()), in.readVInt());
                 final String clusterAlias;
@@ -160,7 +160,7 @@ static Map<ShardId, ReaderIdForNode> decodeReaderIds(String readerId) {
                 }
                 final String nodeId = in.readString();
                 final SearchContextId contextId = new SearchContextId(in.readString(), in.readLong());
-                contexts.put(shardId, new ReaderIdForNode(clusterAlias, nodeId, contextId));
+                contexts.put(shardId, new SearchContextIdForNode(clusterAlias, nodeId, contextId));
             }
             if (in.getPosition() != bytes.length) {
                 throw new IllegalArgumentException("Not all bytes were read");
diff --git a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
index fff7e88e505cb..777f35b569b7b 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
@@ -107,22 +107,14 @@ default void onNewScrollContext(ReaderContext readerContext) {}
     default void onFreeScrollContext(ReaderContext readerContext) {}
 
     /**
-     * Executed prior to using a {@link SearchContext} that has been retrieved
+     * Executed prior to using a {@link ReaderContext} that has been retrieved
      * from the active contexts. If the context is deemed invalid a runtime
      * exception can be thrown, which will prevent the context from being used.
-     * @param readerContext The reader context used by this request.
-     * @param searchContext The newly created {@link SearchContext}.
+     *
+     * @param readerContext    The reader context used by this request.
      * @param transportRequest the request that is going to use the search context
      */
-    default void validateSearchContext(ReaderContext readerContext, SearchContext searchContext, TransportRequest transportRequest) {}
-
-    /**
-     * Executed when a search context was freed. The implementor can implement
-     * this method to release resources used by the search context.
-     */
-    default void onFreeSearchContext(SearchContext context) {
-
-    }
+    default void validateSearchContext(ReaderContext readerContext, TransportRequest transportRequest) {}
 
     /**
      * A Composite listener that multiplexes calls to each of the listeners methods.
@@ -247,27 +239,16 @@ public void onFreeScrollContext(ReaderContext readerContext) {
         }
 
         @Override
-        public void validateSearchContext(ReaderContext readerContext, SearchContext searchContext, TransportRequest request) {
+        public void validateSearchContext(ReaderContext readerContext, TransportRequest request) {
             Exception exception = null;
             for (SearchOperationListener listener : listeners) {
                 try {
-                    listener.validateSearchContext(readerContext, searchContext, request);
+                    listener.validateSearchContext(readerContext, request);
                 } catch (Exception e) {
                     exception = ExceptionsHelper.useOrSuppress(exception, e);
                 }
             }
             ExceptionsHelper.reThrowIfNotNull(exception);
         }
-
-        @Override
-        public void onFreeSearchContext(SearchContext context) {
-            for (SearchOperationListener listener : listeners) {
-                try {
-                    listener.onFreeSearchContext(context);
-                } catch (Exception e) {
-                    logger.warn(() -> new ParameterizedMessage("onFreeSearchContext listener [{}] failed", listener), e);
-                }
-            }
-        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index e281b766fd2e9..99d31c1e5c8b5 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -501,7 +501,6 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
                     checkKeepAliveLimit(keepAlive);
                     readerContext.keepAlive(keepAlive);
                 }
-                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
                 searchContext.searcher().setAggregatedDfs(readerContext.getAggregatedDfs(null));
                 processScroll(request, readerContext, searchContext);
                 queryPhase.execute(searchContext);
@@ -526,7 +525,6 @@ public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task,
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, true);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
-                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
                 searchContext.searcher().setAggregatedDfs(request.dfs());
                 queryPhase.execute(searchContext);
                 if (searchContext.queryResult().hasSearchContext() == false && readerContext.singleSession()) {
@@ -569,7 +567,6 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
                 }
                 searchContext.assignRescoreDocIds(readerContext.getRescoreDocIds(null));
                 searchContext.searcher().setAggregatedDfs(readerContext.getAggregatedDfs(null));
-                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
                 processScroll(request, readerContext, searchContext);
                 queryPhase.execute(searchContext);
                 final long afterQueryTime = executor.success();
@@ -589,7 +586,6 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest());
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false)) {
-                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, searchContext, request);
                 if (request.lastEmittedDoc() != null) {
                     searchContext.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
                 }
@@ -635,7 +631,6 @@ private ReaderContext findReaderContext(SearchContextId id) throws SearchContext
     final ReaderContext createOrGetReaderContext(ShardSearchRequest request, boolean keepStatesInContext) {
         if (request.readerId() != null) {
             assert keepStatesInContext == false;
-            // NORELEASE: either wrap the searcher with the right security context or make sure the user owns it.
             final ReaderContext readerContext = findReaderContext(request.readerId());
             final long keepAlive = request.keepAlive().millis();
             checkKeepAliveLimit(keepAlive);
@@ -730,11 +725,12 @@ public void openReaderContext(ShardId shardId, TimeValue keepAlive, ActionListen
         });
     }
 
-    final SearchContext createContext(ReaderContext reader,
+    final SearchContext createContext(ReaderContext readerContext,
                                       ShardSearchRequest request,
                                       SearchShardTask task,
                                       boolean includeAggregations) throws IOException {
-        final DefaultSearchContext context = createSearchContext(reader, request, defaultSearchTimeout);
+        readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
+        final DefaultSearchContext context = createSearchContext(readerContext, request, defaultSearchTimeout);
         try {
             if (request.scroll() != null) {
                 context.scrollContext().scroll = request.scroll();
@@ -754,7 +750,6 @@ final SearchContext createContext(ReaderContext reader,
             dfsPhase.preProcess(context);
             queryPhase.preProcess(context);
             fetchPhase.preProcess(context);
-            context.addReleasable(() -> reader.indexShard().getSearchOperationListener().onFreeSearchContext(context));
         } catch (Exception e) {
             context.close();
             throw e;
diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
index c2a58d0688ff8..1682f4a3eb986 100644
--- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
@@ -167,8 +167,7 @@ public void testBuildSearchResponse() {
         AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(searchRequest,
             phaseResults, null, false, new AtomicLong());
         InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
-        SearchResponse searchResponse = action.buildSearchResponse(
-            internalSearchResponse, phaseResults.results, action.buildShardFailures());
+        SearchResponse searchResponse = action.buildSearchResponse(internalSearchResponse, action.buildShardFailures(), null, null);
         assertSame(searchResponse.getAggregations(), internalSearchResponse.aggregations());
         assertSame(searchResponse.getSuggest(), internalSearchResponse.suggest());
         assertSame(searchResponse.getProfileResults(), internalSearchResponse.profile());
@@ -182,8 +181,7 @@ public void testBuildSearchResponseAllowPartialFailures() {
         action.onShardFailure(0, new SearchShardTarget("node", new ShardId("index", "index-uuid", 0), null, OriginalIndices.NONE),
             new IllegalArgumentException());
         InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
-        SearchResponse searchResponse = action.buildSearchResponse(
-            internalSearchResponse, queryResult.results, action.buildShardFailures());
+        SearchResponse searchResponse = action.buildSearchResponse(internalSearchResponse, action.buildShardFailures(), null, null);
         assertSame(searchResponse.getAggregations(), internalSearchResponse.aggregations());
         assertSame(searchResponse.getSuggest(), internalSearchResponse.suggest());
         assertSame(searchResponse.getProfileResults(), internalSearchResponse.profile());
diff --git a/server/src/test/java/org/elasticsearch/action/search/ClearContextControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/ClearSearchContextControllerTests.java
similarity index 97%
rename from server/src/test/java/org/elasticsearch/action/search/ClearContextControllerTests.java
rename to server/src/test/java/org/elasticsearch/action/search/ClearSearchContextControllerTests.java
index e81164ca2a2f4..3238e0e654dbc 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ClearContextControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ClearSearchContextControllerTests.java
@@ -44,7 +44,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 
-public class ClearContextControllerTests extends ESTestCase {
+public class ClearSearchContextControllerTests extends ESTestCase {
 
     public void testClearAll() throws InterruptedException {
         DiscoveryNode node1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
@@ -80,7 +80,7 @@ Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
         };
         ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
         clearScrollRequest.scrollIds(Arrays.asList("_all"));
-        ClearContextController controller = new ClearContextController(
+        ClearSearchContextController controller = new ClearSearchContextController(
             clearScrollRequest, listener, nodes, logger, searchTransportService);
         controller.run();
         latch.await();
@@ -145,7 +145,7 @@ Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
         };
         ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
         clearScrollRequest.scrollIds(Arrays.asList(scrollId));
-        ClearContextController controller = new ClearContextController(clearScrollRequest, listener,
+        ClearSearchContextController controller = new ClearSearchContextController(clearScrollRequest, listener,
             nodes, logger, searchTransportService);
         controller.run();
         latch.await();
@@ -229,7 +229,7 @@ Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
         };
         ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
         clearScrollRequest.scrollIds(Arrays.asList(scrollId));
-        ClearContextController controller = new ClearContextController(clearScrollRequest, listener,
+        ClearSearchContextController controller = new ClearSearchContextController(clearScrollRequest, listener,
             nodes, logger, searchTransportService);
         controller.run();
         latch.await();
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index e7524fcb3b74b..99844c010a7bf 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -152,7 +152,7 @@ protected void executeNext(Runnable runnable, Thread originalThread) {
         asyncAction.start();
         latch.await();
         assertTrue(searchPhaseDidRun.get());
-        SearchResponse searchResponse = asyncAction.buildSearchResponse(null, null, asyncAction.buildShardFailures());
+        SearchResponse searchResponse = asyncAction.buildSearchResponse(null, asyncAction.buildShardFailures(), null, null);
         assertEquals(shardsIter.size() - numSkipped, numRequests.get());
         assertEquals(0, searchResponse.getFailedShards());
         assertEquals(numSkipped, searchResponse.getSkippedShards());
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
index e6e5f2af93f20..8171848cfbe4a 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
@@ -184,7 +184,7 @@ public void testValidate() throws IOException {
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNotNull(validationErrors);
             assertEquals(1, validationErrors.validationErrors().size());
-            assertEquals("[index] cannot be used with reader contexts", validationErrors.validationErrors().get(0));
+            assertEquals("[index] cannot be used with search context", validationErrors.validationErrors().get(0));
         }
         {
             // Reader context with preference
@@ -195,7 +195,7 @@ public void testValidate() throws IOException {
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNotNull(validationErrors);
             assertEquals(1, validationErrors.validationErrors().size());
-            assertEquals("[preference] cannot be used with reader contexts", validationErrors.validationErrors().get(0));
+            assertEquals("[preference] cannot be used with search context", validationErrors.validationErrors().get(0));
         }
         {
             // Reader context with routing
@@ -206,7 +206,7 @@ public void testValidate() throws IOException {
             ActionRequestValidationException validationErrors = searchRequest.validate();
             assertNotNull(validationErrors);
             assertEquals(1, validationErrors.validationErrors().size());
-            assertEquals("[routing] cannot be used with reader contexts", validationErrors.validationErrors().get(0));
+            assertEquals("[routing] cannot be used with search context", validationErrors.validationErrors().get(0));
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
index 1bc18dddde16b..cbfaf4ad4b888 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
@@ -50,11 +50,11 @@ public class SearchScrollAsyncActionTests extends ESTestCase {
     public void testSendRequestsToNodes() throws InterruptedException {
 
         ParsedScrollId scrollId = getParsedScrollId(
-            new ReaderIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 1)),
-            new ReaderIdForNode(null, "node2", new SearchContextId(UUIDs.randomBase64UUID(), 2)),
-            new ReaderIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 17)),
-            new ReaderIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 0)),
-            new ReaderIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 0)));
+            new SearchContextIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 1)),
+            new SearchContextIdForNode(null, "node2", new SearchContextId(UUIDs.randomBase64UUID(), 2)),
+            new SearchContextIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 17)),
+            new SearchContextIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 0)),
+            new SearchContextIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -108,10 +108,10 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         latch.await();
         ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
         assertEquals(0, shardSearchFailures.length);
-        ReaderIdForNode[] context = scrollId.getContext();
+        SearchContextIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             assertNotNull(results.get(i));
-            assertEquals(context[i].getContextId(), results.get(i).getContextId());
+            assertEquals(context[i].getSearchContextId(), results.get(i).getContextId());
             assertEquals(context[i].getNode(), results.get(i).node.getId());
         }
     }
@@ -119,11 +119,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
     public void testFailNextPhase() throws InterruptedException {
 
         ParsedScrollId scrollId = getParsedScrollId(
-            new ReaderIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ReaderIdForNode(null, "node2", new SearchContextId("a", 2)),
-            new ReaderIdForNode(null, "node3", new SearchContextId("b", 17)),
-            new ReaderIdForNode(null, "node1", new SearchContextId("c", 0)),
-            new ReaderIdForNode(null, "node3", new SearchContextId("d", 0)));
+            new SearchContextIdForNode(null, "node1", new SearchContextId("", 1)),
+            new SearchContextIdForNode(null, "node2", new SearchContextId("a", 2)),
+            new SearchContextIdForNode(null, "node3", new SearchContextId("b", 17)),
+            new SearchContextIdForNode(null, "node1", new SearchContextId("c", 0)),
+            new SearchContextIdForNode(null, "node3", new SearchContextId("d", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -199,21 +199,21 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         latch.await();
         ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
         assertEquals(0, shardSearchFailures.length);
-        ReaderIdForNode[] context = scrollId.getContext();
+        SearchContextIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             assertNotNull(results.get(i));
-            assertEquals(context[i].getContextId(), results.get(i).getContextId());
+            assertEquals(context[i].getSearchContextId(), results.get(i).getContextId());
             assertEquals(context[i].getNode(), results.get(i).node.getId());
         }
     }
 
     public void testNodeNotAvailable() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ReaderIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ReaderIdForNode(null, "node2", new SearchContextId("", 2)),
-            new ReaderIdForNode(null, "node3", new SearchContextId("", 17)),
-            new ReaderIdForNode(null, "node1", new SearchContextId("", 0)),
-            new ReaderIdForNode(null, "node3", new SearchContextId("", 0)));
+            new SearchContextIdForNode(null, "node1", new SearchContextId("", 1)),
+            new SearchContextIdForNode(null, "node2", new SearchContextId("", 2)),
+            new SearchContextIdForNode(null, "node3", new SearchContextId("", 17)),
+            new SearchContextIdForNode(null, "node1", new SearchContextId("", 0)),
+            new SearchContextIdForNode(null, "node3", new SearchContextId("", 0)));
         // node2 is not available
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -275,13 +275,13 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         // .reason() returns the full stack trace
         assertThat(shardSearchFailures[0].reason(), startsWith("java.lang.IllegalStateException: node [node2] is not available"));
 
-        ReaderIdForNode[] context = scrollId.getContext();
+        SearchContextIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
             if (context[i].getNode().equals("node2")) {
                 assertNull(results.get(i));
             } else {
                 assertNotNull(results.get(i));
-                assertEquals(context[i].getContextId(), results.get(i).getContextId());
+                assertEquals(context[i].getSearchContextId(), results.get(i).getContextId());
                 assertEquals(context[i].getNode(), results.get(i).node.getId());
             }
         }
@@ -289,11 +289,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testShardFailures() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ReaderIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ReaderIdForNode(null, "node2", new SearchContextId("", 2)),
-            new ReaderIdForNode(null, "node3", new SearchContextId("",17)),
-            new ReaderIdForNode(null, "node1", new SearchContextId("", 0)),
-            new ReaderIdForNode(null, "node3", new SearchContextId("", 0)));
+            new SearchContextIdForNode(null, "node1", new SearchContextId("", 1)),
+            new SearchContextIdForNode(null, "node2", new SearchContextId("", 2)),
+            new SearchContextIdForNode(null, "node3", new SearchContextId("",17)),
+            new SearchContextIdForNode(null, "node1", new SearchContextId("", 0)),
+            new SearchContextIdForNode(null, "node3", new SearchContextId("", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -353,13 +353,13 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         assertEquals(1, shardSearchFailures.length);
         assertThat(shardSearchFailures[0].reason(), containsString("IllegalArgumentException: BOOM on shard"));
 
-        ReaderIdForNode[] context = scrollId.getContext();
+        SearchContextIdForNode[] context = scrollId.getContext();
         for (int i = 0; i < results.length(); i++) {
-            if (context[i].getContextId().getId() == 17) {
+            if (context[i].getSearchContextId().getId() == 17) {
                 assertNull(results.get(i));
             } else {
                 assertNotNull(results.get(i));
-                assertEquals(context[i].getContextId(), results.get(i).getContextId());
+                assertEquals(context[i].getSearchContextId(), results.get(i).getContextId());
                 assertEquals(context[i].getNode(), results.get(i).node.getId());
             }
         }
@@ -367,11 +367,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testAllShardsFailed() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new ReaderIdForNode(null, "node1", new SearchContextId("", 1)),
-            new ReaderIdForNode(null, "node2", new SearchContextId("", 2)),
-            new ReaderIdForNode(null, "node3", new SearchContextId("", 17)),
-            new ReaderIdForNode(null, "node1", new SearchContextId("", 0)),
-            new ReaderIdForNode(null, "node3", new SearchContextId("", 0)));
+            new SearchContextIdForNode(null, "node1", new SearchContextId("", 1)),
+            new SearchContextIdForNode(null, "node2", new SearchContextId("", 2)),
+            new SearchContextIdForNode(null, "node3", new SearchContextId("", 17)),
+            new SearchContextIdForNode(null, "node1", new SearchContextId("", 0)),
+            new SearchContextIdForNode(null, "node3", new SearchContextId("", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -433,7 +433,7 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
         action.run();
         latch.await();
-        ReaderIdForNode[] context = scrollId.getContext();
+        SearchContextIdForNode[] context = scrollId.getContext();
 
         ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
         assertEquals(context.length, shardSearchFailures.length);
@@ -444,10 +444,10 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
         }
     }
 
-    private static ParsedScrollId getParsedScrollId(ReaderIdForNode... idsForNodes) {
-        List<ReaderIdForNode> readerIdForNodes = Arrays.asList(idsForNodes);
-        Collections.shuffle(readerIdForNodes, random());
-        return new ParsedScrollId("", "test", readerIdForNodes.toArray(new ReaderIdForNode[0]));
+    private static ParsedScrollId getParsedScrollId(SearchContextIdForNode... idsForNodes) {
+        List<SearchContextIdForNode> searchContextIdForNodes = Arrays.asList(idsForNodes);
+        Collections.shuffle(searchContextIdForNodes, random());
+        return new ParsedScrollId("", "test", searchContextIdForNodes.toArray(new SearchContextIdForNode[0]));
     }
 
     private ActionListener<SearchResponse> dummyListener() {
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
index f18284daedd8e..994102aa23b83 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
@@ -65,55 +65,55 @@ public void testParseScrollId()  {
         assertEquals(3, parseScrollId.getContext().length);
         assertEquals("node_1", parseScrollId.getContext()[0].getNode());
         assertEquals("cluster_x", parseScrollId.getContext()[0].getClusterAlias());
-        assertEquals(1, parseScrollId.getContext()[0].getContextId().getId());
+        assertEquals(1, parseScrollId.getContext()[0].getSearchContextId().getId());
         if (includeUUID) {
-            assertThat(parseScrollId.getContext()[0].getContextId().getReaderId(), equalTo("a"));
+            assertThat(parseScrollId.getContext()[0].getSearchContextId().getReaderId(), equalTo("a"));
         } else {
-            assertThat(parseScrollId.getContext()[0].getContextId().getReaderId(), equalTo(""));
+            assertThat(parseScrollId.getContext()[0].getSearchContextId().getReaderId(), equalTo(""));
         }
 
         assertEquals("node_2", parseScrollId.getContext()[1].getNode());
         assertEquals("cluster_y", parseScrollId.getContext()[1].getClusterAlias());
-        assertEquals(12, parseScrollId.getContext()[1].getContextId().getId());
+        assertEquals(12, parseScrollId.getContext()[1].getSearchContextId().getId());
         if (includeUUID) {
-            assertThat(parseScrollId.getContext()[1].getContextId().getReaderId(), equalTo("b"));
+            assertThat(parseScrollId.getContext()[1].getSearchContextId().getReaderId(), equalTo("b"));
         } else {
-            assertThat(parseScrollId.getContext()[1].getContextId().getReaderId(), equalTo(""));
+            assertThat(parseScrollId.getContext()[1].getSearchContextId().getReaderId(), equalTo(""));
         }
 
         assertEquals("node_3", parseScrollId.getContext()[2].getNode());
         assertNull(parseScrollId.getContext()[2].getClusterAlias());
-        assertEquals(42, parseScrollId.getContext()[2].getContextId().getId());
+        assertEquals(42, parseScrollId.getContext()[2].getSearchContextId().getId());
         if (includeUUID) {
-            assertThat(parseScrollId.getContext()[2].getContextId().getReaderId(), equalTo("c"));
+            assertThat(parseScrollId.getContext()[2].getSearchContextId().getReaderId(), equalTo("c"));
         } else {
-            assertThat(parseScrollId.getContext()[2].getContextId().getReaderId(), equalTo(""));
+            assertThat(parseScrollId.getContext()[2].getSearchContextId().getReaderId(), equalTo(""));
         }
     }
 
     public void testEncodeDecodeReaderId() {
         final AtomicArray<SearchPhaseResult> queryResults = generateQueryResults();
         final Version version = VersionUtils.randomVersion(random());
-        final String readerId = TransportSearchHelper.encodeReaderIds(queryResults, version);
-        final Map<ShardId, ReaderIdForNode> contextIds = TransportSearchHelper.decodeReaderIds(readerId);
+        final String readerId = TransportSearchHelper.encodeSearchContextId(queryResults, version);
+        final Map<ShardId, SearchContextIdForNode> contextIds = TransportSearchHelper.decodeSearchContextId(readerId);
         assertThat(contextIds.keySet(), hasSize(3));
 
-        ReaderIdForNode node1 = contextIds.get(new ShardId("idx", "uuid1", 2));
+        SearchContextIdForNode node1 = contextIds.get(new ShardId("idx", "uuid1", 2));
         assertThat(node1.getClusterAlias(), equalTo("cluster_x"));
         assertThat(node1.getNode(), equalTo("node_1"));
-        assertThat(node1.getContextId().getId(), equalTo(1L));
-        assertThat(node1.getContextId().getReaderId(), equalTo("a"));
+        assertThat(node1.getSearchContextId().getId(), equalTo(1L));
+        assertThat(node1.getSearchContextId().getReaderId(), equalTo("a"));
 
-        ReaderIdForNode node2 = contextIds.get(new ShardId("idy", "uuid2", 42));
+        SearchContextIdForNode node2 = contextIds.get(new ShardId("idy", "uuid2", 42));
         assertThat(node2.getClusterAlias(), equalTo("cluster_y"));
         assertThat(node2.getNode(), equalTo("node_2"));
-        assertThat(node2.getContextId().getId(), equalTo(12L));
-        assertThat(node2.getContextId().getReaderId(), equalTo("b"));
+        assertThat(node2.getSearchContextId().getId(), equalTo(12L));
+        assertThat(node2.getSearchContextId().getReaderId(), equalTo("b"));
 
-        ReaderIdForNode node3 = contextIds.get(new ShardId("idy", "uuid2", 43));
+        SearchContextIdForNode node3 = contextIds.get(new ShardId("idy", "uuid2", 43));
         assertThat(node3.getClusterAlias(), nullValue());
         assertThat(node3.getNode(), equalTo("node_3"));
-        assertThat(node3.getContextId().getId(), equalTo(42L));
-        assertThat(node3.getContextId().getReaderId(), equalTo("c"));
+        assertThat(node3.getSearchContextId().getId(), equalTo(42L));
+        assertThat(node3.getSearchContextId().getReaderId(), equalTo("c"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
index 10db9a7f80421..edca5df7e4d5d 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java
@@ -116,9 +116,8 @@ public void onFreeScrollContext(ReaderContext readerContext) {
             }
 
             @Override
-            public void validateSearchContext(ReaderContext readerContext, SearchContext searchContext, TransportRequest request) {
+            public void validateSearchContext(ReaderContext readerContext, TransportRequest request) {
                 assertNotNull(readerContext);
-                assertNotNull(searchContext);
                 validateSearchContext.incrementAndGet();
             }
         };
@@ -272,10 +271,10 @@ public void validateSearchContext(ReaderContext readerContext, SearchContext sea
         assertEquals(0, validateSearchContext.get());
 
         if (throwingListeners == 0) {
-            compositeListener.validateSearchContext(mock(ReaderContext.class), ctx, Empty.INSTANCE);
+            compositeListener.validateSearchContext(mock(ReaderContext.class), Empty.INSTANCE);
         } else {
             RuntimeException expected = expectThrows(RuntimeException.class,
-                () -> compositeListener.validateSearchContext(mock(ReaderContext.class), ctx, Empty.INSTANCE));
+                () -> compositeListener.validateSearchContext(mock(ReaderContext.class), Empty.INSTANCE));
             assertNull(expected.getMessage());
             assertEquals(throwingListeners - 1, expected.getSuppressed().length);
             if (throwingListeners > 1) {
diff --git a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
index bf86bb90631fb..19611b3c7a8a1 100644
--- a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
+++ b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
@@ -124,7 +124,7 @@ public void testCloseFreezeAndOpen() throws Exception {
             for (int from = 0; from < 3; from++) {
                 searchResponse = client().prepareSearch()
                     .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED)
-                    .setReader(readerId, TimeValue.timeValueMinutes(1))
+                    .setSearchContext(readerId, TimeValue.timeValueMinutes(1))
                     .setSize(1)
                     .setFrom(from)
                     .get();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
index 5c591bdeb4166..c7a6c78bd2057 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.internal.ReaderContext;
-import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContextId;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.SecurityContext;
@@ -55,13 +54,13 @@ public void onNewReaderContext(ReaderContext readerContext) {
      * authentication context
      */
     @Override
-    public void validateSearchContext(ReaderContext readerContext, SearchContext searchContext, TransportRequest request) {
+    public void validateSearchContext(ReaderContext readerContext, TransportRequest request) {
         if (licenseState.isSecurityEnabled()) {
             final Authentication originalAuth = readerContext.getFromContext(AuthenticationField.AUTHENTICATION_KEY);
             final Authentication current = securityContext.getAuthentication();
             final ThreadContext threadContext = securityContext.getThreadContext();
             final String action = threadContext.getTransient(ORIGINATING_ACTION_KEY);
-            ensureAuthenticatedUserIsSame(originalAuth, current, auditTrailService, searchContext.id(), action, request,
+            ensureAuthenticatedUserIsSame(originalAuth, current, auditTrailService, readerContext.id(), action, request,
                 AuditUtil.extractRequestId(threadContext), threadContext.getTransient(AUTHORIZATION_INFO_KEY));
         }
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
index 9389deccb792a..675a7876e757e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
@@ -789,7 +789,7 @@ public void testReaderId() throws Exception {
                     .prepareSearch()
                     .setSize(1)
                     .setFrom(from)
-                    .setReader(readerId, TimeValue.timeValueMinutes(1))
+                    .setSearchContext(readerId, TimeValue.timeValueMinutes(1))
                     .setQuery(termQuery("field1", "value1"))
                     .get();
                 assertNoFailures(response);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
index 17754ae429432..91e911306e75d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
@@ -758,7 +758,7 @@ public void testReaderId() throws Exception {
                 response = client()
                     .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
                     .prepareSearch()
-                    .setReader(readerId, TimeValue.timeValueMinutes(1L))
+                    .setSearchContext(readerId, TimeValue.timeValueMinutes(1L))
                     .setSize(1)
                     .setFrom(from)
                     .setQuery(constantScoreQuery(termQuery("field1", "value1")))
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index b390ccbb85f81..072cf53f942a0 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -70,7 +70,7 @@ public void testUnlicensed() {
             SecuritySearchOperationListener listener =
                 new SecuritySearchOperationListener(securityContext, licenseState, auditTrailService);
             listener.onNewReaderContext(readerContext);
-            listener.validateSearchContext(readerContext, searchContext, Empty.INSTANCE);
+            listener.validateSearchContext(readerContext, Empty.INSTANCE);
             verify(licenseState, times(2)).isSecurityEnabled();
             verifyZeroInteractions(auditTrailService, searchContext);
         }
@@ -119,7 +119,7 @@ public void testValidateSearchContext() throws Exception {
             try (StoredContext ignore = threadContext.newStoredContext(false)) {
                 Authentication authentication = new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
                 authentication.writeToContext(threadContext);
-                listener.validateSearchContext(readerContext, testSearchContext, Empty.INSTANCE);
+                listener.validateSearchContext(readerContext, Empty.INSTANCE);
                 verify(licenseState).isSecurityEnabled();
                 verifyZeroInteractions(auditTrail);
             }
@@ -130,7 +130,7 @@ public void testValidateSearchContext() throws Exception {
                 Authentication authentication =
                     new Authentication(new User("test", "role"), new RealmRef(realmName, "file", nodeName), null);
                 authentication.writeToContext(threadContext);
-                listener.validateSearchContext(readerContext, testSearchContext, Empty.INSTANCE);
+                listener.validateSearchContext(readerContext, Empty.INSTANCE);
                 verify(licenseState, times(2)).isSecurityEnabled();
                 verifyZeroInteractions(auditTrail);
             }
@@ -147,7 +147,7 @@ public void testValidateSearchContext() throws Exception {
                     (AuthorizationInfo) () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, authentication.getUser().roles()));
                 final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
                 SearchContextMissingException expected = expectThrows(SearchContextMissingException.class,
-                    () -> listener.validateSearchContext(readerContext, testSearchContext, request));
+                    () -> listener.validateSearchContext(readerContext, request));
                 assertEquals(testSearchContext.id(), expected.contextId());
                 verify(licenseState, Mockito.atLeast(3)).isSecurityEnabled();
                 verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
@@ -165,7 +165,7 @@ public void testValidateSearchContext() throws Exception {
                 authentication.writeToContext(threadContext);
                 threadContext.putTransient(ORIGINATING_ACTION_KEY, "action");
                 final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
-                listener.validateSearchContext(readerContext, testSearchContext, request);
+                listener.validateSearchContext(readerContext, request);
                 verify(licenseState, Mockito.atLeast(4)).isSecurityEnabled();
                 verifyNoMoreInteractions(auditTrail);
             }
@@ -183,7 +183,7 @@ public void testValidateSearchContext() throws Exception {
                     (AuthorizationInfo) () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, authentication.getUser().roles()));
                 final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
                 SearchContextMissingException expected = expectThrows(SearchContextMissingException.class,
-                    () -> listener.validateSearchContext(readerContext, testSearchContext, request));
+                    () -> listener.validateSearchContext(readerContext, request));
                 assertEquals(testSearchContext.id(), expected.contextId());
                 verify(licenseState, Mockito.atLeast(5)).isSecurityEnabled();
                 verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
diff --git a/x-pack/plugin/sql/qa/security/with-ssl/build.gradle b/x-pack/plugin/sql/qa/security/with-ssl/build.gradle
index 91de5df698f00..6f82ea53ef70d 100644
--- a/x-pack/plugin/sql/qa/security/with-ssl/build.gradle
+++ b/x-pack/plugin/sql/qa/security/with-ssl/build.gradle
@@ -9,6 +9,9 @@ integTest.runner {
 // needed to be consistent with ssl host checking
 Object san = new SanEvaluator()
 
+// needed to be consistent with ssl host checking
+String host = InetAddress.getLoopbackAddress().getHostAddress();
+
 // location of generated keystores and certificates
 File keystoreDir = new File(project.buildDir, 'keystore')
 
@@ -23,7 +26,7 @@ task createNodeKeyStore(type: LoggedExec) {
       delete nodeKeystore
     }
   }
-  executable = "${BuildParams.runtimeJavaHome}/bin/keytool"
+  executable = "${BuildParams.compilerJavaHome}/bin/keytool"
   standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
   args '-genkey',
     '-alias', 'test-node',
@@ -31,7 +34,7 @@ task createNodeKeyStore(type: LoggedExec) {
     '-keyalg', 'RSA',
     '-keysize', '2048',
     '-validity', '712',
-    '-dname', 'CN=smoke-test-plugins-ssl',
+    '-dname', 'CN=' + host,
     '-keypass', 'keypass',
     '-storepass', 'keypass',
     '-ext', san
@@ -56,7 +59,7 @@ task createClientKeyStore(type: LoggedExec) {
     '-keyalg', 'RSA',
     '-keysize', '2048',
     '-validity', '712',
-    '-dname', 'CN=smoke-test-plugins-ssl',
+    '-dname', 'CN=' + host,
     '-keypass', 'keypass',
     '-storepass', 'keypass',
     '-ext', san

From 13ef912c332b95d8e0c97102baa9441e6059fe37 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sat, 9 May 2020 12:47:47 -0400
Subject: [PATCH 19/47] fix security listener

---
 .../SecuritySearchOperationListenerTests.java | 25 ++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index 072cf53f942a0..2b6c02021dba3 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -16,12 +16,12 @@
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.search.internal.LegacyReaderContext;
 import org.elasticsearch.search.internal.ReaderContext;
-import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequest.Empty;
 import org.elasticsearch.xpack.core.security.SecurityContext;
@@ -54,7 +54,7 @@ public class SecuritySearchOperationListenerTests extends ESSingleNodeTestCase {
     private IndexShard shard;
 
     @Before
-    private void setupShard() {
+    public void setupShard() {
         shard = createIndex("index").getShard(0);
     }
 
@@ -99,12 +99,17 @@ public void testOnNewContextSetsAuthentication() throws Exception {
     }
 
     public void testValidateSearchContext() throws Exception {
-        try (ReaderContext readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false)) {
-            ScrollContext scrollContext = new ScrollContext();
-            TestSearchContext testSearchContext = new TestSearchContext(null, null, null, scrollContext);
+        final ReaderContext readerContext;
+        if (randomBoolean()) {
+            readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false);
+        } else {
+            ShardSearchRequest request = mock(ShardSearchRequest.class);
+            when(request.scroll()).thenReturn(new Scroll(TimeValue.timeValueMinutes(between(1, 10))));
+            readerContext = new LegacyReaderContext(0L, shard, shard.acquireSearcherSupplier(), request, Long.MAX_VALUE);
+        }
+        try {
             readerContext.putInContext(AuthenticationField.AUTHENTICATION_KEY,
                 new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null));
-            testSearchContext.scrollContext().scroll = new Scroll(TimeValue.timeValueSeconds(2L));
             XPackLicenseState licenseState = mock(XPackLicenseState.class);
             when(licenseState.isSecurityEnabled()).thenReturn(true);
             when(licenseState.isAllowed(Feature.SECURITY_AUDITING)).thenReturn(true);
@@ -148,7 +153,7 @@ public void testValidateSearchContext() throws Exception {
                 final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
                 SearchContextMissingException expected = expectThrows(SearchContextMissingException.class,
                     () -> listener.validateSearchContext(readerContext, request));
-                assertEquals(testSearchContext.id(), expected.contextId());
+                assertEquals(readerContext.id(), expected.contextId());
                 verify(licenseState, Mockito.atLeast(3)).isSecurityEnabled();
                 verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
                     authzInfoRoles(authentication.getUser().roles()));
@@ -184,11 +189,13 @@ public void testValidateSearchContext() throws Exception {
                 final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
                 SearchContextMissingException expected = expectThrows(SearchContextMissingException.class,
                     () -> listener.validateSearchContext(readerContext, request));
-                assertEquals(testSearchContext.id(), expected.contextId());
+                assertEquals(readerContext.id(), expected.contextId());
                 verify(licenseState, Mockito.atLeast(5)).isSecurityEnabled();
                 verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
                     authzInfoRoles(authentication.getUser().roles()));
             }
+        } finally {
+            readerContext.close();
         }
     }
 

From 6bc453f08198120b690aaced837599e74689bd25 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sat, 9 May 2020 14:21:29 -0400
Subject: [PATCH 20/47] simplify validation of reader context

---
 .../index/shard/SearchOperationListener.java  |  3 +-
 .../elasticsearch/search/SearchService.java   | 47 ++++++++-----------
 .../SecuritySearchOperationListenerTests.java | 18 +------
 3 files changed, 22 insertions(+), 46 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
index 777f35b569b7b..c0d98b434a300 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
@@ -110,8 +110,7 @@ default void onFreeScrollContext(ReaderContext readerContext) {}
      * Executed prior to using a {@link ReaderContext} that has been retrieved
      * from the active contexts. If the context is deemed invalid a runtime
      * exception can be thrown, which will prevent the context from being used.
-     *
-     * @param readerContext    The reader context used by this request.
+     * @param readerContext The reader context used by this request.
      * @param transportRequest the request that is going to use the search context
      */
     default void validateSearchContext(ReaderContext readerContext, TransportRequest transportRequest) {}
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 99d31c1e5c8b5..29f88d7423cc2 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchShardTask;
@@ -112,6 +111,7 @@
 import org.elasticsearch.threadpool.Scheduler.Cancellable;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Names;
+import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -443,10 +443,6 @@ private <T> void runAsync(IndexShard shard, CheckedSupplier<T, Exception> comman
         });
     }
 
-    private <T> void runAsync(SearchContextId id, CheckedSupplier<T, IOException> executable, ActionListener<T> listener) {
-        getExecutor(id).execute(ActionRunnable.supply(listener, executable::get));
-    }
-
     private SearchPhaseResult executeQueryPhase(ShardSearchRequest request,
                                                 SearchShardTask task,
                                                 ReaderContext readerContext) throws Exception {
@@ -490,9 +486,9 @@ private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchCon
     public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
-        runAsync(request.contextId(), () -> {
-            final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
-            final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
+        final LegacyReaderContext readerContext = (LegacyReaderContext) findAndValidateReaderContext(request.contextId(), request);
+        final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
+        runAsync(readerContext.indexShard(), () -> {
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
@@ -518,8 +514,8 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
     }
 
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
-        runAsync(request.contextId(), () -> {
-            final ReaderContext readerContext = findReaderContext(request.contextId());
+        final ReaderContext readerContext = findAndValidateReaderContext(request.contextId(), request);
+        runAsync(readerContext.indexShard(), () -> {
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.shardSearchRequest());
             readerContext.setAggregatedDfs(request.dfs());
             try (Releasable ignored = readerContext.markAsUsed();
@@ -544,10 +540,6 @@ public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task,
         }, listener);
     }
 
-    final Executor getExecutor(SearchContextId id) {
-        return getExecutor(findReaderContext(id).indexShard());
-    }
-
     private Executor getExecutor(IndexShard indexShard) {
         assert indexShard != null;
         return threadPool.executor(indexShard.indexSettings().isSearchThrottled() ? Names.SEARCH_THROTTLED : Names.SEARCH);
@@ -555,8 +547,8 @@ private Executor getExecutor(IndexShard indexShard) {
 
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
-        runAsync(request.contextId(), () -> {
-            final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
+        final LegacyReaderContext readerContext = (LegacyReaderContext) findAndValidateReaderContext(request.contextId(), request);
+        runAsync(readerContext.indexShard(), () -> {
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
@@ -581,8 +573,8 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
     }
 
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
-        runAsync(request.contextId(), () -> {
-            final ReaderContext readerContext = findReaderContext(request.contextId());
+        final ReaderContext readerContext = findAndValidateReaderContext(request.contextId(), request);
+        runAsync(readerContext.indexShard(), () -> {
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest());
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false)) {
@@ -620,18 +612,20 @@ private ReaderContext getReaderContext(SearchContextId id) {
         return null;
     }
 
-    private ReaderContext findReaderContext(SearchContextId id) throws SearchContextMissingException {
-        final ReaderContext reader = getReaderContext(id);
-        if (reader == null) {
+    private ReaderContext findAndValidateReaderContext(SearchContextId id, TransportRequest request) throws SearchContextMissingException {
+        final ReaderContext readerContext = getReaderContext(id);
+        if (readerContext == null) {
             throw new SearchContextMissingException(id);
         }
-        return reader;
+        readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
+        return readerContext;
     }
 
     final ReaderContext createOrGetReaderContext(ShardSearchRequest request, boolean keepStatesInContext) {
         if (request.readerId() != null) {
             assert keepStatesInContext == false;
-            final ReaderContext readerContext = findReaderContext(request.readerId());
+            // NORELEASE: either wrap the searcher with the right security context or make sure the user owns it.
+            final ReaderContext readerContext = findAndValidateReaderContext(request.readerId(), request);
             final long keepAlive = request.keepAlive().millis();
             checkKeepAliveLimit(keepAlive);
             readerContext.keepAlive(keepAlive);
@@ -725,12 +719,11 @@ public void openReaderContext(ShardId shardId, TimeValue keepAlive, ActionListen
         });
     }
 
-    final SearchContext createContext(ReaderContext readerContext,
+    final SearchContext createContext(ReaderContext reader,
                                       ShardSearchRequest request,
                                       SearchShardTask task,
                                       boolean includeAggregations) throws IOException {
-        readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
-        final DefaultSearchContext context = createSearchContext(readerContext, request, defaultSearchTimeout);
+        final DefaultSearchContext context = createSearchContext(reader, request, defaultSearchTimeout);
         try {
             if (request.scroll() != null) {
                 context.scrollContext().scroll = request.scroll();
@@ -1129,7 +1122,7 @@ public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException
         final boolean hasRefreshPending;
         final Engine.Searcher canMatchSearcher;
         if (request.readerId() != null) {
-            final ReaderContext readerContext = findReaderContext(request.readerId());
+            final ReaderContext readerContext = findAndValidateReaderContext(request.readerId(), request);
             checkKeepAliveLimit(request.keepAlive().millis());
             readerContext.keepAlive(request.keepAlive().millis());
             Releasable releasable = readerContext.markAsUsed();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index 2b6c02021dba3..d2077f5a7eece 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -7,20 +7,16 @@
 
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.license.XPackLicenseState.Feature;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.LegacyReaderContext;
 import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContextId;
-import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequest.Empty;
@@ -99,17 +95,7 @@ public void testOnNewContextSetsAuthentication() throws Exception {
     }
 
     public void testValidateSearchContext() throws Exception {
-        final ReaderContext readerContext;
-        if (randomBoolean()) {
-            readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false);
-        } else {
-            ShardSearchRequest request = mock(ShardSearchRequest.class);
-            when(request.scroll()).thenReturn(new Scroll(TimeValue.timeValueMinutes(between(1, 10))));
-            readerContext = new LegacyReaderContext(0L, shard, shard.acquireSearcherSupplier(), request, Long.MAX_VALUE);
-        }
-        try {
-            readerContext.putInContext(AuthenticationField.AUTHENTICATION_KEY,
-                new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null));
+        try (ReaderContext readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false)) {
             XPackLicenseState licenseState = mock(XPackLicenseState.class);
             when(licenseState.isSecurityEnabled()).thenReturn(true);
             when(licenseState.isAllowed(Feature.SECURITY_AUDITING)).thenReturn(true);
@@ -194,8 +180,6 @@ public void testValidateSearchContext() throws Exception {
                 verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
                     authzInfoRoles(authentication.getUser().roles()));
             }
-        } finally {
-            readerContext.close();
         }
     }
 

From 63a35955edf210faa873e5a73eece492acb6a041 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sat, 9 May 2020 16:02:56 -0400
Subject: [PATCH 21/47] more on renaming

---
 .../search/TransportSearchHelperTests.java     |  2 +-
 .../xpack/security/authz/RBACEngine.java       |  1 -
 .../SecuritySearchOperationListenerTests.java  | 18 +++++++++++++++++-
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
index 994102aa23b83..d4de57d3cec2e 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
@@ -91,7 +91,7 @@ public void testParseScrollId()  {
         }
     }
 
-    public void testEncodeDecodeReaderId() {
+    public void testEncodeDecodeSearchContext() {
         final AtomicArray<SearchPhaseResult> queryResults = generateQueryResults();
         final Version version = VersionUtils.randomVersion(random());
         final String readerId = TransportSearchHelper.encodeSearchContextId(queryResults, version);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
index e105b23b401b0..4ac232fb10d09 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
@@ -279,7 +279,6 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth
                     listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
                 }
             } else if (action.equals(CloseSearchContextAction.NAME)) {
-                // NORELEASE: ensure security works correctly
                 listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
             } else {
                 assert false : "only scroll and async-search related requests are known indices api that don't " +
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index d2077f5a7eece..2b6c02021dba3 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -7,16 +7,20 @@
 
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.license.XPackLicenseState.Feature;
+import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
+import org.elasticsearch.search.internal.LegacyReaderContext;
 import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequest.Empty;
@@ -95,7 +99,17 @@ public void testOnNewContextSetsAuthentication() throws Exception {
     }
 
     public void testValidateSearchContext() throws Exception {
-        try (ReaderContext readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false)) {
+        final ReaderContext readerContext;
+        if (randomBoolean()) {
+            readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false);
+        } else {
+            ShardSearchRequest request = mock(ShardSearchRequest.class);
+            when(request.scroll()).thenReturn(new Scroll(TimeValue.timeValueMinutes(between(1, 10))));
+            readerContext = new LegacyReaderContext(0L, shard, shard.acquireSearcherSupplier(), request, Long.MAX_VALUE);
+        }
+        try {
+            readerContext.putInContext(AuthenticationField.AUTHENTICATION_KEY,
+                new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null));
             XPackLicenseState licenseState = mock(XPackLicenseState.class);
             when(licenseState.isSecurityEnabled()).thenReturn(true);
             when(licenseState.isAllowed(Feature.SECURITY_AUDITING)).thenReturn(true);
@@ -180,6 +194,8 @@ public void testValidateSearchContext() throws Exception {
                 verify(auditTrail).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
                     authzInfoRoles(authentication.getUser().roles()));
             }
+        } finally {
+            readerContext.close();
         }
     }
 

From 454f7fb1450d38d0433b08fbf2b00b178f5d95a4 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sat, 9 May 2020 18:16:23 -0400
Subject: [PATCH 22/47] bwc: remove scroll on failed authorization

---
 .../elasticsearch/search/SearchService.java   | 52 ++++++++++++-------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 29f88d7423cc2..bd6654d4505e1 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchShardTask;
@@ -111,7 +112,6 @@
 import org.elasticsearch.threadpool.Scheduler.Cancellable;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Names;
-import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -443,6 +443,10 @@ private <T> void runAsync(IndexShard shard, CheckedSupplier<T, Exception> comman
         });
     }
 
+    private <T> void runAsync(SearchContextId id, CheckedSupplier<T, IOException> executable, ActionListener<T> listener) {
+        getExecutor(id).execute(ActionRunnable.supply(listener, executable::get));
+    }
+
     private SearchPhaseResult executeQueryPhase(ShardSearchRequest request,
                                                 SearchShardTask task,
                                                 ReaderContext readerContext) throws Exception {
@@ -486,12 +490,13 @@ private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchCon
     public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
-        final LegacyReaderContext readerContext = (LegacyReaderContext) findAndValidateReaderContext(request.contextId(), request);
-        final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
-        runAsync(readerContext.indexShard(), () -> {
+        runAsync(request.contextId(), () -> {
+            final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
+            final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
+                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
                 if (request.scroll() != null && request.scroll().keepAlive() != null) {
                     final long keepAlive = request.scroll().keepAlive().millis();
                     checkKeepAliveLimit(keepAlive);
@@ -514,13 +519,14 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
     }
 
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
-        final ReaderContext readerContext = findAndValidateReaderContext(request.contextId(), request);
-        runAsync(readerContext.indexShard(), () -> {
+        runAsync(request.contextId(), () -> {
+            final ReaderContext readerContext = findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.shardSearchRequest());
             readerContext.setAggregatedDfs(request.dfs());
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, true);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
+                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
                 searchContext.searcher().setAggregatedDfs(request.dfs());
                 queryPhase.execute(searchContext);
                 if (searchContext.queryResult().hasSearchContext() == false && readerContext.singleSession()) {
@@ -540,6 +546,10 @@ public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task,
         }, listener);
     }
 
+    final Executor getExecutor(SearchContextId id) {
+        return getExecutor(findReaderContext(id).indexShard());
+    }
+
     private Executor getExecutor(IndexShard indexShard) {
         assert indexShard != null;
         return threadPool.executor(indexShard.indexSettings().isSearchThrottled() ? Names.SEARCH_THROTTLED : Names.SEARCH);
@@ -547,12 +557,13 @@ private Executor getExecutor(IndexShard indexShard) {
 
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
-        final LegacyReaderContext readerContext = (LegacyReaderContext) findAndValidateReaderContext(request.contextId(), request);
-        runAsync(readerContext.indexShard(), () -> {
+        runAsync(request.contextId(), () -> {
+            final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
                  SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext)) {
+                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
                 if (request.scroll() != null && request.scroll().keepAlive() != null) {
                     checkKeepAliveLimit(request.scroll().keepAlive().millis());
                     readerContext.keepAlive(request.scroll().keepAlive().millis());
@@ -573,11 +584,12 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
     }
 
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
-        final ReaderContext readerContext = findAndValidateReaderContext(request.contextId(), request);
-        runAsync(readerContext.indexShard(), () -> {
+        runAsync(request.contextId(), () -> {
+            final ReaderContext readerContext = findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest());
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false)) {
+                readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
                 if (request.lastEmittedDoc() != null) {
                     searchContext.scrollContext().lastEmittedDoc = request.lastEmittedDoc();
                 }
@@ -612,20 +624,19 @@ private ReaderContext getReaderContext(SearchContextId id) {
         return null;
     }
 
-    private ReaderContext findAndValidateReaderContext(SearchContextId id, TransportRequest request) throws SearchContextMissingException {
-        final ReaderContext readerContext = getReaderContext(id);
-        if (readerContext == null) {
+    private ReaderContext findReaderContext(SearchContextId id) throws SearchContextMissingException {
+        final ReaderContext reader = getReaderContext(id);
+        if (reader == null) {
             throw new SearchContextMissingException(id);
         }
-        readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
-        return readerContext;
+        return reader;
     }
 
     final ReaderContext createOrGetReaderContext(ShardSearchRequest request, boolean keepStatesInContext) {
         if (request.readerId() != null) {
             assert keepStatesInContext == false;
-            // NORELEASE: either wrap the searcher with the right security context or make sure the user owns it.
-            final ReaderContext readerContext = findAndValidateReaderContext(request.readerId(), request);
+            final ReaderContext readerContext = findReaderContext(request.readerId());
+            readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
             final long keepAlive = request.keepAlive().millis();
             checkKeepAliveLimit(keepAlive);
             readerContext.keepAlive(keepAlive);
@@ -719,11 +730,11 @@ public void openReaderContext(ShardId shardId, TimeValue keepAlive, ActionListen
         });
     }
 
-    final SearchContext createContext(ReaderContext reader,
+    final SearchContext createContext(ReaderContext readerContext,
                                       ShardSearchRequest request,
                                       SearchShardTask task,
                                       boolean includeAggregations) throws IOException {
-        final DefaultSearchContext context = createSearchContext(reader, request, defaultSearchTimeout);
+        final DefaultSearchContext context = createSearchContext(readerContext, request, defaultSearchTimeout);
         try {
             if (request.scroll() != null) {
                 context.scrollContext().scroll = request.scroll();
@@ -1122,7 +1133,8 @@ public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException
         final boolean hasRefreshPending;
         final Engine.Searcher canMatchSearcher;
         if (request.readerId() != null) {
-            final ReaderContext readerContext = findAndValidateReaderContext(request.readerId(), request);
+            final ReaderContext readerContext = findReaderContext(request.readerId());
+            readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
             checkKeepAliveLimit(request.keepAlive().millis());
             readerContext.keepAlive(request.keepAlive().millis());
             Releasable releasable = readerContext.markAsUsed();

From 085c4ea197bffadc330ee59627e326dce28de045 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Tue, 12 May 2020 17:49:29 -0400
Subject: [PATCH 23/47] Remove reader context if index gets index midway

---
 .../elasticsearch/search/SearchService.java   | 28 +++++++++++--------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 2fd5bd7ab5899..bcad618b44103 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -700,6 +700,9 @@ final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexS
             readerContext = null;
             return finalReaderContext;
         } finally {
+            if (readerContext != null) {
+                removeReaderContext(readerContext.id().getId());
+            }
             Releasables.close(reader, readerContext, decreaseScrollContexts);
         }
     }
@@ -714,22 +717,25 @@ public void openReaderContext(ShardId shardId, TimeValue keepAlive, ActionListen
         final IndexShard shard = indexService.getShard(shardId.id());
         final SearchOperationListener searchOperationListener = shard.getSearchOperationListener();
         shard.awaitShardSearchActive(ignored -> {
-            Releasable releasable = null;
+            Engine.SearcherSupplier searcherSupplier = null;
+            ReaderContext readerContext = null;
             try {
-                final Engine.SearcherSupplier reader = shard.acquireSearcherSupplier();
-                releasable = reader;
-                final ReaderContext readerContext = new ReaderContext(
-                    idGenerator.incrementAndGet(), shard, reader, keepAlive.millis(), false);
-                releasable = readerContext;
+                searcherSupplier = shard.acquireSearcherSupplier();
+                readerContext = new ReaderContext(
+                    idGenerator.incrementAndGet(), shard, searcherSupplier, keepAlive.millis(), false);
+                final ReaderContext finalReaderContext = readerContext;
+                searcherSupplier = null; // transfer ownership to reader context
                 searchOperationListener.onNewReaderContext(readerContext);
-                readerContext.addOnClose(() -> searchOperationListener.onFreeReaderContext(readerContext));
+                readerContext.addOnClose(() -> searchOperationListener.onFreeReaderContext(finalReaderContext));
                 putReaderContext(readerContext);
-                releasable = null;
-                listener.onResponse(readerContext.id());
+                readerContext = null;
+                listener.onResponse(finalReaderContext.id());
             } catch (Exception exc) {
+                if (readerContext != null) {
+                    removeReaderContext(readerContext.id().getId());
+                }
+                Releasables.closeWhileHandlingException(searcherSupplier, readerContext);
                 listener.onFailure(exc);
-            } finally {
-                Releasables.close(releasable);
             }
         });
     }

From d654b263970c570677c951fbdfb12018b9311019 Mon Sep 17 00:00:00 2001
From: jimczi <jim.ferenczi@elastic.co>
Date: Wed, 27 May 2020 10:00:15 +0200
Subject: [PATCH 24/47] remove unused code

---
 .../action/search/AbstractSearchAsyncAction.java           | 2 +-
 .../main/java/org/elasticsearch/search/SearchService.java  | 2 +-
 .../elasticsearch/search/internal/ShardSearchRequest.java  | 7 -------
 3 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 9911adf079f7e..9c84c6ebf396a 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -623,7 +623,7 @@ public final ShardSearchRequest buildShardSearchRequest(SearchShardIterator shar
         // can return a null response if the request rewrites to match none rather
         // than creating an empty response in the search thread pool.
         // Note that, we have to disable this shortcut for queries that create a context (scroll and search context).
-        shardRequest.canReturnNullResponseIfMatchNoDocs(hasShardResponse.get() && shardRequest.shouldCreatePersistentReader() == false);
+        shardRequest.canReturnNullResponseIfMatchNoDocs(hasShardResponse.get() && shardRequest.scroll() == null);
         return shardRequest;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index bcad618b44103..fb01c6a9f207b 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -381,7 +381,7 @@ public void onResponse(ShardSearchRequest orig) {
                 final Releasable releasable = readerContext.markAsUsed();
                 try {
                     if (orig.canReturnNullResponseIfMatchNoDocs()) {
-                        assert orig.shouldCreatePersistentReader() == false;
+                        assert orig.scroll() == null;
                         // we clone the shard request and perform a quick rewrite using a lightweight
                         // searcher since we are outside of the search thread pool.
                         // If the request rewrites to "match none" we can shortcut the query phase
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index 5b16937749516..8386e57003d8a 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -381,13 +381,6 @@ public void canReturnNullResponseIfMatchNoDocs(boolean value) {
         this.canReturnNullResponseIfMatchNoDocs = value;
     }
 
-    /**
-     * Returns <code>true</code> if the request needs to create a long-lived context.
-     */
-    public boolean shouldCreatePersistentReader() {
-        return scroll != null  || (keepAlive != null && readerId == null);
-    }
-
     /**
      * Returns a non-null value if this request should execute using a specific point-in-time reader;
      * otherwise, using the most up to date point-in-time reader.

From 0d0d647e544edf93bdf8bec27ae58f66829a0c7a Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 4 Jun 2020 13:31:34 -0400
Subject: [PATCH 25/47] Use IndexShard from reader context (#57384)

Previously, we resolve IndexService and IndexShard once when creating a
search context. However, since #51282, we resolve them in every phase as
we recreate a search context in each phase. This change can cause the
dfs_query or fetch phase to fail with ShardNotFoundException while
previously did not. This behavior is quite subtle, but I think we should
make it in line with the master branch.
---
 .../search/DefaultSearchContext.java          |  6 +--
 .../elasticsearch/search/SearchService.java   | 45 +++++++++++--------
 .../search/internal/LegacyReaderContext.java  |  5 ++-
 .../search/internal/ReaderContext.java        |  7 +++
 .../search/DefaultSearchContextTests.java     | 24 +++++-----
 .../search/SearchServiceTests.java            | 29 ++++++------
 .../SecuritySearchOperationListenerTests.java | 15 ++++---
 7 files changed, 78 insertions(+), 53 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 945e5e4673bd9..93ab95ae99e69 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -152,8 +152,6 @@ final class DefaultSearchContext extends SearchContext {
                          ShardSearchRequest request,
                          SearchShardTarget shardTarget,
                          ClusterService clusterService,
-                         IndexService indexService,
-                         IndexShard indexShard,
                          BigArrays bigArrays,
                          LongSupplier relativeTimeSupplier,
                          TimeValue timeout,
@@ -169,8 +167,8 @@ final class DefaultSearchContext extends SearchContext {
         this.dfsResult = new DfsSearchResult(readerContext.id(), shardTarget, request);
         this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget, request);
         this.fetchResult = new FetchSearchResult(readerContext.id(), shardTarget);
-        this.indexShard = indexShard;
-        this.indexService = indexService;
+        this.indexService = readerContext.indexService();
+        this.indexShard = readerContext.indexShard();
         this.clusterService = clusterService;
         this.engineSearcher = readerContext.acquireSearcher("search");
         this.searcher = new ContextIndexSearcher(engineSearcher.getIndexReader(), engineSearcher.getSimilarity(),
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index fb01c6a9f207b..d49c08b1bd247 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchShardTask;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedSupplier;
@@ -320,8 +321,7 @@ protected void doClose() {
 
     public void executeDfsPhase(ShardSearchRequest request, boolean keepStatesInContext,
                                 SearchShardTask task, ActionListener<SearchPhaseResult> listener) {
-        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-        IndexShard shard = indexService.getShard(request.shardId().id());
+        final IndexShard shard = getShard(request);
         rewriteAndFetchShardRequest(shard, request, new ActionListener<ShardSearchRequest>() {
             @Override
             public void onResponse(ShardSearchRequest rewritten) {
@@ -372,8 +372,7 @@ public void executeQueryPhase(ShardSearchRequest request, boolean keepStatesInCo
                                   SearchShardTask task, ActionListener<SearchPhaseResult> listener) {
         assert request.canReturnNullResponseIfMatchNoDocs() == false || request.numberOfShards() > 1
             : "empty responses require more than one shard";
-        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-        IndexShard shard = indexService.getShard(request.shardId().id());
+        final IndexShard shard = getShard(request);
         rewriteAndFetchShardRequest(shard, request, new ActionListener<ShardSearchRequest>() {
             @Override
             public void onResponse(ShardSearchRequest orig) {
@@ -388,8 +387,8 @@ public void onResponse(ShardSearchRequest orig) {
                         // entirely. Otherwise we fork the execution in the search thread pool.
                         ShardSearchRequest canMatchRequest = new ShardSearchRequest(orig);
                         try (Engine.Searcher searcher = readerContext.acquireSearcher("can_match")) {
-                            QueryShardContext context = indexService.newQueryShardContext(canMatchRequest.shardId().id(), searcher,
-                                canMatchRequest::nowInMillis, canMatchRequest.getClusterAlias());
+                            QueryShardContext context = readerContext.indexService().newQueryShardContext(canMatchRequest.shardId().id(),
+                                searcher, canMatchRequest::nowInMillis, canMatchRequest.getClusterAlias());
                             Rewriteable.rewrite(canMatchRequest.getRewriteable(), context, true);
                         }
                         if (canRewriteToMatchNone(canMatchRequest.source())
@@ -433,6 +432,14 @@ public void onFailure(Exception exc) {
         });
     }
 
+    private IndexShard getShard(ShardSearchRequest request) {
+        if (request.readerId() != null) {
+            return findReaderContext(request.readerId()).indexShard();
+        } else {
+            return indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id());
+        }
+    }
+
     private <T> void runAsync(IndexShard shard, CheckedSupplier<T, Exception> command, ActionListener<T> listener) {
         Executor executor = getExecutor(shard);
         executor.execute(() -> {
@@ -543,6 +550,7 @@ public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task,
                 readerContext.setRescoreDocIds(rescoreDocIds);
                 return searchContext.queryResult();
             } catch (Exception e) {
+                assert TransportActions.isShardNotAvailableException(e) == false : new AssertionError(e);
                 logger.trace("Query phase failed", e);
                 processFailure(readerContext, e);
                 throw e;
@@ -580,6 +588,7 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
                 QueryFetchSearchResult fetchSearchResult = executeFetchPhase(readerContext, searchContext, afterQueryTime);
                 return new ScrollQueryFetchSearchResult(fetchSearchResult, searchContext.shardTarget());
             } catch (Exception e) {
+                assert TransportActions.isShardNotAvailableException(e) == false : new AssertionError(e);
                 logger.trace("Fetch phase failed", e);
                 processFailure(readerContext, e);
                 throw e;
@@ -610,6 +619,7 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
                 }
                 return searchContext.fetchResult();
             } catch (Exception e) {
+                assert TransportActions.isShardNotAvailableException(e) == false : new AssertionError(e);
                 logger.trace("Fetch phase failed", e);
                 processFailure(readerContext, e);
                 throw e;
@@ -649,10 +659,10 @@ final ReaderContext createOrGetReaderContext(ShardSearchRequest request, boolean
         IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
         IndexShard shard = indexService.getShard(request.shardId().id());
         Engine.SearcherSupplier reader = shard.acquireSearcherSupplier();
-        return createAndPutReaderContext(request, shard, reader, keepStatesInContext);
+        return createAndPutReaderContext(request, indexService, shard, reader, keepStatesInContext);
     }
 
-    final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexShard shard,
+    final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexService indexService, IndexShard shard,
                                                   Engine.SearcherSupplier reader, boolean keepStatesInContext) {
         assert request.readerId() == null;
         assert request.keepAlive() == null;
@@ -671,13 +681,13 @@ final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexS
             final long keepAlive = getKeepAlive(request);
             checkKeepAliveLimit(keepAlive);
             if (keepStatesInContext || request.scroll() != null) {
-                readerContext = new LegacyReaderContext(idGenerator.incrementAndGet(), shard, reader, request, keepAlive);
+                readerContext = new LegacyReaderContext(idGenerator.incrementAndGet(), indexService, shard, reader, request, keepAlive);
                 if (request.scroll() != null) {
                     readerContext.addOnClose(decreaseScrollContexts);
                     decreaseScrollContexts = null;
                 }
             } else {
-                readerContext = new ReaderContext(idGenerator.incrementAndGet(), shard, reader, keepAlive,
+                readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexService, shard, reader, keepAlive,
                     request.keepAlive() == null);
             }
             reader = null;
@@ -722,7 +732,7 @@ public void openReaderContext(ShardId shardId, TimeValue keepAlive, ActionListen
             try {
                 searcherSupplier = shard.acquireSearcherSupplier();
                 readerContext = new ReaderContext(
-                    idGenerator.incrementAndGet(), shard, searcherSupplier, keepAlive.millis(), false);
+                    idGenerator.incrementAndGet(), indexService, shard, searcherSupplier, keepAlive.millis(), false);
                 final ReaderContext finalReaderContext = readerContext;
                 searcherSupplier = null; // transfer ownership to reader context
                 searchOperationListener.onNewReaderContext(readerContext);
@@ -773,9 +783,10 @@ final SearchContext createContext(ReaderContext readerContext,
     }
 
     public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout) throws IOException {
-        IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().getId());
-        Engine.SearcherSupplier reader = indexShard.acquireSearcherSupplier();
-        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexShard, reader, -1L, true)) {
+        final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
+        final IndexShard indexShard = indexService.getShard(request.shardId().getId());
+        final Engine.SearcherSupplier reader = indexShard.acquireSearcherSupplier();
+        try (ReaderContext readerContext = new ReaderContext(idGenerator.incrementAndGet(), indexService, indexShard, reader, -1L, true)) {
             DefaultSearchContext searchContext = createSearchContext(readerContext, request, timeout);
             searchContext.addReleasable(readerContext.markAsUsed());
             return searchContext;
@@ -787,11 +798,9 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
         boolean success = false;
         DefaultSearchContext searchContext = null;
         try {
-            IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-            IndexShard indexShard = indexService.getShard(request.shardId().getId());
             SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(),
-                indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE);
-            searchContext = new DefaultSearchContext(reader, request, shardTarget, clusterService, indexService, indexShard,
+                reader.indexShard().shardId(), request.getClusterAlias(), OriginalIndices.NONE);
+            searchContext = new DefaultSearchContext(reader, request, shardTarget, clusterService,
                 bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase, lowLevelCancellation);
             // we clone the query shard context here just for rewriting otherwise we
             // might end up with incorrect state since we are using now() or script services
diff --git a/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
index 5be1aeea9be73..1c3c14ab14d38 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/LegacyReaderContext.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.search.RescoreDocIds;
@@ -37,9 +38,9 @@ public class LegacyReaderContext extends ReaderContext {
     private Engine.Searcher searcher;
     private Releasable onClose;
 
-    public LegacyReaderContext(long id, IndexShard indexShard, Engine.SearcherSupplier reader,
+    public LegacyReaderContext(long id, IndexService indexService, IndexShard indexShard, Engine.SearcherSupplier reader,
                                ShardSearchRequest shardSearchRequest, long keepAliveInMillis) {
-        super(id, indexShard, reader, keepAliveInMillis, false);
+        super(id, indexService, indexShard, reader, keepAliveInMillis, false);
         assert shardSearchRequest.readerId() == null;
         assert shardSearchRequest.keepAlive() == null;
         this.shardSearchRequest = Objects.requireNonNull(shardSearchRequest);
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
index 1caea59c7e608..43f0a8272af6f 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.search.RescoreDocIds;
@@ -45,6 +46,7 @@
  */
 public class ReaderContext implements Releasable {
     private final SearchContextId id;
+    private final IndexService indexService;
     private final IndexShard indexShard;
     protected final Engine.SearcherSupplier searcherSupplier;
     private final AtomicBoolean closed = new AtomicBoolean(false);
@@ -62,11 +64,13 @@ public class ReaderContext implements Releasable {
     private Map<String, Object> context;
 
     public ReaderContext(long id,
+                         IndexService indexService,
                          IndexShard indexShard,
                          Engine.SearcherSupplier searcherSupplier,
                          long keepAliveInMillis,
                          boolean singleSession) {
         this.id = new SearchContextId(UUIDs.base64UUID(), id);
+        this.indexService = indexService;
         this.indexShard = indexShard;
         this.searcherSupplier = searcherSupplier;
         this.singleSession = singleSession;
@@ -103,6 +107,9 @@ public SearchContextId id() {
         return id;
     }
 
+    public IndexService indexService() {
+        return indexService;
+    }
 
     public IndexShard indexShard() {
         return indexShard;
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index de96b084f3e6c..0d1cef7a9ef7e 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -138,9 +138,9 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
             ReaderContext readerWithoutScroll = new ReaderContext(
-                randomNonNegativeLong(), indexShard, engineReader, randomNonNegativeLong(), false);
+                randomNonNegativeLong(), indexService, indexShard, engineReader, randomNonNegativeLong(), false);
             DefaultSearchContext contextWithoutScroll = new DefaultSearchContext(readerWithoutScroll, shardSearchRequest, target, null,
-                indexService, indexShard, bigArrays, null, timeout, null, false);
+                bigArrays, null, timeout, null, false);
             contextWithoutScroll.from(300);
             contextWithoutScroll.close();
 
@@ -154,9 +154,9 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
             // resultWindow greater than maxResultWindow and scrollContext isn't null
             when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMillis(randomInt(1000))));
             ReaderContext readerContext = new LegacyReaderContext(
-                randomNonNegativeLong(), indexShard, engineReader, shardSearchRequest, randomNonNegativeLong());
-            DefaultSearchContext context1 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null, indexService,
-                indexShard, bigArrays, null, timeout, null, false);
+                randomNonNegativeLong(), indexService, indexShard, engineReader, shardSearchRequest, randomNonNegativeLong());
+            DefaultSearchContext context1 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
+                bigArrays, null, timeout, null, false);
             context1.from(300);
             exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false));
             assertThat(exception.getMessage(), equalTo("Batch size is too large, size must be less than or equal to: ["
@@ -186,10 +186,11 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
                 + "] index level setting."));
 
             readerContext.close();
-            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineReader, randomNonNegativeLong(), false);
+            readerContext = new ReaderContext(
+                randomNonNegativeLong(), indexService, indexShard, engineReader, randomNonNegativeLong(), false);
             // rescore is null but sliceBuilder is not null
             DefaultSearchContext context2 = new DefaultSearchContext(readerContext, shardSearchRequest, target,
-                null, indexService, indexShard, bigArrays, null, timeout, null, false);
+                null, bigArrays, null, timeout, null, false);
 
             SliceBuilder sliceBuilder = mock(SliceBuilder.class);
             int numSlices = maxSlicesPerScroll + randomIntBetween(1, 100);
@@ -206,7 +207,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
             when(shardSearchRequest.indexBoost()).thenReturn(AbstractQueryBuilder.DEFAULT_BOOST);
 
             DefaultSearchContext context3 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
-                indexService, indexShard, bigArrays, null, timeout, null, false);
+                bigArrays, null, timeout, null, false);
             ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery();
             context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false);
             assertEquals(context3.query(), context3.buildFilteredQuery(parsedQuery.query()));
@@ -216,9 +217,10 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
             readerContext.close();
-            readerContext = new ReaderContext(randomNonNegativeLong(), indexShard, engineReader, randomNonNegativeLong(), false);
-            DefaultSearchContext context4 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
-                indexService, indexShard, bigArrays, null, timeout, null, false);
+            readerContext =
+                new ReaderContext(randomNonNegativeLong(), indexService, indexShard, engineReader, randomNonNegativeLong(), false);
+            DefaultSearchContext context4 =
+                new DefaultSearchContext(readerContext, shardSearchRequest, target, null, bigArrays, null, timeout, null, false);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
             Query query1 = context4.query();
             context4.sliceBuilder(new SliceBuilder(0,2)).parsedQuery(parsedQuery).preProcess(false);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 033403fce0282..2fbc6b63325b8 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -415,7 +415,7 @@ public void testTimeout() throws IOException {
             new AliasFilter(null, Strings.EMPTY_ARRAY),
             1.0f, -1, null, null);
 
-        try (ReaderContext reader = createReaderContext(indexShard);
+        try (ReaderContext reader = createReaderContext(indexService, indexShard);
              SearchContext contextWithDefaultTimeout = service.createContext(reader, requestWithDefaultTimeout, null, randomBoolean())) {
             // the search context should inherit the default timeout
             assertThat(contextWithDefaultTimeout.timeout(), equalTo(TimeValue.timeValueSeconds(5)));
@@ -430,7 +430,7 @@ public void testTimeout() throws IOException {
             1,
             new AliasFilter(null, Strings.EMPTY_ARRAY),
             1.0f, -1, null, null);
-        try (ReaderContext reader = createReaderContext(indexShard);
+        try (ReaderContext reader = createReaderContext(indexService, indexShard);
              SearchContext context = service.createContext(reader, requestWithCustomTimeout, null, randomBoolean())) {
             // the search context should inherit the query timeout
             assertThat(context.timeout(), equalTo(TimeValue.timeValueSeconds(seconds)));
@@ -456,12 +456,12 @@ public void testMaxDocvalueFieldsSearch() throws IOException {
         }
         final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1,
             new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
-        try (ReaderContext reader = createReaderContext(indexShard);
+        try (ReaderContext reader = createReaderContext(indexService, indexShard);
              SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
             assertNotNull(context);
         }
         searchSourceBuilder.docValueField("one_field_too_much");
-        try (ReaderContext reader = createReaderContext(indexShard)) {
+        try (ReaderContext reader = createReaderContext(indexService, indexShard)) {
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
                 () -> service.createContext(reader, request, null, randomBoolean()));
             assertEquals(
@@ -493,7 +493,7 @@ public void testMaxScriptFieldsSearch() throws IOException {
         final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE, searchRequest,
             indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
 
-        try(ReaderContext reader = createReaderContext(indexShard)) {
+        try(ReaderContext reader = createReaderContext(indexService, indexShard)) {
             try (SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
                 assertNotNull(context);
             }
@@ -525,7 +525,7 @@ public void testIgnoreScriptfieldIfSizeZero() throws IOException {
         final ShardSearchRequest request = new ShardSearchRequest(OriginalIndices.NONE,
             searchRequest, indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY),
             1.0f, -1, null, null);
-        try (ReaderContext reader = createReaderContext(indexShard);
+        try (ReaderContext reader = createReaderContext(indexService, indexShard);
              SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
             assertEquals(0, context.scriptFields().fields().size());
         }
@@ -562,7 +562,8 @@ public void testMaxOpenScrollContexts() throws Exception {
 
         final ShardScrollRequestTest request = new ShardScrollRequestTest(indexShard.shardId());
         ElasticsearchException ex = expectThrows(ElasticsearchException.class,
-            () -> service.createAndPutReaderContext(request, indexShard, indexShard.acquireSearcherSupplier(), randomBoolean()));
+            () -> service.createAndPutReaderContext(
+                request, indexService, indexShard, indexShard.acquireSearcherSupplier(), randomBoolean()));
         assertEquals(
             "Trying to create too many scroll contexts. Must be less than or equal to: [" +
                 SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY) + "]. " +
@@ -575,7 +576,8 @@ public void testMaxOpenScrollContexts() throws Exception {
     public void testOpenScrollContextsConcurrently() throws Exception {
         createIndex("index");
         final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        final IndexShard indexShard = indicesService.indexServiceSafe(resolveIndex("index")).getShard(0);
+        final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
+        final IndexShard indexShard = indexService.getShard(0);
 
         final int maxScrollContexts = SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY);
         final SearchService searchService = getInstanceFromNode(SearchService.class);
@@ -590,7 +592,7 @@ public void testOpenScrollContextsConcurrently() throws Exception {
                         Engine.SearcherSupplier reader = indexShard.acquireSearcherSupplier();
                         try {
                             searchService.createAndPutReaderContext(
-                                new ShardScrollRequestTest(indexShard.shardId()), indexShard, reader, true);
+                                new ShardScrollRequestTest(indexShard.shardId()), indexService, indexShard, reader, true);
                             reader = null;
                         } catch (ElasticsearchException e) {
                             assertThat(e.getMessage(), equalTo(
@@ -889,7 +891,7 @@ public SearchType searchType() {
                 throw new NullPointerException("expected");
             }
         };
-        try (ReaderContext reader = createReaderContext(indexService.getShard(shardId.id()))) {
+        try (ReaderContext reader = createReaderContext(indexService, indexService.getShard(shardId.id()))) {
             NullPointerException e = expectThrows(NullPointerException.class,
                 () -> service.createContext(reader, request, null, randomBoolean()));
             assertEquals("expected", e.getMessage());
@@ -1047,7 +1049,7 @@ public void testLookUpSearchContext() throws Exception {
                     ShardSearchRequest request = new ShardSearchRequest(
                         OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
                         indexShard.shardId(), 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null);
-                    final ReaderContext context = searchService.createAndPutReaderContext(request, indexShard,
+                    final ReaderContext context = searchService.createAndPutReaderContext(request, indexService, indexShard,
                         indexShard.acquireSearcherSupplier(), randomBoolean());
                     assertThat(context.id().getId(), equalTo((long) (i + 1)));
                     contextIds.add(context.id());
@@ -1085,7 +1087,8 @@ public void testOpenReaderContext() {
         assertTrue(searchService.freeReaderContext(future.actionGet()));
     }
 
-    private ReaderContext createReaderContext(IndexShard shard) {
-        return new ReaderContext(randomNonNegativeLong(), shard, shard.acquireSearcherSupplier(), randomNonNegativeLong(), false);
+    private ReaderContext createReaderContext(IndexService indexService, IndexShard indexShard) {
+        return new ReaderContext(randomNonNegativeLong(), indexService, indexShard,
+            indexShard.acquireSearcherSupplier(), randomNonNegativeLong(), false);
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index 2b6c02021dba3..c1840ff26cda6 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.license.XPackLicenseState.Feature;
@@ -51,15 +52,18 @@
 import static org.mockito.Mockito.when;
 
 public class SecuritySearchOperationListenerTests extends ESSingleNodeTestCase {
+    private IndexService indexService;
     private IndexShard shard;
 
     @Before
     public void setupShard() {
-        shard = createIndex("index").getShard(0);
+        indexService = createIndex("index");
+        shard = indexService.getShard(0);
     }
 
     public void testUnlicensed() {
-        try (ReaderContext readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false)) {
+        try (ReaderContext readerContext =
+                 new ReaderContext(0L, indexService, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false)) {
             XPackLicenseState licenseState = mock(XPackLicenseState.class);
             when(licenseState.isSecurityEnabled()).thenReturn(false);
             ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
@@ -77,7 +81,8 @@ public void testUnlicensed() {
     }
 
     public void testOnNewContextSetsAuthentication() throws Exception {
-        try (ReaderContext readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false)) {
+        try (ReaderContext readerContext =
+                 new ReaderContext(0L, indexService, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false)) {
             XPackLicenseState licenseState = mock(XPackLicenseState.class);
             when(licenseState.isSecurityEnabled()).thenReturn(true);
             ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
@@ -101,11 +106,11 @@ public void testOnNewContextSetsAuthentication() throws Exception {
     public void testValidateSearchContext() throws Exception {
         final ReaderContext readerContext;
         if (randomBoolean()) {
-            readerContext = new ReaderContext(0L, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false);
+            readerContext = new ReaderContext(0L, indexService, shard, shard.acquireSearcherSupplier(), Long.MAX_VALUE, false);
         } else {
             ShardSearchRequest request = mock(ShardSearchRequest.class);
             when(request.scroll()).thenReturn(new Scroll(TimeValue.timeValueMinutes(between(1, 10))));
-            readerContext = new LegacyReaderContext(0L, shard, shard.acquireSearcherSupplier(), request, Long.MAX_VALUE);
+            readerContext = new LegacyReaderContext(0L, indexService, shard, shard.acquireSearcherSupplier(), request, Long.MAX_VALUE);
         }
         try {
             readerContext.putInContext(AuthenticationField.AUTHENTICATION_KEY,

From f68dc64ae15c60f4bcb9bf7d3f8a80649b2318fb Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 26 Jun 2020 15:18:08 +0200
Subject: [PATCH 26/47] Update docs/reference/search/search_context.asciidoc

Co-authored-by: Adrien Grand <jpountz@gmail.com>
---
 docs/reference/search/search_context.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/search/search_context.asciidoc b/docs/reference/search/search_context.asciidoc
index e9566efe42b21..a9bafba3ff755 100644
--- a/docs/reference/search/search_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -78,7 +78,7 @@ GET /_nodes/stats/indices/search
 ===== Close search context API
 
 Search contexts are automatically closed when the `keep_alive` has
-been elapsed. However keeping search contexts as a cost, as discussed in the
+been elapsed. However keeping search contexts has a cost, as discussed in the
 <<search-context-keep-alive,previous section>> so search context should be
 explicitly closed as soon as they are no longer used in search requests.
 

From 699c4d455497a4336cf7b13fcf13d518185103a2 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 26 Jun 2020 15:18:19 +0200
Subject: [PATCH 27/47] Update docs/reference/search/search_context.asciidoc

Co-authored-by: Adrien Grand <jpountz@gmail.com>
---
 docs/reference/search/search_context.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/search/search_context.asciidoc b/docs/reference/search/search_context.asciidoc
index a9bafba3ff755..c53674f70e595 100644
--- a/docs/reference/search/search_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -79,7 +79,7 @@ GET /_nodes/stats/indices/search
 
 Search contexts are automatically closed when the `keep_alive` has
 been elapsed. However keeping search contexts has a cost, as discussed in the
-<<search-context-keep-alive,previous section>> so search context should be
+<<search-context-keep-alive,previous section>> so search contexts should be
 explicitly closed as soon as they are no longer used in search requests.
 
 [source,console]

From 0bc08e15223251b4ddb2d099f3a17222bf991a9f Mon Sep 17 00:00:00 2001
From: jimczi <jim.ferenczi@elastic.co>
Date: Fri, 26 Jun 2020 16:25:12 +0200
Subject: [PATCH 28/47] address review

---
 docs/reference/search/search_context.asciidoc |  4 ++
 .../action/search/TransportSearchAction.java  |  4 --
 .../elasticsearch/search/SearchService.java   | 72 +++++++------------
 .../search/builder/SearchSourceBuilder.java   |  3 +-
 4 files changed, 31 insertions(+), 52 deletions(-)

diff --git a/docs/reference/search/search_context.asciidoc b/docs/reference/search/search_context.asciidoc
index c53674f70e595..8867d62735873 100644
--- a/docs/reference/search/search_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -20,6 +20,8 @@ POST /twitter/_search_context?keep_alive=1m
 The result from the above request includes a `id`, which should
 be passed to the `id` of the `search_context` parameter of a search request.
 
+[source,console]
+--------------------------------------------------
 POST /_search <1>
 {
     "size": 100,
@@ -33,6 +35,8 @@ POST /_search <1>
 	    "keep_alive": "1m"  <3>
     }
 }
+--------------------------------------------------
+// TEST[catch:missing]
 
 <1> A search request with `search_context` must not specify `index`, `routing`,
 and `preferences` as these parameters are copied from the `search_context`.
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index c5082d107f6e5..ba900736c2f80 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -495,10 +495,6 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
                                SearchAsyncActionProvider searchAsyncActionProvider) {
 
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
-        if (searchRequest.allowPartialSearchResults() == null) {
-            // No user preference defined in search request - apply cluster service default
-            searchRequest.allowPartialSearchResults(searchService.defaultAllowPartialSearchResults());
-        }
 
         // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
         // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index d49c08b1bd247..864ea7334e9a5 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -1,3 +1,4 @@
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -326,11 +327,7 @@ public void executeDfsPhase(ShardSearchRequest request, boolean keepStatesInCont
             @Override
             public void onResponse(ShardSearchRequest rewritten) {
                 // fork the execution in the search thread pool
-                try {
-                    runAsync(shard, () -> executeDfsPhase(request, task, keepStatesInContext), listener);
-                } catch (Exception exc) {
-                    listener.onFailure(exc);
-                }
+                runAsync(getExecutor(shard), () -> executeDfsPhase(request, task, keepStatesInContext), listener);
             }
 
             @Override
@@ -409,12 +406,6 @@ public void onResponse(ShardSearchRequest orig) {
                             return;
                         }
                     }
-                    // fork the execution in the search thread pool
-                    runAsync(shard, () -> {
-                        try (releasable) {
-                            return executeQueryPhase(orig, task, readerContext);
-                        }
-                    }, listener);
                 } catch (Exception exc) {
                     try {
                         listener.onFailure(exc);
@@ -423,6 +414,12 @@ public void onResponse(ShardSearchRequest orig) {
                         removeReaderContext(readerContext.id().getId());
                     }
                 }
+                // fork the execution in the search thread pool
+                runAsync(getExecutor(shard), () -> {
+                    try (releasable) {
+                        return executeQueryPhase(orig, task, readerContext);
+                    }
+                }, ActionListener.runAfter(listener, releasable::close));
             }
 
             @Override
@@ -440,22 +437,8 @@ private IndexShard getShard(ShardSearchRequest request) {
         }
     }
 
-    private <T> void runAsync(IndexShard shard, CheckedSupplier<T, Exception> command, ActionListener<T> listener) {
-        Executor executor = getExecutor(shard);
-        executor.execute(() -> {
-            T result;
-            try {
-                result = command.get();
-            } catch (Exception exc) {
-                listener.onFailure(exc);
-                return;
-            }
-            listener.onResponse(result);
-        });
-    }
-
-    private <T> void runAsync(SearchContextId id, CheckedSupplier<T, IOException> executable, ActionListener<T> listener) {
-        getExecutor(id).execute(ActionRunnable.supply(listener, executable::get));
+    private <T> void runAsync(Executor executor, CheckedSupplier<T, Exception> executable, ActionListener<T> listener) {
+        executor.execute(ActionRunnable.supply(listener, executable::get));
     }
 
     private SearchPhaseResult executeQueryPhase(ShardSearchRequest request,
@@ -501,7 +484,7 @@ private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchCon
     public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
-        runAsync(request.contextId(), () -> {
+        runAsync(getExecutor(request.contextId()), () -> {
             final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
             try (Releasable ignored = readerContext.markAsUsed();
@@ -530,7 +513,7 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
     }
 
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
-        runAsync(request.contextId(), () -> {
+        runAsync(getExecutor(request.contextId()), () -> {
             final ReaderContext readerContext = findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.shardSearchRequest());
             readerContext.setAggregatedDfs(request.dfs());
@@ -569,7 +552,7 @@ private Executor getExecutor(IndexShard indexShard) {
 
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
-        runAsync(request.contextId(), () -> {
+        runAsync(getExecutor(request.contextId()), () -> {
             final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
             try (Releasable ignored = readerContext.markAsUsed();
@@ -597,7 +580,7 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
     }
 
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
-        runAsync(request.contextId(), () -> {
+        runAsync(getExecutor(request.contextId()), () -> {
             final ReaderContext readerContext = findReaderContext(request.contextId());
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest());
             try (Releasable ignored = readerContext.markAsUsed();
@@ -1143,36 +1126,33 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Set<String
 
     /**
      * This method does a very quick rewrite of the query and returns true if the query can potentially match any documents.
+     * It uses a light weight searcher that can only access the statistics of the fields in order to avoid opening a
+     * full reader on frozen indices.
      * This method can have false positives while if it returns <code>false</code> the query won't match any documents on the current
      * shard.
      */
     public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException {
         assert request.searchType() == SearchType.QUERY_THEN_FETCH : "unexpected search type: " + request.searchType();
-        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-        final boolean hasRefreshPending;
+        final ReaderContext readerContext = request.readerId() != null ? getReaderContext(request.readerId()) :  null;
+        final Releasable markAsUsed = readerContext != null ? readerContext.markAsUsed() : null;
+        final IndexService indexService;
         final Engine.Searcher canMatchSearcher;
-        if (request.readerId() != null) {
-            final ReaderContext readerContext = findReaderContext(request.readerId());
+        final boolean hasRefreshPending;
+        if (readerContext != null) {
             readerContext.indexShard().getSearchOperationListener().validateSearchContext(readerContext, request);
             checkKeepAliveLimit(request.keepAlive().millis());
             readerContext.keepAlive(request.keepAlive().millis());
-            Releasable releasable = readerContext.markAsUsed();
-            try {
-                final Engine.Searcher searcher = readerContext.acquireSearcher("can_match");
-                canMatchSearcher = new Engine.Searcher(searcher.source(), searcher.getDirectoryReader(),
-                    searcher.getSimilarity(), searcher.getQueryCache(), searcher.getQueryCachingPolicy(),
-                    Releasables.wrap(searcher, releasable));
-                releasable = null;
-            } finally {
-                Releasables.close(releasable);
-            }
+            indexService = readerContext.indexService();
+            canMatchSearcher = readerContext.acquireSearcher("can_match");
             hasRefreshPending = false;
         } else {
+            indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
             IndexShard indexShard = indexService.getShard(request.shardId().getId());
             hasRefreshPending = indexShard.hasRefreshPending();
             canMatchSearcher = indexShard.acquireSearcher("can_match");
         }
-        try (Releasable ignored = canMatchSearcher) {
+
+        try (markAsUsed; Releasable ignored = canMatchSearcher) {
             final boolean aliasFilterCanMatch = request.getAliasFilter()
                 .getQueryBuilder() instanceof MatchNoneQueryBuilder == false;
             QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher,
diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index d03100d8f1f0a..09f1b5d2fa54e 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -1137,8 +1137,7 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th
                 } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) {
                     collapse = CollapseBuilder.fromXContent(parser);
                 } else if (SEARCH_CONTEXT.match(currentFieldName, parser.getDeprecationHandler())) {
-                    searchContextBuilder = SearchContextBuilder.
-                        fromXContent(parser);
+                    searchContextBuilder = SearchContextBuilder.fromXContent(parser);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());

From 7b765d24a8ff254bca3ed8b16996128ee3d23d85 Mon Sep 17 00:00:00 2001
From: jimczi <jim.ferenczi@elastic.co>
Date: Fri, 26 Jun 2020 19:05:29 +0200
Subject: [PATCH 29/47] ensure that we remove the reader context on failures

---
 .../elasticsearch/search/SearchService.java   | 107 +++++++++---------
 1 file changed, 56 insertions(+), 51 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 864ea7334e9a5..49cdb431e733f 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -347,7 +347,7 @@ private DfsSearchResult executeDfsPhase(ShardSearchRequest request,
             return context.dfsResult();
         } catch (Exception e) {
             logger.trace("Dfs phase failed", e);
-            processFailure(readerContext, e);
+            processFailure(request, readerContext, e);
             throw e;
         }
     }
@@ -374,52 +374,57 @@ public void executeQueryPhase(ShardSearchRequest request, boolean keepStatesInCo
             @Override
             public void onResponse(ShardSearchRequest orig) {
                 final ReaderContext readerContext = createOrGetReaderContext(orig, keepStatesInContext);
-                final Releasable releasable = readerContext.markAsUsed();
-                try {
-                    if (orig.canReturnNullResponseIfMatchNoDocs()) {
-                        assert orig.scroll() == null;
-                        // we clone the shard request and perform a quick rewrite using a lightweight
-                        // searcher since we are outside of the search thread pool.
-                        // If the request rewrites to "match none" we can shortcut the query phase
-                        // entirely. Otherwise we fork the execution in the search thread pool.
-                        ShardSearchRequest canMatchRequest = new ShardSearchRequest(orig);
-                        try (Engine.Searcher searcher = readerContext.acquireSearcher("can_match")) {
-                            QueryShardContext context = readerContext.indexService().newQueryShardContext(canMatchRequest.shardId().id(),
-                                searcher, canMatchRequest::nowInMillis, canMatchRequest.getClusterAlias());
-                            Rewriteable.rewrite(canMatchRequest.getRewriteable(), context, true);
+                final Releasable markAsUsed = readerContext.markAsUsed();
+                if (orig.canReturnNullResponseIfMatchNoDocs()) {
+                    assert orig.scroll() == null;
+                    // we clone the shard request and perform a quick rewrite using a lightweight
+                    // searcher since we are outside of the search thread pool.
+                    // If the request rewrites to "match none" we can shortcut the query phase
+                    // entirely. Otherwise we fork the execution in the search thread pool.
+                    ShardSearchRequest canMatchRequest = new ShardSearchRequest(orig);
+                    try (Engine.Searcher searcher = readerContext.acquireSearcher("can_match")) {
+                        QueryShardContext context = readerContext.indexService().newQueryShardContext(canMatchRequest.shardId().id(),
+                            searcher, canMatchRequest::nowInMillis, canMatchRequest.getClusterAlias());
+                        Rewriteable.rewrite(canMatchRequest.getRewriteable(), context, true);
+                    } catch (IOException exc) {
+                        try (markAsUsed) {
+                            listener.onFailure(exc);
+                        } finally {
+                            processFailure(request, readerContext, exc);
                         }
-                        if (canRewriteToMatchNone(canMatchRequest.source())
-                                && canMatchRequest.source().query() instanceof MatchNoneQueryBuilder) {
-                            if (orig.readerId() != null) {
-                                try (releasable) {
-                                    listener.onResponse(QuerySearchResult.nullInstance());
-                                }
-                            } else {
+                    }
+
+                    if (canRewriteToMatchNone(canMatchRequest.source())
+                        && canMatchRequest.source().query() instanceof MatchNoneQueryBuilder) {
+                        try (markAsUsed) {
+                            if (orig.readerId() == null) {
                                 try {
                                     listener.onResponse(QuerySearchResult.nullInstance());
                                 } finally {
                                     // close and remove the ephemeral reader context
-                                    Releasables.close(releasable, readerContext);
+                                    Releasables.close(readerContext);
                                     removeReaderContext(readerContext.id().getId());
                                 }
+                            } else {
+                                listener.onResponse(QuerySearchResult.nullInstance());
                             }
-                            return;
                         }
-                    }
-                } catch (Exception exc) {
-                    try {
-                        listener.onFailure(exc);
-                    } finally {
-                        Releasables.close(releasable, readerContext);
-                        removeReaderContext(readerContext.id().getId());
+                        return;
                     }
                 }
+
                 // fork the execution in the search thread pool
                 runAsync(getExecutor(shard), () -> {
-                    try (releasable) {
+                    try (markAsUsed) {
                         return executeQueryPhase(orig, task, readerContext);
                     }
-                }, ActionListener.runAfter(listener, releasable::close));
+                }, ActionListener.wrap(listener::onResponse, exc -> {
+                    try (markAsUsed) {
+                        listener.onFailure(exc);
+                    } finally {
+                        processFailure(request, readerContext, exc);
+                    }
+                }));
             }
 
             @Override
@@ -464,7 +469,7 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request,
                     (Exception) e.getCause() : new ElasticsearchException(e.getCause());
             }
             logger.trace("Query phase failed", e);
-            processFailure(readerContext, e);
+            processFailure(request, readerContext, e);
             throw e;
         }
     }
@@ -484,8 +489,8 @@ private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchCon
     public void executeQueryPhase(InternalScrollSearchRequest request,
                                   SearchShardTask task,
                                   ActionListener<ScrollQuerySearchResult> listener) {
-        runAsync(getExecutor(request.contextId()), () -> {
-            final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
+        final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
+        runAsync(getExecutor(readerContext.indexShard()), () -> {
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
@@ -506,15 +511,15 @@ public void executeQueryPhase(InternalScrollSearchRequest request,
                 return new ScrollQuerySearchResult(searchContext.queryResult(), searchContext.shardTarget());
             } catch (Exception e) {
                 logger.trace("Query phase failed", e);
-                processFailure(readerContext, e);
+                processFailure(shardSearchRequest, readerContext, e);
                 throw e;
             }
         }, listener);
     }
 
     public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener<QuerySearchResult> listener) {
-        runAsync(getExecutor(request.contextId()), () -> {
-            final ReaderContext readerContext = findReaderContext(request.contextId());
+        final ReaderContext readerContext = findReaderContext(request.contextId());
+        runAsync(getExecutor(readerContext.indexShard()), () -> {
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.shardSearchRequest());
             readerContext.setAggregatedDfs(request.dfs());
             try (Releasable ignored = readerContext.markAsUsed();
@@ -535,16 +540,12 @@ public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task,
             } catch (Exception e) {
                 assert TransportActions.isShardNotAvailableException(e) == false : new AssertionError(e);
                 logger.trace("Query phase failed", e);
-                processFailure(readerContext, e);
+                processFailure(shardSearchRequest, readerContext, e);
                 throw e;
             }
         }, listener);
     }
 
-    final Executor getExecutor(SearchContextId id) {
-        return getExecutor(findReaderContext(id).indexShard());
-    }
-
     private Executor getExecutor(IndexShard indexShard) {
         assert indexShard != null;
         return threadPool.executor(indexShard.indexSettings().isSearchThrottled() ? Names.SEARCH_THROTTLED : Names.SEARCH);
@@ -552,8 +553,8 @@ private Executor getExecutor(IndexShard indexShard) {
 
     public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTask task,
                                   ActionListener<ScrollQueryFetchSearchResult> listener) {
-        runAsync(getExecutor(request.contextId()), () -> {
-            final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
+        final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId());
+        runAsync(getExecutor(readerContext.indexShard()), () -> {
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null);
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false);
@@ -573,15 +574,15 @@ public void executeFetchPhase(InternalScrollSearchRequest request, SearchShardTa
             } catch (Exception e) {
                 assert TransportActions.isShardNotAvailableException(e) == false : new AssertionError(e);
                 logger.trace("Fetch phase failed", e);
-                processFailure(readerContext, e);
+                processFailure(shardSearchRequest, readerContext, e);
                 throw e;
             }
         }, listener);
     }
 
     public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, ActionListener<FetchSearchResult> listener) {
-        runAsync(getExecutor(request.contextId()), () -> {
-            final ReaderContext readerContext = findReaderContext(request.contextId());
+        final ReaderContext readerContext = findReaderContext(request.contextId());
+        runAsync(getExecutor(readerContext.indexShard()), () -> {
             final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest());
             try (Releasable ignored = readerContext.markAsUsed();
                  SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false)) {
@@ -604,7 +605,7 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
             } catch (Exception e) {
                 assert TransportActions.isShardNotAvailableException(e) == false : new AssertionError(e);
                 logger.trace("Fetch phase failed", e);
-                processFailure(readerContext, e);
+                processFailure(shardSearchRequest, readerContext, e);
                 throw e;
             }
         }, listener);
@@ -846,8 +847,11 @@ private void checkKeepAliveLimit(long keepAlive) {
         }
     }
 
-    private void processFailure(ReaderContext context, Exception e) {
-        freeReaderContext(context.id());
+    private void processFailure(ShardSearchRequest request, ReaderContext context, Exception e) {
+        if (request.readerId() == null || request.scroll() != null) {
+            // we release the reader on failure if the request is a normal search or a scroll
+            freeReaderContext(context.id());
+        }
         try {
             if (Lucene.isCorruptionException(e)) {
                 context.indexShard().failShard("search execution corruption failure", e);
@@ -1169,6 +1173,7 @@ public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException
                 canMatch = aliasFilterCanMatch;
             }
             return new CanMatchResponse(canMatch || hasRefreshPending, minMax);
+
         }
     }
 

From e61bd20c328276090eb84dbdf42c62d64c2a5078 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sun, 28 Jun 2020 22:45:44 -0400
Subject: [PATCH 30/47] remove reader from the active list during put

---
 .../org/elasticsearch/search/SearchService.java  | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 49cdb431e733f..3b49cbb9085fc 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -53,6 +53,7 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.Engine;
@@ -294,9 +295,14 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem
     protected void putReaderContext(ReaderContext context) {
         final ReaderContext previous = activeReaders.put(context.id().getId(), context);
         assert previous == null;
-        // ensure that if we race against afterIndexRemoved, we free the context here.
+        // ensure that if we race against afterIndexRemoved, we remove the context from the active list.
         // this is important to ensure store can be cleaned up, in particular if the search is a scroll with a long timeout.
-        indicesService.indexServiceSafe(context.indexShard().shardId().getIndex());
+        final Index index = context.indexShard().shardId().getIndex();
+        if (indicesService.hasIndex(index) == false) {
+            final ReaderContext removed = removeReaderContext(context.id().getId());
+            assert removed == context;
+            throw new IndexNotFoundException(index);
+        }
     }
 
     protected ReaderContext removeReaderContext(long id) {
@@ -694,9 +700,6 @@ final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexS
             readerContext = null;
             return finalReaderContext;
         } finally {
-            if (readerContext != null) {
-                removeReaderContext(readerContext.id().getId());
-            }
             Releasables.close(reader, readerContext, decreaseScrollContexts);
         }
     }
@@ -725,9 +728,6 @@ public void openReaderContext(ShardId shardId, TimeValue keepAlive, ActionListen
                 readerContext = null;
                 listener.onResponse(finalReaderContext.id());
             } catch (Exception exc) {
-                if (readerContext != null) {
-                    removeReaderContext(readerContext.id().getId());
-                }
                 Releasables.closeWhileHandlingException(searcherSupplier, readerContext);
                 listener.onFailure(exc);
             }

From 44aca91af592aad6ddee482019091a2a8b8ee90a Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sun, 28 Jun 2020 23:05:39 -0400
Subject: [PATCH 31/47] Ensure open before acquire searcher

---
 .../elasticsearch/index/engine/Engine.java    | 43 +++++++++++--------
 .../search/DefaultSearchContextTests.java     |  2 +-
 .../index/engine/FrozenEngine.java            |  2 +-
 3 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index c6b8722375631..9824af9e7e6b7 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -610,7 +610,6 @@ public SearcherSupplier acquireSearcherSupplier(Function<Searcher, Searcher> wra
         try {
             ReferenceManager<ElasticsearchDirectoryReader> referenceManager = getReferenceManager(scope);
             ElasticsearchDirectoryReader acquire = referenceManager.acquire();
-            AtomicBoolean released = new AtomicBoolean(false);
             SearcherSupplier reader = new SearcherSupplier(wrapper) {
                 @Override
                 public Searcher acquireSearcherInternal(String source) {
@@ -620,23 +619,16 @@ public Searcher acquireSearcherInternal(String source) {
                 }
 
                 @Override
-                public void close() {
-                    if (released.compareAndSet(false, true)) {
-                        try {
-                            referenceManager.release(acquire);
-                        } catch (IOException e) {
-                            throw new UncheckedIOException("failed to close", e);
-                        } catch (AlreadyClosedException e) {
-                            // This means there's a bug somewhere: don't suppress it
-                            throw new AssertionError(e);
-                        } finally {
-                            store.decRef();
-                        }
-                    } else {
-                        /* In general, readers should never be released twice or this would break reference counting. There is one rare case
-                         * when it might happen though: when the request and the Reaper thread would both try to release it in a very short
-                         * amount of time, this is why we only log a warning instead of throwing an exception. */
-                        logger.warn("Reader was released twice", new IllegalStateException("Double release"));
+                protected void doClose() {
+                    try {
+                        referenceManager.release(acquire);
+                    } catch (IOException e) {
+                        throw new UncheckedIOException("failed to close", e);
+                    } catch (AlreadyClosedException e) {
+                        // This means there's a bug somewhere: don't suppress it
+                        throw new AssertionError(e);
+                    } finally {
+                        store.decRef();
                     }
                 }
             };
@@ -1176,16 +1168,31 @@ default void onFailedEngine(String reason, @Nullable Exception e) {
 
     public abstract static class SearcherSupplier implements Releasable {
         private final Function<Searcher, Searcher> wrapper;
+        private final AtomicBoolean released = new AtomicBoolean(false);
 
         public SearcherSupplier(Function<Searcher, Searcher> wrapper) {
             this.wrapper = wrapper;
         }
 
         public final Searcher acquireSearcher(String source) {
+            if (released.get()) {
+                throw new AlreadyClosedException("SearcherSupplier was closed");
+            }
             final Searcher searcher = acquireSearcherInternal(source);
             return "can_match".equals(source) ? searcher : wrapper.apply(searcher);
         }
 
+        @Override
+        public final void close() {
+            if (released.compareAndSet(false, true)) {
+                doClose();
+            } else {
+                assert false : "SearchSupplier was released twice";
+            }
+        }
+
+        protected abstract void doClose();
+
         protected abstract Searcher acquireSearcherInternal(String source);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 0d1cef7a9ef7e..4992fcd6f69f6 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -121,7 +121,7 @@ public void testPreProcess() throws Exception {
 
             final Engine.SearcherSupplier engineReader = new Engine.SearcherSupplier(Function.identity()) {
                 @Override
-                public void close() {}
+                protected void doClose() {}
 
                 @Override
                 protected Engine.Searcher acquireSearcherInternal(String source) {
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index e99767b896db5..7db7be30a4a61 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -190,7 +190,7 @@ public Searcher acquireSearcherInternal(String source) {
             }
 
             @Override
-            public void close() {
+            public void doClose() {
                 store.decRef();
             }
         };

From e36509371e0900523c0597217d1756bb954c6c7f Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sun, 28 Jun 2020 23:19:39 -0400
Subject: [PATCH 32/47] check wrapper once in search supplier

---
 .../main/java/org/elasticsearch/index/engine/Engine.java    | 6 +++++-
 .../main/java/org/elasticsearch/index/shard/IndexShard.java | 3 +--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index 9824af9e7e6b7..4562167a6fa5c 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -651,9 +651,13 @@ public final Searcher acquireSearcher(String source) throws EngineException {
     }
 
     public Searcher acquireSearcher(String source, SearcherScope scope) throws EngineException {
+        return acquireSearcher(source, scope, Function.identity());
+    }
+
+    public Searcher acquireSearcher(String source, SearcherScope scope, Function<Searcher, Searcher> wrapper) throws EngineException {
         SearcherSupplier releasable = null;
         try {
-            SearcherSupplier reader = releasable = acquireSearcherSupplier(Function.identity(), scope);
+            SearcherSupplier reader = releasable = acquireSearcherSupplier(wrapper, scope);
             Searcher searcher = reader.acquireSearcher(source);
             releasable = null;
             return new Searcher(source, searcher.getDirectoryReader(), searcher.getSimilarity(),
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index bf3a0f31a379d..9d5a7592db39d 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1231,8 +1231,7 @@ private Engine.Searcher acquireSearcher(String source, Engine.SearcherScope scop
         readAllowed();
         markSearcherAccessed();
         final Engine engine = getEngine();
-        final Engine.Searcher searcher = engine.acquireSearcher(source, scope);
-        return "can_match".equals(source) ? searcher : wrapSearcher(searcher);
+        return engine.acquireSearcher(source, scope, this::wrapSearcher);
     }
 
     private Engine.Searcher wrapSearcher(Engine.Searcher searcher) {

From 969ddf4400660d7f0e6ef1199f5d771c1b752241 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 29 Jun 2020 08:32:56 -0400
Subject: [PATCH 33/47] fix test

---
 .../search/DefaultSearchContextTests.java       | 17 ++++++++++-------
 .../search/SearchServiceTests.java              |  5 +----
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 4992fcd6f69f6..91ef4c996ffe8 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -63,6 +63,7 @@
 import java.io.IOException;
 import java.util.UUID;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.anyObject;
@@ -119,9 +120,11 @@ public void testPreProcess() throws Exception {
         try (Directory dir = newDirectory();
              RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
 
-            final Engine.SearcherSupplier engineReader = new Engine.SearcherSupplier(Function.identity()) {
+
+            final Supplier<Engine.SearcherSupplier> searcherSupplier = () -> new Engine.SearcherSupplier(Function.identity()) {
                 @Override
-                protected void doClose() {}
+                protected void doClose() {
+                }
 
                 @Override
                 protected Engine.Searcher acquireSearcherInternal(String source) {
@@ -138,7 +141,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
             SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE);
 
             ReaderContext readerWithoutScroll = new ReaderContext(
-                randomNonNegativeLong(), indexService, indexShard, engineReader, randomNonNegativeLong(), false);
+                randomNonNegativeLong(), indexService, indexShard, searcherSupplier.get(), randomNonNegativeLong(), false);
             DefaultSearchContext contextWithoutScroll = new DefaultSearchContext(readerWithoutScroll, shardSearchRequest, target, null,
                 bigArrays, null, timeout, null, false);
             contextWithoutScroll.from(300);
@@ -154,7 +157,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
             // resultWindow greater than maxResultWindow and scrollContext isn't null
             when(shardSearchRequest.scroll()).thenReturn(new Scroll(TimeValue.timeValueMillis(randomInt(1000))));
             ReaderContext readerContext = new LegacyReaderContext(
-                randomNonNegativeLong(), indexService, indexShard, engineReader, shardSearchRequest, randomNonNegativeLong());
+                randomNonNegativeLong(), indexService, indexShard, searcherSupplier.get(), shardSearchRequest, randomNonNegativeLong());
             DefaultSearchContext context1 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null,
                 bigArrays, null, timeout, null, false);
             context1.from(300);
@@ -187,7 +190,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
 
             readerContext.close();
             readerContext = new ReaderContext(
-                randomNonNegativeLong(), indexService, indexShard, engineReader, randomNonNegativeLong(), false);
+                randomNonNegativeLong(), indexService, indexShard, searcherSupplier.get(), randomNonNegativeLong(), false);
             // rescore is null but sliceBuilder is not null
             DefaultSearchContext context2 = new DefaultSearchContext(readerContext, shardSearchRequest, target,
                 null, bigArrays, null, timeout, null, false);
@@ -217,8 +220,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
             when(shardSearchRequest.indexRoutings()).thenReturn(new String[0]);
 
             readerContext.close();
-            readerContext =
-                new ReaderContext(randomNonNegativeLong(), indexService, indexShard, engineReader, randomNonNegativeLong(), false);
+            readerContext = new ReaderContext(randomNonNegativeLong(), indexService, indexShard,
+                searcherSupplier.get(), randomNonNegativeLong(), false);
             DefaultSearchContext context4 =
                 new DefaultSearchContext(readerContext, shardSearchRequest, target, null, bigArrays, null, timeout, null, false);
             context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 2fbc6b63325b8..61bdb6e5da808 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -589,19 +589,16 @@ public void testOpenScrollContextsConcurrently() throws Exception {
                 try {
                     latch.await();
                     for (; ; ) {
-                        Engine.SearcherSupplier reader = indexShard.acquireSearcherSupplier();
+                        final Engine.SearcherSupplier reader = indexShard.acquireSearcherSupplier();
                         try {
                             searchService.createAndPutReaderContext(
                                 new ShardScrollRequestTest(indexShard.shardId()), indexService, indexShard, reader, true);
-                            reader = null;
                         } catch (ElasticsearchException e) {
                             assertThat(e.getMessage(), equalTo(
                                 "Trying to create too many scroll contexts. Must be less than or equal to: " +
                                     "[" + maxScrollContexts + "]. " +
                                     "This limit can be set by changing the [search.max_open_scroll_context] setting."));
                             return;
-                        } finally {
-                            IOUtils.close(reader);
                         }
                     }
                 } catch (Exception e) {

From ae7f1a43665cee3cd1e37c6989993c2d7ea6026e Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 29 Jun 2020 09:54:45 -0400
Subject: [PATCH 34/47] add doc for can match

---
 .../java/org/elasticsearch/search/SearchService.java   | 10 ++++------
 .../org/elasticsearch/search/SearchServiceTests.java   |  1 -
 .../org/elasticsearch/index/engine/FrozenEngine.java   |  2 +-
 3 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 3b49cbb9085fc..54b3752a3825a 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -1129,11 +1129,9 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Set<String
     }
 
     /**
-     * This method does a very quick rewrite of the query and returns true if the query can potentially match any documents.
-     * It uses a light weight searcher that can only access the statistics of the fields in order to avoid opening a
-     * full reader on frozen indices.
-     * This method can have false positives while if it returns <code>false</code> the query won't match any documents on the current
-     * shard.
+     * This method uses a lightweight searcher without wrapping (i.e., not open a full reader on frozen indices) to rewrite the query
+     * to check if the query can match any documents. This method can have false positives while if it returns {@code false} the query
+     * won't match any documents on the current shard.
      */
     public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException {
         assert request.searchType() == SearchType.QUERY_THEN_FETCH : "unexpected search type: " + request.searchType();
@@ -1156,7 +1154,7 @@ public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException
             canMatchSearcher = indexShard.acquireSearcher("can_match");
         }
 
-        try (markAsUsed; Releasable ignored = canMatchSearcher) {
+        try (markAsUsed; canMatchSearcher) {
             final boolean aliasFilterCanMatch = request.getAliasFilter()
                 .getQueryBuilder() instanceof MatchNoneQueryBuilder == false;
             QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher,
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 61bdb6e5da808..4017c9377e969 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -44,7 +44,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexNotFoundException;
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index 7db7be30a4a61..be4b99d6e9cff 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -190,7 +190,7 @@ public Searcher acquireSearcherInternal(String source) {
             }
 
             @Override
-            public void doClose() {
+            protected void doClose() {
                 store.decRef();
             }
         };

From b3fe07400887d8b2bf2d598d721ece0b010e94a2 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 29 Jun 2020 10:44:47 -0400
Subject: [PATCH 35/47] can_match constant

---
 .../main/java/org/elasticsearch/index/engine/Engine.java    | 3 ++-
 .../main/java/org/elasticsearch/search/SearchService.java   | 6 +++---
 .../java/org/elasticsearch/index/engine/FrozenEngine.java   | 4 ++--
 .../org/elasticsearch/index/engine/FrozenEngineTests.java   | 4 ++--
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index 4562167a6fa5c..b79b713ccef53 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -108,6 +108,7 @@ public abstract class Engine implements Closeable {
     public static final String FORCE_MERGE_UUID_KEY = "force_merge_uuid";
     public static final String MIN_RETAINED_SEQNO = "min_retained_seq_no";
     public static final String MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID = "max_unsafe_auto_id_timestamp";
+    public static final String CAN_MATCH_SEARCH_SOURCE = "can_match"; // TODO: Make source of search enum?
 
     protected final ShardId shardId;
     protected final String allocationId;
@@ -1183,7 +1184,7 @@ public final Searcher acquireSearcher(String source) {
                 throw new AlreadyClosedException("SearcherSupplier was closed");
             }
             final Searcher searcher = acquireSearcherInternal(source);
-            return "can_match".equals(source) ? searcher : wrapper.apply(searcher);
+            return CAN_MATCH_SEARCH_SOURCE.equals(source) ? searcher : wrapper.apply(searcher);
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 54b3752a3825a..10d4a0674d9bf 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -388,7 +388,7 @@ public void onResponse(ShardSearchRequest orig) {
                     // If the request rewrites to "match none" we can shortcut the query phase
                     // entirely. Otherwise we fork the execution in the search thread pool.
                     ShardSearchRequest canMatchRequest = new ShardSearchRequest(orig);
-                    try (Engine.Searcher searcher = readerContext.acquireSearcher("can_match")) {
+                    try (Engine.Searcher searcher = readerContext.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) {
                         QueryShardContext context = readerContext.indexService().newQueryShardContext(canMatchRequest.shardId().id(),
                             searcher, canMatchRequest::nowInMillis, canMatchRequest.getClusterAlias());
                         Rewriteable.rewrite(canMatchRequest.getRewriteable(), context, true);
@@ -1145,13 +1145,13 @@ public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException
             checkKeepAliveLimit(request.keepAlive().millis());
             readerContext.keepAlive(request.keepAlive().millis());
             indexService = readerContext.indexService();
-            canMatchSearcher = readerContext.acquireSearcher("can_match");
+            canMatchSearcher = readerContext.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE);
             hasRefreshPending = false;
         } else {
             indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
             IndexShard indexShard = indexService.getShard(request.shardId().getId());
             hasRefreshPending = indexShard.hasRefreshPending();
-            canMatchSearcher = indexShard.acquireSearcher("can_match");
+            canMatchSearcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE);
         }
 
         try (markAsUsed; canMatchSearcher) {
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index be4b99d6e9cff..3ebe32c933c8b 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -211,7 +211,7 @@ private Engine.Searcher openSearcher(String source, SearcherScope scope) throws
             case "segments":
             case "segments_stats":
             case "completion_stats":
-            case "can_match": // special case for can_match phase - we use the cached point values reader
+            case CAN_MATCH_SEARCH_SOURCE: // special case for can_match phase - we use the cached point values reader
                 maybeOpenReader = false;
                 break;
             default:
@@ -219,7 +219,7 @@ private Engine.Searcher openSearcher(String source, SearcherScope scope) throws
         }
         ElasticsearchDirectoryReader reader = maybeOpenReader ? getOrOpenReader() : getReader();
         if (reader == null) {
-            if ("can_match".equals(source)) {
+            if (CAN_MATCH_SEARCH_SOURCE.equals(source)) {
                 canMatchReader.incRef();
                 return new Searcher(source, canMatchReader, engineConfig.getSimilarity(), engineConfig.getQueryCache(),
                     engineConfig.getQueryCachingPolicy(), canMatchReader::decRef);
diff --git a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java
index 86a02d4c17a46..8cc3df534a294 100644
--- a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java
+++ b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java
@@ -319,7 +319,7 @@ public void testCanMatch() throws IOException {
                 try (FrozenEngine frozenEngine = new FrozenEngine(engine.engineConfig, true)) {
                     DirectoryReader dirReader;
                     try (Engine.SearcherSupplier reader = frozenEngine.acquireSearcherSupplier(Function.identity())) {
-                        try (Engine.Searcher searcher = reader.acquireSearcher("can_match")) {
+                        try (Engine.Searcher searcher = reader.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) {
                             dirReader = searcher.getDirectoryReader();
                             assertNotNull(ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(searcher.getDirectoryReader()));
                             assertEquals(config.getShardId(), ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(searcher
@@ -332,7 +332,7 @@ public void testCanMatch() throws IOException {
                     }
 
                     try (Engine.SearcherSupplier reader = frozenEngine.acquireSearcherSupplier(Function.identity())) {
-                        try (Engine.Searcher searcher = reader.acquireSearcher("can_match")) {
+                        try (Engine.Searcher searcher = reader.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) {
                             assertSame(dirReader, searcher.getDirectoryReader());
                             assertEquals(0, listener.afterRefresh.get());
                             DirectoryReader unwrap = FilterDirectoryReader.unwrap(searcher.getDirectoryReader());

From 05dd518ed1994848e43e126434ba5434a9277294 Mon Sep 17 00:00:00 2001
From: jimczi <jim.ferenczi@elastic.co>
Date: Tue, 30 Jun 2020 00:15:50 +0200
Subject: [PATCH 36/47] stop execution on query rewrite failures

---
 server/src/main/java/org/elasticsearch/search/SearchService.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 10d4a0674d9bf..393742fc949ba 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -398,6 +398,7 @@ public void onResponse(ShardSearchRequest orig) {
                         } finally {
                             processFailure(request, readerContext, exc);
                         }
+                        return;
                     }
 
                     if (canRewriteToMatchNone(canMatchRequest.source())

From aeeee70b15305fa9a7450f25a3efe1a8a3216bba Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Wed, 1 Jul 2020 17:20:49 -0400
Subject: [PATCH 37/47] add javadocs for RescoreDocIds

---
 .../org/elasticsearch/search/RescoreDocIds.java     | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/RescoreDocIds.java b/server/src/main/java/org/elasticsearch/search/RescoreDocIds.java
index 7d908f53518b9..412a8dcee9c38 100644
--- a/server/src/main/java/org/elasticsearch/search/RescoreDocIds.java
+++ b/server/src/main/java/org/elasticsearch/search/RescoreDocIds.java
@@ -27,6 +27,11 @@
 import java.util.Map;
 import java.util.Set;
 
+/**
+ * Since {@link org.elasticsearch.search.internal.SearchContext} no longer hold the states of search, the top K results
+ * (i.e., documents that will be rescored by query rescorers) need to be serialized/ deserialized between search phases.
+ * A {@link RescoreDocIds} encapsulates the top K results for each rescorer by its ordinal index.
+ */
 public final class RescoreDocIds implements Writeable {
     public static final RescoreDocIds EMPTY = new RescoreDocIds(Map.of());
 
@@ -40,12 +45,12 @@ public RescoreDocIds(StreamInput in) throws IOException {
         docIds = in.readMap(StreamInput::readVInt, i -> i.readSet(StreamInput::readVInt));
     }
 
-    public Set<Integer> getId(int index) {
-        return docIds.get(index);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeMap(docIds, StreamOutput::writeVInt, (o, v) -> o.writeCollection(v, StreamOutput::writeVInt));
     }
+
+    public Set<Integer> getId(int index) {
+        return docIds.get(index);
+    }
 }

From 96fe51f860d5aa5184aad53bbf7101b8d5671b0f Mon Sep 17 00:00:00 2001
From: jimczi <jim.ferenczi@elastic.co>
Date: Thu, 2 Jul 2020 21:53:48 +0200
Subject: [PATCH 38/47] add AliasFilter in the search context id and rename
 SearchContextId into ShardSearchContextId

---
 .../search/AbstractSearchAsyncAction.java     |   2 +-
 .../search/ClearSearchContextController.java  |   7 +-
 .../action/search/FetchSearchPhase.java       |   4 +-
 .../action/search/SearchContextId.java        | 122 ++++++++++++++++++
 .../action/search/SearchContextIdForNode.java |   8 +-
 .../action/search/SearchPhaseContext.java     |   6 +-
 .../search/SearchScrollAsyncAction.java       |   4 +-
 .../action/search/SearchShardIterator.java    |   8 +-
 .../action/search/SearchTransportService.java |  16 +--
 .../TransportOpenSearchContextAction.java     |   6 +-
 .../action/search/TransportSearchAction.java  |  40 +++---
 .../action/search/TransportSearchHelper.java  |  69 +---------
 .../search/DefaultSearchContext.java          |   4 +-
 .../search/SearchContextMissingException.java |  10 +-
 .../search/SearchPhaseResult.java             |   6 +-
 .../elasticsearch/search/SearchService.java   |  17 ++-
 .../search/dfs/DfsSearchResult.java           |   6 +-
 .../search/fetch/FetchSearchResult.java       |   6 +-
 .../search/fetch/QueryFetchSearchResult.java  |   4 +-
 .../search/fetch/ShardFetchRequest.java       |  10 +-
 .../search/fetch/ShardFetchSearchRequest.java |   4 +-
 .../internal/FilteredSearchContext.java       |   2 +-
 .../internal/InternalScrollSearchRequest.java |   8 +-
 .../search/internal/ReaderContext.java        |   6 +-
 .../search/internal/SearchContext.java        |   2 +-
 ...ntextId.java => ShardSearchContextId.java} |   9 +-
 .../search/internal/ShardSearchRequest.java   |  10 +-
 .../search/query/QuerySearchRequest.java      |  10 +-
 .../search/query/QuerySearchResult.java       |   8 +-
 .../ElasticsearchExceptionTests.java          |   4 +-
 .../ExceptionSerializationTests.java          |   4 +-
 .../AbstractSearchAsyncActionTests.java       |  16 +--
 .../ClearSearchContextControllerTests.java    |  18 +--
 .../action/search/CountedCollectorTests.java  |   4 +-
 .../action/search/DfsQueryPhaseTests.java     |  26 ++--
 .../action/search/FetchSearchPhaseTests.java  |  20 +--
 .../action/search/MockSearchPhaseContext.java |   6 +-
 .../action/search/SearchAsyncActionTests.java |  26 ++--
 .../search/SearchPhaseControllerTests.java    |  28 ++--
 .../SearchQueryThenFetchAsyncActionTests.java |   4 +-
 .../search/SearchScrollAsyncActionTests.java  |  52 ++++----
 .../search/SearchScrollRequestTests.java      |   4 +-
 .../search/TransportSearchHelperTests.java    |  28 ++--
 .../search/SearchServiceTests.java            |  14 +-
 .../search/query/QuerySearchResultTests.java  |   4 +-
 .../elasticsearch/test/TestSearchContext.java |   6 +-
 .../SecuritySearchOperationListener.java      |   4 +-
 .../SecuritySearchOperationListenerTests.java |   4 +-
 48 files changed, 374 insertions(+), 312 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/action/search/SearchContextId.java
 rename server/src/main/java/org/elasticsearch/search/internal/{SearchContextId.java => ShardSearchContextId.java} (89%)

diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 97c30f5a71fb9..57e6613ef9100 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -549,7 +549,7 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At
             final Version minNodeVersion = clusterState.nodes().getMinNodeVersion();
             final String scrollId = request.scroll() != null ? TransportSearchHelper.buildScrollId(queryResults, minNodeVersion) : null;
             final String searchContextId =
-                includeSearchContextInResponse() ? TransportSearchHelper.encodeSearchContextId(queryResults, minNodeVersion) : null;
+                includeSearchContextInResponse() ? SearchContextId.encode(queryResults.asList(), aliasFilter, minNodeVersion) : null;
             listener.onResponse(buildSearchResponse(internalSearchResponse, failures, scrollId, searchContextId));
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java b/server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java
index 571be1c6f6c98..5733ddceff1f2 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java
@@ -74,10 +74,9 @@ final class ClearSearchContextController implements Runnable {
         this.logger = logger;
         this.searchTransportService = searchTransportService;
         this.listener = listener;
-        final Collection<SearchContextIdForNode> contexts =
-            TransportSearchHelper.decodeSearchContextId(closeSearchContextRequest.getId()).values();
-        expectedOps = new CountDown(contexts.size());
-        runner = () -> cleanReaderIds(contexts);
+        final SearchContextId context = SearchContextId.decode(closeSearchContextRequest.getId());
+        expectedOps = new CountDown(context.shards().size());
+        runner = () -> cleanReaderIds(context.shards().values());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 2f1f2cee1ccc4..85b00fde26a0f 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.Transport;
@@ -161,7 +161,7 @@ private void innerRun() {
         }
     }
 
-    protected ShardFetchSearchRequest createFetchRequest(SearchContextId contextId, int index, IntArrayList entry,
+    protected ShardFetchSearchRequest createFetchRequest(ShardSearchContextId contextId, int index, IntArrayList entry,
                                                          ScoreDoc[] lastEmittedDocPerShard, OriginalIndices originalIndices,
                                                          ShardSearchRequest shardSearchRequest, RescoreDocIds rescoreDocIds) {
         final ScoreDoc lastEmittedDoc = (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[index] : null;
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java
new file mode 100644
index 0000000000000..2250f5415918e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.ShardSearchContextId;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SearchContextId {
+    private final Map<ShardId, SearchContextIdForNode> shards;
+    private final Map<String, AliasFilter> aliasFilter;
+
+    private SearchContextId(Map<ShardId, SearchContextIdForNode> shards, Map<String, AliasFilter> aliasFilter) {
+        this.shards = shards;
+        this.aliasFilter = aliasFilter;
+    }
+
+    public Map<ShardId, SearchContextIdForNode> shards() {
+        return shards;
+    }
+
+    public Map<String, AliasFilter> aliasFilter() {
+        return aliasFilter;
+    }
+
+    static String encode(List<SearchPhaseResult> searchPhaseResults, Map<String, AliasFilter> aliasFilter, Version version) {
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.writeVInt(version.id);
+            out.writeVInt(searchPhaseResults.size());
+            for (SearchPhaseResult searchPhaseResult : searchPhaseResults) {
+                SearchShardTarget target = searchPhaseResult.getSearchShardTarget();
+                ShardId shardId = target.getShardId();
+                // shardId
+                out.writeString(shardId.getIndex().getName());
+                out.writeString(shardId.getIndex().getUUID());
+                out.writeVInt(shardId.getId());
+                // nodeId
+                if (target.getClusterAlias() != null) {
+                    out.writeByte((byte) 1);
+                    out.writeString(target.getClusterAlias());
+                } else {
+                    out.writeByte((byte) 0);
+                }
+                out.writeString(target.getNodeId());
+                // readerId
+                out.writeString(searchPhaseResult.getContextId().getReaderId());
+                out.writeLong(searchPhaseResult.getContextId().getId());
+            }
+            out.writeMap(aliasFilter, StreamOutput::writeString, (outstream, val) -> val.writeTo(outstream));
+            return Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(out.bytes()));
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    static SearchContextId decode(String id) {
+        final ByteBuffer byteBuffer;
+        try {
+            byteBuffer = ByteBuffer.wrap(Base64.getUrlDecoder().decode(id));
+        } catch (Exception e) {
+            throw new IllegalArgumentException("invalid id: [" + id + "]", e);
+        }
+        try (StreamInput in = new ByteBufferStreamInput(byteBuffer)) {
+            Version version = Version.fromId(in.readVInt());
+            final int numOfContexts = in.readVInt();
+            final Map<ShardId, SearchContextIdForNode> shards = new HashMap<>();
+            for (int i = 0; i < numOfContexts; i++) {
+                final ShardId shardId = new ShardId(new Index(in.readString(), in.readString()), in.readVInt());
+                final String clusterAlias;
+                if (in.readByte() == 1) {
+                    clusterAlias = in.readString();
+                } else {
+                    clusterAlias = null;
+                }
+                final String nodeId = in.readString();
+                final ShardSearchContextId contextId = new ShardSearchContextId(in.readString(), in.readLong());
+                shards.put(shardId, new SearchContextIdForNode(clusterAlias, nodeId, contextId));
+            }
+            Map<String, AliasFilter> aliasFilterMap = in.readMap(StreamInput::readString, AliasFilter::new);
+            if (in.available() > 0) {
+                throw new IllegalArgumentException("Not all bytes were read");
+            }
+            return new SearchContextId(Collections.unmodifiableMap(shards), Collections.unmodifiableMap(aliasFilterMap));
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
index 798087481a39a..6f19d056ccc86 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
@@ -20,14 +20,14 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 
 final class SearchContextIdForNode {
     private final String node;
-    private final SearchContextId searchContextId;
+    private final ShardSearchContextId searchContextId;
     private final String clusterAlias;
 
-    SearchContextIdForNode(@Nullable String clusterAlias, String node, SearchContextId searchContextId) {
+    SearchContextIdForNode(@Nullable String clusterAlias, String node, ShardSearchContextId searchContextId) {
         this.node = node;
         this.clusterAlias = clusterAlias;
         this.searchContextId = searchContextId;
@@ -42,7 +42,7 @@ public String getClusterAlias() {
         return clusterAlias;
     }
 
-    public SearchContextId getSearchContextId() {
+    public ShardSearchContextId getSearchContextId() {
         return searchContextId;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
index 601407cc1ca0f..75ce64dc264eb 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.transport.Transport;
 
@@ -104,7 +104,9 @@ interface SearchPhaseContext extends Executor {
      * @see org.elasticsearch.search.fetch.FetchSearchResult#getContextId()
      *
      */
-    default void sendReleaseSearchContext(SearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) {
+    default void sendReleaseSearchContext(ShardSearchContextId contextId,
+                                          Transport.Connection connection,
+                                          OriginalIndices originalIndices) {
         if (connection != null) {
             getSearchTransport().sendFreeContext(connection, contextId, originalIndices);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
index e8d3387b9ca18..1db433278620d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.Transport;
 
@@ -248,7 +248,7 @@ protected final void sendResponse(SearchPhaseController.ReducedQueryPhase queryP
         }
     }
 
-    protected void onShardFailure(String phaseName, final CountDown counter, final SearchContextId searchId, Exception failure,
+    protected void onShardFailure(String phaseName, final CountDown counter, final ShardSearchContextId searchId, Exception failure,
                                   @Nullable SearchShardTarget searchShardTarget,
                                   Supplier<SearchPhase> nextPhaseSupplier) {
         if (logger.isDebugEnabled()) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
index d46bfb449e5de..50a1351c3642f 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 
 import java.util.List;
 import java.util.Objects;
@@ -44,7 +44,7 @@ public final class SearchShardIterator extends PlainShardIterator {
     private final String clusterAlias;
     private boolean skip = false;
 
-    private final SearchContextId searchContextId;
+    private final ShardSearchContextId searchContextId;
     private final TimeValue searchContextKeepAlive;
 
     /**
@@ -62,7 +62,7 @@ public SearchShardIterator(@Nullable String clusterAlias, ShardId shardId, List<
 
     public SearchShardIterator(@Nullable String clusterAlias, ShardId shardId,
                                List<ShardRouting> shards, OriginalIndices originalIndices,
-                               SearchContextId searchContextId, TimeValue searchContextKeepAlive) {
+                               ShardSearchContextId searchContextId, TimeValue searchContextKeepAlive) {
         super(shardId, shards);
         this.originalIndices = originalIndices;
         this.clusterAlias = clusterAlias;
@@ -97,7 +97,7 @@ SearchShardTarget newSearchShardTarget(String nodeId) {
     /**
      * Returns a non-null value if this request should use a specific search context instead of the latest one.
      */
-    SearchContextId getSearchContextId() {
+    ShardSearchContextId getSearchContextId() {
         return searchContextId;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index ff01b92c35a27..5345213f71c24 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -41,7 +41,7 @@
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
@@ -91,7 +91,7 @@ public SearchTransportService(TransportService transportService,
         this.responseWrapper = responseWrapper;
     }
 
-    public void sendFreeContext(Transport.Connection connection, final SearchContextId contextId, OriginalIndices originalIndices) {
+    public void sendFreeContext(Transport.Connection connection, final ShardSearchContextId contextId, OriginalIndices originalIndices) {
         transportService.sendRequest(connection, FREE_CONTEXT_ACTION_NAME, new SearchFreeContextRequest(originalIndices, contextId),
             TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(new ActionListener<SearchFreeContextResponse>() {
                 @Override
@@ -106,7 +106,7 @@ public void onFailure(Exception e) {
             }, SearchFreeContextResponse::new));
     }
 
-    public void sendFreeContext(Transport.Connection connection, SearchContextId contextId,
+    public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId,
                                 ActionListener<SearchFreeContextResponse> listener) {
         transportService.sendRequest(connection, FREE_CONTEXT_SCROLL_ACTION_NAME, new ScrollFreeContextRequest(contextId),
             TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(listener, SearchFreeContextResponse::new));
@@ -206,15 +206,15 @@ public Map<String, Long> getPendingSearchRequests() {
     }
 
     static class ScrollFreeContextRequest extends TransportRequest {
-        private SearchContextId contextId;
+        private ShardSearchContextId contextId;
 
-        ScrollFreeContextRequest(SearchContextId contextId) {
+        ScrollFreeContextRequest(ShardSearchContextId contextId) {
             this.contextId = Objects.requireNonNull(contextId);
         }
 
         ScrollFreeContextRequest(StreamInput in) throws IOException {
             super(in);
-            contextId = new SearchContextId(in);
+            contextId = new ShardSearchContextId(in);
         }
 
         @Override
@@ -223,7 +223,7 @@ public void writeTo(StreamOutput out) throws IOException {
             contextId.writeTo(out);
         }
 
-        public SearchContextId id() {
+        public ShardSearchContextId id() {
             return this.contextId;
         }
 
@@ -232,7 +232,7 @@ public SearchContextId id() {
     static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest {
         private OriginalIndices originalIndices;
 
-        SearchFreeContextRequest(OriginalIndices originalIndices, SearchContextId id) {
+        SearchFreeContextRequest(OriginalIndices originalIndices, ShardSearchContextId id) {
             super(id);
             this.originalIndices = originalIndices;
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java
index 65c810b803aef..6440e6bb35216 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenSearchContextAction.java
@@ -40,7 +40,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
@@ -136,13 +136,13 @@ public IndicesOptions indicesOptions() {
     }
 
     static final class ShardOpenReaderResponse extends SearchPhaseResult {
-        ShardOpenReaderResponse(SearchContextId contextId) {
+        ShardOpenReaderResponse(ShardSearchContextId contextId) {
             this.contextId = contextId;
         }
 
         ShardOpenReaderResponse(StreamInput in) throws IOException {
             super(in);
-            contextId = new SearchContextId(in);
+            contextId = new ShardSearchContextId(in);
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index ba900736c2f80..ecfc59cd2c065 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
@@ -214,27 +215,27 @@ void executeRequest(Task task, SearchRequest searchRequest,
                 // situations when source is rewritten to null due to a bug
                 searchRequest.source(source);
             }
-            final Map<ShardId, SearchContextIdForNode> searchContexts;
+            final SearchContextId searchContext;
             final Map<String, OriginalIndices> remoteClusterIndices;
             if (searchRequest.searchContextBuilder() != null) {
-                searchContexts = TransportSearchHelper.decodeSearchContextId(searchRequest.searchContextBuilder().getId());
-                remoteClusterIndices = getIndicesFromSearchContexts(searchContexts, searchRequest.indicesOptions());
+                searchContext = SearchContextId.decode(searchRequest.searchContextBuilder().getId());
+                remoteClusterIndices = getIndicesFromSearchContexts(searchContext, searchRequest.indicesOptions());
             } else {
-                searchContexts = null;
+                searchContext = null;
                 remoteClusterIndices = remoteClusterService.groupIndices(searchRequest.indicesOptions(), searchRequest.indices());
             }
             OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
             final ClusterState clusterState = clusterService.state();
             if (remoteClusterIndices.isEmpty()) {
                 executeLocalSearch(
-                    task, timeProvider, searchRequest, localIndices, clusterState, listener, searchContexts, searchAsyncActionProvider);
+                    task, timeProvider, searchRequest, localIndices, clusterState, listener, searchContext, searchAsyncActionProvider);
             } else {
                 if (shouldMinimizeRoundtrips(searchRequest)) {
                     ccsRemoteReduce(searchRequest, localIndices, remoteClusterIndices, timeProvider,
                         searchService.aggReduceContextBuilder(searchRequest),
                         remoteClusterService, threadPool, listener,
                         (r, l) -> executeLocalSearch(
-                            task, timeProvider, r, localIndices, clusterState, l, searchContexts, searchAsyncActionProvider));
+                            task, timeProvider, r, localIndices, clusterState, l, searchContext, searchAsyncActionProvider));
                 } else {
                     AtomicInteger skippedClusters = new AtomicInteger(0);
                     collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(),
@@ -251,7 +252,7 @@ void executeRequest(Task task, SearchRequest searchRequest,
                                 executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices, remoteShardIterators,
                                     clusterNodeLookup, clusterState, remoteAliasFilters, listener,
                                     new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters.get()),
-                                    searchContexts, searchAsyncActionProvider);
+                                    searchContext, searchAsyncActionProvider);
                             },
                             listener::onFailure));
                 }
@@ -423,11 +424,11 @@ SearchResponse createFinalResponse() {
 
     private void executeLocalSearch(Task task, SearchTimeProvider timeProvider, SearchRequest searchRequest, OriginalIndices localIndices,
                                     ClusterState clusterState, ActionListener<SearchResponse> listener,
-                                    Map<ShardId, SearchContextIdForNode> searchContexts,
+                                    SearchContextId searchContext,
                                     SearchAsyncActionProvider searchAsyncActionProvider) {
         executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, Collections.emptyList(),
             (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener, SearchResponse.Clusters.EMPTY,
-            searchContexts, searchAsyncActionProvider);
+            searchContext, searchAsyncActionProvider);
     }
 
     static BiFunction<String, String, DiscoveryNode> processRemoteShards(Map<String, ClusterSearchShardsResponse> searchShardsResponses,
@@ -491,7 +492,7 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
                                OriginalIndices localIndices, List<SearchShardIterator> remoteShardIterators,
                                BiFunction<String, String, DiscoveryNode> remoteConnections, ClusterState clusterState,
                                Map<String, AliasFilter> remoteAliasMap, ActionListener<SearchResponse> listener,
-                               SearchResponse.Clusters clusters, Map<ShardId, SearchContextIdForNode> searchContexts,
+                               SearchResponse.Clusters clusters, @Nullable SearchContextId searchContext,
                                SearchAsyncActionProvider searchAsyncActionProvider) {
 
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
@@ -504,15 +505,12 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
         final Map<String, Set<String>> indexRoutings;
 
         boolean preFilterSearchShards;
-        if (searchContexts != null) {
+        if (searchContext != null) {
             assert searchRequest.searchContextBuilder() != null;
-            aliasFilter = new HashMap<>();
-            for (ShardId shardId : searchContexts.keySet()) {
-                aliasFilter.put(shardId.getIndex().getUUID(), AliasFilter.EMPTY);
-            }
+            aliasFilter = searchContext.aliasFilter();
             indexRoutings = Map.of();
             localShardIterators = getSearchShardsFromSearchContexts(clusterState, localIndices, searchRequest.getLocalClusterAlias(),
-                searchContexts, searchRequest.searchContextBuilder().getKeepAlive());
+                searchContext, searchRequest.searchContextBuilder().getKeepAlive());
             preFilterSearchShards = shouldPreFilterSearchShards(clusterState, searchRequest, localIndices.indices(),
                 localShardIterators.size() + remoteShardIterators.size());
         } else {
@@ -774,10 +772,10 @@ private static RemoteTransportException wrapRemoteClusterFailure(String clusterA
         return new RemoteTransportException("error while communicating with remote cluster [" + clusterAlias + "]", e);
     }
 
-    static Map<String, OriginalIndices> getIndicesFromSearchContexts(Map<ShardId, SearchContextIdForNode> searchContexts,
+    static Map<String, OriginalIndices> getIndicesFromSearchContexts(SearchContextId searchContext,
                                                                      IndicesOptions indicesOptions) {
         final Map<String, List<String>> indices = new HashMap<>();
-        for (Map.Entry<ShardId, SearchContextIdForNode> entry : searchContexts.entrySet()) {
+        for (Map.Entry<ShardId, SearchContextIdForNode> entry : searchContext.shards().entrySet()) {
             String clusterAlias = entry.getValue().getClusterAlias() == null ?
                 RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY : entry.getValue().getClusterAlias();
             indices.computeIfAbsent(clusterAlias, k -> new ArrayList<>()).add(entry.getKey().getIndexName());
@@ -788,10 +786,10 @@ static Map<String, OriginalIndices> getIndicesFromSearchContexts(Map<ShardId, Se
 
     static List<SearchShardIterator> getSearchShardsFromSearchContexts(ClusterState clusterState, OriginalIndices originalIndices,
                                                                        String localClusterAlias,
-                                                                       Map<ShardId, SearchContextIdForNode> searchContexts,
+                                                                       SearchContextId searchContext,
                                                                        TimeValue keepAlive) {
-        final List<SearchShardIterator> iterators = new ArrayList<>(searchContexts.size());
-        for (Map.Entry<ShardId, SearchContextIdForNode> entry : searchContexts.entrySet()) {
+        final List<SearchShardIterator> iterators = new ArrayList<>(searchContext.shards().size());
+        for (Map.Entry<ShardId, SearchContextIdForNode> entry : searchContext.shards().entrySet()) {
             final ShardId shardId = entry.getKey();
             final ShardIterator shards = OperationRouting.getShards(clusterState, shardId);
             final List<ShardRouting> matchingNodeFirstRoutings = new ArrayList<>();
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
index f545f1ca3c641..9a01b74579aa2 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
@@ -23,25 +23,21 @@
 import org.apache.lucene.store.RAMOutputStream;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.transport.RemoteClusterAware;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Base64;
-import java.util.HashMap;
-import java.util.Map;
 
 final class TransportSearchHelper {
 
     private static final String INCLUDE_CONTEXT_UUID = "include_context_uuid";
 
-    static InternalScrollSearchRequest internalScrollSearchRequest(SearchContextId id, SearchScrollRequest request) {
+    static InternalScrollSearchRequest internalScrollSearchRequest(ShardSearchContextId id, SearchScrollRequest request) {
         return new InternalScrollSearchRequest(request, id);
     }
 
@@ -101,7 +97,7 @@ static ParsedScrollId parseScrollId(String scrollId) {
                     clusterAlias = target.substring(0, index);
                     target = target.substring(index+1);
                 }
-                context[i] = new SearchContextIdForNode(clusterAlias, target, new SearchContextId(contextUUID, id));
+                context[i] = new SearchContextIdForNode(clusterAlias, target, new ShardSearchContextId(contextUUID, id));
             }
             if (in.getPosition() != bytes.length) {
                 throw new IllegalArgumentException("Not all bytes were read");
@@ -112,65 +108,6 @@ static ParsedScrollId parseScrollId(String scrollId) {
         }
     }
 
-    static String encodeSearchContextId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults, Version version) {
-        try (RAMOutputStream out = new RAMOutputStream()) {
-            out.writeVInt(version.id);
-            out.writeVInt(searchPhaseResults.asList().size());
-            for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) {
-                SearchShardTarget target = searchPhaseResult.getSearchShardTarget();
-                ShardId shardId = target.getShardId();
-                // shardId
-                out.writeString(shardId.getIndex().getName());
-                out.writeString(shardId.getIndex().getUUID());
-                out.writeVInt(shardId.getId());
-                // nodeId
-                if (target.getClusterAlias() != null) {
-                    out.writeByte((byte) 1);
-                    out.writeString(target.getClusterAlias());
-                } else {
-                    out.writeByte((byte) 0);
-                }
-                out.writeString(target.getNodeId());
-                // readerId
-                out.writeString(searchPhaseResult.getContextId().getReaderId());
-                out.writeLong(searchPhaseResult.getContextId().getId());
-            }
-            byte[] bytes = new byte[(int) out.getFilePointer()];
-            out.writeTo(bytes, 0);
-            return Base64.getUrlEncoder().encodeToString(bytes);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
-    static Map<ShardId, SearchContextIdForNode> decodeSearchContextId(String searchContextId) {
-        try {
-            final byte[] bytes = Base64.getUrlDecoder().decode(searchContextId);
-            final ByteArrayDataInput in = new ByteArrayDataInput(bytes);
-            Version.fromId(in.readVInt());
-            final int numOfContexts = in.readVInt();
-            final Map<ShardId, SearchContextIdForNode> contexts = new HashMap<>();
-            for (int i = 0; i < numOfContexts; i++) {
-                final ShardId shardId = new ShardId(new Index(in.readString(), in.readString()), in.readVInt());
-                final String clusterAlias;
-                if (in.readByte() == 1) {
-                    clusterAlias = in.readString();
-                } else {
-                    clusterAlias = null;
-                }
-                final String nodeId = in.readString();
-                final SearchContextId contextId = new SearchContextId(in.readString(), in.readLong());
-                contexts.put(shardId, new SearchContextIdForNode(clusterAlias, nodeId, contextId));
-            }
-            if (in.getPosition() != bytes.length) {
-                throw new IllegalArgumentException("Not all bytes were read");
-            }
-            return contexts;
-        } catch (Exception e) {
-            throw new IllegalArgumentException("Can't parse reader context ids", e);
-        }
-    }
-
     private TransportSearchHelper() {
 
     }
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 5ebce9d7332b3..bd6e3a58f66f0 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -62,7 +62,7 @@
 import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
@@ -297,7 +297,7 @@ && new NestedHelper(mapperService()).mightMatchNestedDocs(query)
     }
 
     @Override
-    public SearchContextId id() {
+    public ShardSearchContextId id() {
         return readerContext.id();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java b/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java
index 03ac85a8d81ac..aab7c5a9a9580 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchContextMissingException.java
@@ -23,20 +23,20 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 
 import java.io.IOException;
 
 public class SearchContextMissingException extends ElasticsearchException {
 
-    private final SearchContextId contextId;
+    private final ShardSearchContextId contextId;
 
-    public SearchContextMissingException(SearchContextId contextId) {
+    public SearchContextMissingException(ShardSearchContextId contextId) {
         super("No search context found for id [" + contextId.getId() + "]");
         this.contextId = contextId;
     }
 
-    public SearchContextId contextId() {
+    public ShardSearchContextId contextId() {
         return this.contextId;
     }
 
@@ -47,7 +47,7 @@ public RestStatus status() {
 
     public SearchContextMissingException(StreamInput in) throws IOException{
         super(in);
-        contextId = new SearchContextId(in);
+        contextId = new ShardSearchContextId(in);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java b/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
index b58bb0fb0f899..b2ef32ef3e2ea 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.TransportResponse;
@@ -42,7 +42,7 @@ public abstract class SearchPhaseResult extends TransportResponse {
 
     private SearchShardTarget searchShardTarget;
     private int shardIndex = -1;
-    protected SearchContextId contextId;
+    protected ShardSearchContextId contextId;
     private ShardSearchRequest shardSearchRequest;
     private RescoreDocIds rescoreDocIds = RescoreDocIds.EMPTY;
 
@@ -59,7 +59,7 @@ protected SearchPhaseResult(StreamInput in) throws IOException {
      * or <code>null</code> if no context was created.
      */
     @Nullable
-    public SearchContextId getContextId() {
+    public ShardSearchContextId getContextId() {
         return contextId;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 393742fc949ba..90768b8b7ad53 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -98,7 +98,7 @@
 import org.elasticsearch.search.internal.LegacyReaderContext;
 import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhase;
@@ -392,7 +392,7 @@ public void onResponse(ShardSearchRequest orig) {
                         QueryShardContext context = readerContext.indexService().newQueryShardContext(canMatchRequest.shardId().id(),
                             searcher, canMatchRequest::nowInMillis, canMatchRequest.getClusterAlias());
                         Rewriteable.rewrite(canMatchRequest.getRewriteable(), context, true);
-                    } catch (IOException exc) {
+                    } catch (Exception exc) {
                         try (markAsUsed) {
                             listener.onFailure(exc);
                         } finally {
@@ -400,7 +400,6 @@ public void onResponse(ShardSearchRequest orig) {
                         }
                         return;
                     }
-
                     if (canRewriteToMatchNone(canMatchRequest.source())
                         && canMatchRequest.source().query() instanceof MatchNoneQueryBuilder) {
                         try (markAsUsed) {
@@ -409,8 +408,8 @@ public void onResponse(ShardSearchRequest orig) {
                                     listener.onResponse(QuerySearchResult.nullInstance());
                                 } finally {
                                     // close and remove the ephemeral reader context
-                                    Releasables.close(readerContext);
                                     removeReaderContext(readerContext.id().getId());
+                                    Releasables.close(readerContext);
                                 }
                             } else {
                                 listener.onResponse(QuerySearchResult.nullInstance());
@@ -618,7 +617,7 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
         }, listener);
     }
 
-    private ReaderContext getReaderContext(SearchContextId id) {
+    private ReaderContext getReaderContext(ShardSearchContextId id) {
         final ReaderContext reader = activeReaders.get(id.getId());
         if (reader == null) {
             return null;
@@ -629,7 +628,7 @@ private ReaderContext getReaderContext(SearchContextId id) {
         return null;
     }
 
-    private ReaderContext findReaderContext(SearchContextId id) throws SearchContextMissingException {
+    private ReaderContext findReaderContext(ShardSearchContextId id) throws SearchContextMissingException {
         final ReaderContext reader = getReaderContext(id);
         if (reader == null) {
             throw new SearchContextMissingException(id);
@@ -709,7 +708,7 @@ final ReaderContext createAndPutReaderContext(ShardSearchRequest request, IndexS
      * Opens the reader context for given shardId. The newly opened reader context will be keep
      * until the {@code keepAlive} elapsed unless it is manually released.
      */
-    public void openReaderContext(ShardId shardId, TimeValue keepAlive, ActionListener<SearchContextId> listener) {
+    public void openReaderContext(ShardId shardId, TimeValue keepAlive, ActionListener<ShardSearchContextId> listener) {
         checkKeepAliveLimit(keepAlive.millis());
         final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         final IndexShard shard = indexService.getShard(shardId.id());
@@ -814,7 +813,7 @@ private void freeAllContextForIndex(Index index) {
         }
     }
 
-    public boolean freeReaderContext(SearchContextId contextId) {
+    public boolean freeReaderContext(ShardSearchContextId contextId) {
         if (getReaderContext(contextId) != null) {
             try (ReaderContext context = removeReaderContext(contextId.getId())) {
                 return context != null;
@@ -849,7 +848,7 @@ private void checkKeepAliveLimit(long keepAlive) {
     }
 
     private void processFailure(ShardSearchRequest request, ReaderContext context, Exception e) {
-        if (request.readerId() == null || request.scroll() != null) {
+        if (context.singleSession() || request.scroll() != null) {
             // we release the reader on failure if the request is a normal search or a scroll
             freeReaderContext(context.id());
         }
diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
index 40fecfc0f9a6b..a7a3ff7085d2e 100644
--- a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 
 import java.io.IOException;
@@ -47,7 +47,7 @@ public class DfsSearchResult extends SearchPhaseResult {
 
     public DfsSearchResult(StreamInput in) throws IOException {
         super(in);
-        contextId = new SearchContextId(in);
+        contextId = new ShardSearchContextId(in);
         int termsSize = in.readVInt();
         if (termsSize == 0) {
             terms = EMPTY_TERMS;
@@ -66,7 +66,7 @@ public DfsSearchResult(StreamInput in) throws IOException {
         }
     }
 
-    public DfsSearchResult(SearchContextId contextId, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) {
+    public DfsSearchResult(ShardSearchContextId contextId, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) {
         this.setSearchShardTarget(shardTarget);
         this.contextId = contextId;
         setShardSearchRequest(shardSearchRequest);
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
index 81a2f5491a9a3..cd6b836df7733 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 
 import java.io.IOException;
@@ -41,11 +41,11 @@ public FetchSearchResult() {
 
     public FetchSearchResult(StreamInput in) throws IOException {
         super(in);
-        contextId = new SearchContextId(in);
+        contextId = new ShardSearchContextId(in);
         hits = new SearchHits(in);
     }
 
-    public FetchSearchResult(SearchContextId id, SearchShardTarget shardTarget) {
+    public FetchSearchResult(ShardSearchContextId id, SearchShardTarget shardTarget) {
         this.contextId = id;
         setSearchShardTarget(shardTarget);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java b/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
index dc76fdf46f792..6bef3fd80c45c 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/QueryFetchSearchResult.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 
 import java.io.IOException;
@@ -45,7 +45,7 @@ public QueryFetchSearchResult(QuerySearchResult queryResult, FetchSearchResult f
     }
 
     @Override
-    public SearchContextId getContextId() {
+    public ShardSearchContextId getContextId() {
         return queryResult.getContextId();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
index f9237c11ee44a..655a87e78ac59 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.search.RescoreDocIds;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.internal.ShardSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
@@ -44,7 +44,7 @@
  */
 public class ShardFetchRequest extends TransportRequest {
 
-    private SearchContextId contextId;
+    private ShardSearchContextId contextId;
 
     private int[] docIds;
 
@@ -52,7 +52,7 @@ public class ShardFetchRequest extends TransportRequest {
 
     private ScoreDoc lastEmittedDoc;
 
-    public ShardFetchRequest(SearchContextId contextId, IntArrayList list, ScoreDoc lastEmittedDoc) {
+    public ShardFetchRequest(ShardSearchContextId contextId, IntArrayList list, ScoreDoc lastEmittedDoc) {
         this.contextId = contextId;
         this.docIds = list.buffer;
         this.size = list.size();
@@ -61,7 +61,7 @@ public ShardFetchRequest(SearchContextId contextId, IntArrayList list, ScoreDoc
 
     public ShardFetchRequest(StreamInput in) throws IOException {
         super(in);
-        contextId = new SearchContextId(in);
+        contextId = new ShardSearchContextId(in);
         size = in.readVInt();
         docIds = new int[size];
         for (int i = 0; i < size; i++) {
@@ -96,7 +96,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    public SearchContextId contextId() {
+    public ShardSearchContextId contextId() {
         return contextId;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
index e721d83e460b2..8a91b88173835 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.RescoreDocIds;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.internal.ShardSearchRequest;
@@ -45,7 +45,7 @@ public class ShardFetchSearchRequest extends ShardFetchRequest implements Indice
     private final RescoreDocIds rescoreDocIds;
     private final AggregatedDfs aggregatedDfs;
 
-    public ShardFetchSearchRequest(OriginalIndices originalIndices, SearchContextId id, ShardSearchRequest shardSearchRequest,
+    public ShardFetchSearchRequest(OriginalIndices originalIndices, ShardSearchContextId id, ShardSearchRequest shardSearchRequest,
                                    IntArrayList list, ScoreDoc lastEmittedDoc, RescoreDocIds rescoreDocIds, AggregatedDfs aggregatedDfs) {
         super(id, list, lastEmittedDoc);
         this.originalIndices = originalIndices;
diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index ab5073017d414..aecba63fbefe7 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -106,7 +106,7 @@ public Query buildFilteredQuery(Query query) {
     }
 
     @Override
-    public SearchContextId id() {
+    public ShardSearchContextId id() {
         return in.id();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
index ff86e44c1704b..f522d7760b2f7 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/InternalScrollSearchRequest.java
@@ -33,21 +33,21 @@
 
 public class InternalScrollSearchRequest extends TransportRequest {
 
-    private SearchContextId contextId;
+    private ShardSearchContextId contextId;
 
     private Scroll scroll;
 
     public InternalScrollSearchRequest() {
     }
 
-    public InternalScrollSearchRequest(SearchScrollRequest request, SearchContextId contextId) {
+    public InternalScrollSearchRequest(SearchScrollRequest request, ShardSearchContextId contextId) {
         this.contextId = contextId;
         this.scroll = request.scroll();
     }
 
     public InternalScrollSearchRequest(StreamInput in) throws IOException {
         super(in);
-        contextId = new SearchContextId(in);
+        contextId = new ShardSearchContextId(in);
         scroll = in.readOptionalWriteable(Scroll::new);
     }
 
@@ -58,7 +58,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(scroll);
     }
 
-    public SearchContextId contextId() {
+    public ShardSearchContextId contextId() {
         return contextId;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
index 43f0a8272af6f..507a2c1fbb4d4 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ReaderContext.java
@@ -45,7 +45,7 @@
  * can be guarded by a reference count and fail if it's been closed by an external event.
  */
 public class ReaderContext implements Releasable {
-    private final SearchContextId id;
+    private final ShardSearchContextId id;
     private final IndexService indexService;
     private final IndexShard indexShard;
     protected final Engine.SearcherSupplier searcherSupplier;
@@ -69,7 +69,7 @@ public ReaderContext(long id,
                          Engine.SearcherSupplier searcherSupplier,
                          long keepAliveInMillis,
                          boolean singleSession) {
-        this.id = new SearchContextId(UUIDs.base64UUID(), id);
+        this.id = new ShardSearchContextId(UUIDs.base64UUID(), id);
         this.indexService = indexService;
         this.indexShard = indexShard;
         this.searcherSupplier = searcherSupplier;
@@ -103,7 +103,7 @@ public void addOnClose(Releasable releasable) {
         onCloses.add(releasable);
     }
 
-    public SearchContextId id() {
+    public ShardSearchContextId id() {
         return id;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 886e7e0e28f28..b10702c26d29c 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -113,7 +113,7 @@ public final void close() {
      *  alias filters, types filters, etc. */
     public abstract Query buildFilteredQuery(Query query);
 
-    public abstract SearchContextId id();
+    public abstract ShardSearchContextId id();
 
     public abstract String source();
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchContextId.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
rename to server/src/main/java/org/elasticsearch/search/internal/ShardSearchContextId.java
index 8cb64322d1938..d9474a279aa0c 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContextId.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchContextId.java
@@ -27,17 +27,16 @@
 import java.io.IOException;
 import java.util.Objects;
 
-// TODO: Rename this to ReaderId?
-public final class SearchContextId implements Writeable {
+public final class ShardSearchContextId implements Writeable {
     private final String readerId;
     private final long id;
 
-    public SearchContextId(String readerId, long id) {
+    public ShardSearchContextId(String readerId, long id) {
         this.readerId = Objects.requireNonNull(readerId);
         this.id = id;
     }
 
-    public SearchContextId(StreamInput in) throws IOException {
+    public ShardSearchContextId(StreamInput in) throws IOException {
         this.id = in.readLong();
         if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
             this.readerId = in.readString();
@@ -66,7 +65,7 @@ public long getId() {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-        SearchContextId other = (SearchContextId) o;
+        ShardSearchContextId other = (ShardSearchContextId) o;
         return id == other.id && readerId.equals(other.readerId);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index d3963ce844b99..3f7645dd46937 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -89,7 +89,7 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque
     //these are the only mutable fields, as they are subject to rewriting
     private AliasFilter aliasFilter;
     private SearchSourceBuilder source;
-    private final SearchContextId readerId;
+    private final ShardSearchContextId readerId;
     private final TimeValue keepAlive;
 
     public ShardSearchRequest(OriginalIndices originalIndices,
@@ -114,7 +114,7 @@ public ShardSearchRequest(OriginalIndices originalIndices,
                               long nowInMillis,
                               @Nullable String clusterAlias,
                               String[] indexRoutings,
-                              SearchContextId readerId,
+                              ShardSearchContextId readerId,
                               TimeValue keepAlive) {
         this(originalIndices,
             shardId,
@@ -158,7 +158,7 @@ private ShardSearchRequest(OriginalIndices originalIndices,
                                Scroll scroll,
                                long nowInMillis,
                                @Nullable String clusterAlias,
-                               SearchContextId readerId,
+                               ShardSearchContextId readerId,
                                TimeValue keepAlive) {
         this.shardId = shardId;
         this.numberOfShards = numberOfShards;
@@ -210,7 +210,7 @@ public ShardSearchRequest(StreamInput in) throws IOException {
             bottomSortValues = null;
         }
         if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
-            this.readerId = in.readOptionalWriteable(SearchContextId::new);
+            this.readerId = in.readOptionalWriteable(ShardSearchContextId::new);
             this.keepAlive = in.readOptionalTimeValue();
         } else {
             this.readerId = null;
@@ -385,7 +385,7 @@ public void canReturnNullResponseIfMatchNoDocs(boolean value) {
      * Returns a non-null value if this request should execute using a specific point-in-time reader;
      * otherwise, using the most up to date point-in-time reader.
      */
-    public SearchContextId readerId() {
+    public ShardSearchContextId readerId() {
         return readerId;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
index eda9711d70cd7..efb9410b88a29 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.dfs.AggregatedDfs;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -40,12 +40,12 @@
 
 public class QuerySearchRequest extends TransportRequest implements IndicesRequest {
 
-    private final SearchContextId contextId;
+    private final ShardSearchContextId contextId;
     private final AggregatedDfs dfs;
     private final OriginalIndices originalIndices;
     private final ShardSearchRequest shardSearchRequest;
 
-    public QuerySearchRequest(OriginalIndices originalIndices, SearchContextId contextId,
+    public QuerySearchRequest(OriginalIndices originalIndices, ShardSearchContextId contextId,
                               ShardSearchRequest shardSearchRequest, AggregatedDfs dfs) {
         this.contextId = contextId;
         this.dfs = dfs;
@@ -55,7 +55,7 @@ public QuerySearchRequest(OriginalIndices originalIndices, SearchContextId conte
 
     public QuerySearchRequest(StreamInput in) throws IOException {
         super(in);
-        contextId = new SearchContextId(in);
+        contextId = new ShardSearchContextId(in);
         dfs = new AggregatedDfs(in);
         originalIndices = OriginalIndices.readOriginalIndices(in);
         if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
@@ -76,7 +76,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    public SearchContextId contextId() {
+    public ShardSearchContextId contextId() {
         return contextId;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 694b54a3158b7..d03741539e17b 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
@@ -82,12 +82,12 @@ public QuerySearchResult(StreamInput in) throws IOException {
             isNull = false;
         }
         if (isNull == false) {
-            SearchContextId id = new SearchContextId(in);
+            ShardSearchContextId id = new ShardSearchContextId(in);
             readFromWithId(id, in);
         }
     }
 
-    public QuerySearchResult(SearchContextId contextId, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) {
+    public QuerySearchResult(ShardSearchContextId contextId, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) {
         this.contextId = contextId;
         setSearchShardTarget(shardTarget);
         isNull = false;
@@ -304,7 +304,7 @@ public boolean hasSearchContext() {
         return hasScoreDocs || hasSuggestHits();
     }
 
-    public void readFromWithId(SearchContextId id, StreamInput in) throws IOException {
+    public void readFromWithId(ShardSearchContextId id, StreamInput in) throws IOException {
         this.contextId = id;
         from = in.readVInt();
         size = in.readVInt();
diff --git a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
index 0321f247b0843..1e3f8dbc0a4b1 100644
--- a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
+++ b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
@@ -57,7 +57,7 @@
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.RemoteTransportException;
 
@@ -821,7 +821,7 @@ public void testFailureToAndFromXContentWithDetails() throws IOException {
                                     OriginalIndices.NONE)), new ShardSearchFailure(new RepositoryException("repository_g", "Repo"),
                                 new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 62), null,
                                     OriginalIndices.NONE)), new ShardSearchFailure(
-                                        new SearchContextMissingException(new SearchContextId(UUIDs.randomBase64UUID(), 0L)), null)
+                                        new SearchContextMissingException(new ShardSearchContextId(UUIDs.randomBase64UUID(), 0L)), null)
                 };
                 failure = new SearchPhaseExecutionException("phase_g", "G", failureCause, shardFailures);
 
diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 6281b5c228fc1..ee6f2a0abf688 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -80,7 +80,7 @@
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotException;
 import org.elasticsearch.snapshots.SnapshotId;
@@ -354,7 +354,7 @@ public void testActionTransportException() throws IOException {
     }
 
     public void testSearchContextMissingException() throws IOException {
-        SearchContextId contextId = new SearchContextId(UUIDs.randomBase64UUID(), randomLong());
+        ShardSearchContextId contextId = new ShardSearchContextId(UUIDs.randomBase64UUID(), randomLong());
         Version version = VersionUtils.randomVersion(random());
         SearchContextMissingException ex = serialize(new SearchContextMissingException(contextId), version);
         assertThat(ex.contextId().getId(), equalTo(contextId.getId()));
diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
index 1682f4a3eb986..0b06e378d11b3 100644
--- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
@@ -35,7 +35,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
@@ -58,7 +58,7 @@
 public class AbstractSearchAsyncActionTests extends ESTestCase {
 
     private final List<Tuple<String, String>> resolvedNodes = new ArrayList<>();
-    private final Set<SearchContextId> releasedContexts = new CopyOnWriteArraySet<>();
+    private final Set<ShardSearchContextId> releasedContexts = new CopyOnWriteArraySet<>();
 
     private AbstractSearchAsyncAction<SearchPhaseResult> createAction(SearchRequest request,
                                                                       ArraySearchPhaseResults<SearchPhaseResult> results,
@@ -113,7 +113,7 @@ long buildTookInMillis() {
             }
 
             @Override
-            public void sendReleaseSearchContext(SearchContextId contextId, Transport.Connection connection,
+            public void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection,
                                                  OriginalIndices originalIndices) {
                 releasedContexts.add(contextId);
             }
@@ -192,7 +192,7 @@ public void testSendSearchResponseDisallowPartialFailures() {
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(false);
         AtomicReference<Exception> exception = new AtomicReference<>();
         ActionListener<SearchResponse> listener = ActionListener.wrap(response -> fail("onResponse should not be called"), exception::set);
-        Set<SearchContextId> requestIds = new HashSet<>();
+        Set<ShardSearchContextId> requestIds = new HashSet<>();
         List<Tuple<String, String>> nodeLookups = new ArrayList<>();
         int numFailures = randomIntBetween(1, 5);
         ArraySearchPhaseResults<SearchPhaseResult> phaseResults = phaseResults(requestIds, nodeLookups, numFailures);
@@ -220,7 +220,7 @@ public void testOnPhaseFailure() {
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(false);
         AtomicReference<Exception> exception = new AtomicReference<>();
         ActionListener<SearchResponse> listener = ActionListener.wrap(response -> fail("onResponse should not be called"), exception::set);
-        Set<SearchContextId> requestIds = new HashSet<>();
+        Set<ShardSearchContextId> requestIds = new HashSet<>();
         List<Tuple<String, String>> nodeLookups = new ArrayList<>();
         ArraySearchPhaseResults<SearchPhaseResult> phaseResults = phaseResults(requestIds, nodeLookups, 0);
         AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(searchRequest, phaseResults, listener, false, new AtomicLong());
@@ -263,14 +263,14 @@ public void testShardNotAvailableWithDisallowPartialFailures() {
         assertEquals(0, searchPhaseExecutionException.getSuppressed().length);
     }
 
-    private static ArraySearchPhaseResults<SearchPhaseResult> phaseResults(Set<SearchContextId> contextIds,
+    private static ArraySearchPhaseResults<SearchPhaseResult> phaseResults(Set<ShardSearchContextId> contextIds,
                                                                            List<Tuple<String, String>> nodeLookups,
                                                                            int numFailures) {
         int numResults = randomIntBetween(1, 10);
         ArraySearchPhaseResults<SearchPhaseResult> phaseResults = new ArraySearchPhaseResults<>(numResults + numFailures);
 
         for (int i = 0; i < numResults; i++) {
-            SearchContextId contextId = new SearchContextId(UUIDs.randomBase64UUID(), randomNonNegativeLong());
+            ShardSearchContextId contextId = new ShardSearchContextId(UUIDs.randomBase64UUID(), randomNonNegativeLong());
             contextIds.add(contextId);
             SearchPhaseResult phaseResult = new PhaseResult(contextId);
             String resultClusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10);
@@ -285,7 +285,7 @@ private static ArraySearchPhaseResults<SearchPhaseResult> phaseResults(Set<Searc
     }
 
     private static final class PhaseResult extends SearchPhaseResult {
-        PhaseResult(SearchContextId contextId) {
+        PhaseResult(ShardSearchContextId contextId) {
             this.contextId = contextId;
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/action/search/ClearSearchContextControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/ClearSearchContextControllerTests.java
index 3238e0e654dbc..dfe7775224bb4 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ClearSearchContextControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ClearSearchContextControllerTests.java
@@ -28,7 +28,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.transport.NodeNotConnectedException;
@@ -96,13 +96,13 @@ public void testClearScrollIds() throws IOException, InterruptedException {
         DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
         AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 1), node1);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 1), node1);
         testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), null, null));
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 12), node2);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 12), node2);
         testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), null, null));
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 42), node3);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 42), node3);
         testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
         array.setOnce(0, testSearchPhaseResult1);
         array.setOnce(1, testSearchPhaseResult2);
@@ -127,7 +127,7 @@ public void onFailure(Exception e) {
         SearchTransportService searchTransportService = new SearchTransportService(null, null) {
 
             @Override
-            public void sendFreeContext(Transport.Connection connection, SearchContextId contextId,
+            public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId,
                                         ActionListener<SearchFreeContextResponse> listener) {
                 nodesInvoked.add(connection.getNode());
                 boolean freed = randomBoolean();
@@ -160,13 +160,13 @@ public void testClearScrollIdsWithFailure() throws IOException, InterruptedExcep
         DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
         AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 1), node1);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 1), node1);
         testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), null, null));
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 12), node2);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 12), node2);
         testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), null, null));
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(), 42), node3);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 42), node3);
         testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
         array.setOnce(0, testSearchPhaseResult1);
         array.setOnce(1, testSearchPhaseResult2);
@@ -198,7 +198,7 @@ public void onFailure(Exception e) {
         SearchTransportService searchTransportService = new SearchTransportService(null, null) {
 
             @Override
-            public void sendFreeContext(Transport.Connection connection, SearchContextId contextId,
+            public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId,
                                         ActionListener<SearchFreeContextResponse> listener) {
                 nodesInvoked.add(connection.getNode());
                 boolean freed = randomBoolean();
diff --git a/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java b/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
index 9678ff3864d48..10bf2a8718924 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -62,7 +62,7 @@ public void testCollect() throws InterruptedException {
                     state.add(1);
                     executor.execute(() -> {
                         DfsSearchResult dfsSearchResult = new DfsSearchResult(
-                            new SearchContextId(UUIDs.randomBase64UUID(), shardID), null, null);
+                            new ShardSearchContextId(UUIDs.randomBase64UUID(), shardID), null, null);
                         dfsSearchResult.setShardIndex(shardID);
                         dfsSearchResult.setSearchShardTarget(new SearchShardTarget("foo",
                             new ShardId("bar", "baz", shardID), null, OriginalIndices.NONE));
diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index 5207d7c7ce696..fdf5c0fbf3143 100644
--- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.test.ESTestCase;
@@ -45,7 +45,7 @@
 
 public class DfsQueryPhaseTests extends ESTestCase {
 
-    private static DfsSearchResult newSearchResult(int shardIndex, SearchContextId contextId, SearchShardTarget target) {
+    private static DfsSearchResult newSearchResult(int shardIndex, ShardSearchContextId contextId, SearchShardTarget target) {
         DfsSearchResult result = new DfsSearchResult(contextId, target, null);
         result.setShardIndex(shardIndex);
         return result;
@@ -54,9 +54,9 @@ private static DfsSearchResult newSearchResult(int shardIndex, SearchContextId c
     public void testDfsWith2Shards() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, new SearchContextId("", 1),
+        results.set(0, newSearchResult(0, new ShardSearchContextId("", 1),
             new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, new SearchContextId("", 2),
+        results.set(1, newSearchResult(1, new ShardSearchContextId("", 2),
             new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
@@ -66,7 +66,7 @@ public void testDfsWith2Shards() throws IOException {
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
                 if (request.contextId().getId() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
+                    QuerySearchResult queryResult = new QuerySearchResult(new ShardSearchContextId("", 123),
                         new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
@@ -74,7 +74,7 @@ public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest
                     queryResult.size(2); // the size of the result set
                     listener.onResponse(queryResult);
                 } else if (request.contextId().getId() == 2) {
-                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
+                    QuerySearchResult queryResult = new QuerySearchResult(new ShardSearchContextId("", 123),
                         new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F),
@@ -114,9 +114,9 @@ public void run() throws IOException {
     public void testDfsWith1ShardFailed() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, new SearchContextId("", 1),
+        results.set(0, newSearchResult(0, new ShardSearchContextId("", 1),
             new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, new SearchContextId("", 2),
+        results.set(1, newSearchResult(1, new ShardSearchContextId("", 2),
             new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
@@ -126,7 +126,7 @@ public void testDfsWith1ShardFailed() throws IOException {
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
                 if (request.contextId().getId() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
+                    QuerySearchResult queryResult = new QuerySearchResult(new ShardSearchContextId("", 123),
                         new SearchShardTarget("node1", new ShardId("test", "na", 0),
                         null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(
@@ -164,7 +164,7 @@ public void run() throws IOException {
         assertEquals(1, mockSearchPhaseContext.failures.size());
         assertTrue(mockSearchPhaseContext.failures.get(0).getCause() instanceof MockDirectoryWrapper.FakeIOException);
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
-        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(new SearchContextId("", 2L)));
+        assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(new ShardSearchContextId("", 2L)));
         assertNull(responseRef.get().get(1));
     }
 
@@ -172,9 +172,9 @@ public void run() throws IOException {
     public void testFailPhaseOnException() throws IOException {
         AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
         AtomicReference<AtomicArray<SearchPhaseResult>> responseRef = new AtomicReference<>();
-        results.set(0, newSearchResult(0, new SearchContextId("", 1),
+        results.set(0, newSearchResult(0, new ShardSearchContextId("", 1),
             new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
-        results.set(1, newSearchResult(1, new SearchContextId("", 2),
+        results.set(1, newSearchResult(1, new ShardSearchContextId("", 2),
             new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE)));
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
@@ -184,7 +184,7 @@ public void testFailPhaseOnException() throws IOException {
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
                 if (request.contextId().getId() == 1) {
-                    QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
+                    QuerySearchResult queryResult = new QuerySearchResult(new ShardSearchContextId("", 123),
                         new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
                     queryResult.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index 4b7130dc88abd..f35865b906a72 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -34,7 +34,7 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.QueryFetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalAggregationTestCase;
@@ -96,7 +96,7 @@ public void testFetchTwoDocument() {
             writableRegistry(), s -> InternalAggregationTestCase.emptyReduceContextBuilder());
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        SearchContextId ctx1 = new SearchContextId(UUIDs.base64UUID(), 123);
+        ShardSearchContextId ctx1 = new ShardSearchContextId(UUIDs.base64UUID(), 123);
         QuerySearchResult queryResult = new QuerySearchResult(ctx1, new SearchShardTarget("node1", new ShardId("test", "na", 0),
             null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
@@ -105,7 +105,7 @@ public void testFetchTwoDocument() {
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        final SearchContextId ctx2 = new SearchContextId(UUIDs.base64UUID(), 321);
+        final ShardSearchContextId ctx2 = new ShardSearchContextId(UUIDs.base64UUID(), 321);
         queryResult = new QuerySearchResult(
             ctx2, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
@@ -157,7 +157,7 @@ public void testFailFetchOneDoc() {
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        final SearchContextId ctx = new SearchContextId(UUIDs.base64UUID(), 123);
+        final ShardSearchContextId ctx = new ShardSearchContextId(UUIDs.base64UUID(), 123);
         QuerySearchResult queryResult = new QuerySearchResult(ctx,
             new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
@@ -166,7 +166,7 @@ public void testFailFetchOneDoc() {
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(new SearchContextId("", 321),
+        queryResult = new QuerySearchResult(new ShardSearchContextId("", 321),
             new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
@@ -221,7 +221,7 @@ public void testFetchDocsConcurrently() throws InterruptedException {
         ArraySearchPhaseResults<SearchPhaseResult> results = controller.newSearchPhaseResults(NOOP,
             mockSearchPhaseContext.getRequest(), numHits);
         for (int i = 0; i < numHits; i++) {
-            QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", i),
+            QuerySearchResult queryResult = new QuerySearchResult(new ShardSearchContextId("", i),
                 new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
             queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                     new ScoreDoc[] {new ScoreDoc(i+1, i)}), i), new DocValueFormat[0]);
@@ -278,7 +278,7 @@ public void testExceptionFailsPhase() {
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = randomIntBetween(2, 10);
-        QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("", 123),
+        QuerySearchResult queryResult = new QuerySearchResult(new ShardSearchContextId("", 123),
             new SearchShardTarget("node1", new ShardId("test", "na", 0),
             null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
@@ -287,7 +287,7 @@ public void testExceptionFailsPhase() {
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        queryResult = new QuerySearchResult(new SearchContextId("", 321),
+        queryResult = new QuerySearchResult(new ShardSearchContextId("", 321),
             new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                 new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]);
@@ -336,7 +336,7 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s
         ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(NOOP, mockSearchPhaseContext.getRequest(), 2);
         int resultSetSize = 1;
-        final SearchContextId ctx1 = new SearchContextId(UUIDs.base64UUID(), 123);
+        final ShardSearchContextId ctx1 = new ShardSearchContextId(UUIDs.base64UUID(), 123);
         QuerySearchResult queryResult = new QuerySearchResult(ctx1,
             new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
@@ -345,7 +345,7 @@ public void testCleanupIrrelevantContexts() { // contexts that are not fetched s
         queryResult.setShardIndex(0);
         results.consumeResult(queryResult);
 
-        final SearchContextId ctx2 = new SearchContextId(UUIDs.base64UUID(), 321);
+        final ShardSearchContextId ctx2 = new ShardSearchContextId(UUIDs.base64UUID(), 321);
         queryResult = new QuerySearchResult(ctx2,
             new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE), null);
         queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
index b0ddabe498153..49af231aeb6ce 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.transport.Transport;
 import org.junit.Assert;
@@ -50,7 +50,7 @@ public final class MockSearchPhaseContext implements SearchPhaseContext {
     final AtomicInteger numSuccess;
     final List<ShardSearchFailure> failures = Collections.synchronizedList(new ArrayList<>());
     SearchTransportService searchTransport;
-    final Set<SearchContextId> releasedSearchContexts = new HashSet<>();
+    final Set<ShardSearchContextId> releasedSearchContexts = new HashSet<>();
     final SearchRequest searchRequest = new SearchRequest();
     final AtomicReference<SearchResponse> searchResponse = new AtomicReference<>();
 
@@ -142,7 +142,7 @@ public void onFailure(Exception e) {
     }
 
     @Override
-    public void sendReleaseSearchContext(SearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) {
+    public void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) {
         releasedSearchContexts.add(contextId);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index 0c349017d7d15..f92902d631a1f 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -35,7 +35,7 @@
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
@@ -127,7 +127,7 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                     new Thread(() -> {
                         Transport.Connection connection = getConnection(null, shard.currentNodeId());
                         TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
-                            new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()),
+                            new ShardSearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()),
                         connection.getNode());
                         listener.onResponse(testSearchPhaseResult);
 
@@ -238,7 +238,7 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                         }
                         Transport.Connection connection = getConnection(null, shard.currentNodeId());
                         TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
-                            new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
+                            new ShardSearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
                         if (shardFailures[shard.shardId().id()]) {
                             listener.onFailure(new RuntimeException());
                         } else {
@@ -282,7 +282,7 @@ public void testFanOutAndCollect() throws InterruptedException {
         DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
 
-        Map<DiscoveryNode, Set<SearchContextId>> nodeToContextMap = newConcurrentMap();
+        Map<DiscoveryNode, Set<ShardSearchContextId>> nodeToContextMap = newConcurrentMap();
         AtomicInteger contextIdGenerator = new AtomicInteger(0);
         int numShards = randomIntBetween(1, 10);
         GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
@@ -291,7 +291,7 @@ public void testFanOutAndCollect() throws InterruptedException {
         AtomicInteger numFreedContext = new AtomicInteger();
         SearchTransportService transportService = new SearchTransportService(null, null) {
             @Override
-            public void sendFreeContext(Transport.Connection connection, SearchContextId contextId, OriginalIndices originalIndices) {
+            public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId, OriginalIndices originalIndices) {
                 numFreedContext.incrementAndGet();
                 assertTrue(nodeToContextMap.containsKey(connection.getNode()));
                 assertTrue(nodeToContextMap.get(connection.getNode()).remove(contextId));
@@ -332,8 +332,8 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                 assertTrue("shard: " + shard.shardId() + " has been queried twice", response.queried.add(shard.shardId()));
                 Transport.Connection connection = getConnection(null, shard.currentNodeId());
                 TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
-                    new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
-                Set<SearchContextId> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> newConcurrentSet());
+                    new ShardSearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
+                Set<ShardSearchContextId> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> newConcurrentSet());
                 ids.add(testSearchPhaseResult.getContextId());
                 if (randomBoolean()) {
                     listener.onResponse(testSearchPhaseResult);
@@ -392,7 +392,7 @@ public void testFanOutAndFail() throws InterruptedException {
         DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
 
-        Map<DiscoveryNode, Set<SearchContextId>> nodeToContextMap = newConcurrentMap();
+        Map<DiscoveryNode, Set<ShardSearchContextId>> nodeToContextMap = newConcurrentMap();
         AtomicInteger contextIdGenerator = new AtomicInteger(0);
         int numShards = randomIntBetween(2, 10);
         GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
@@ -401,7 +401,7 @@ public void testFanOutAndFail() throws InterruptedException {
         AtomicInteger numFreedContext = new AtomicInteger();
         SearchTransportService transportService = new SearchTransportService(null, null) {
             @Override
-            public void sendFreeContext(Transport.Connection connection, SearchContextId contextId, OriginalIndices originalIndices) {
+            public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId, OriginalIndices originalIndices) {
                 assertNotNull(contextId);
                 numFreedContext.incrementAndGet();
                 assertTrue(nodeToContextMap.containsKey(connection.getNode()));
@@ -446,9 +446,9 @@ protected void executePhaseOnShard(SearchShardIterator shardIt,
                     if (shard.shardId().id() == 0) {
                         testSearchPhaseResult = new TestSearchPhaseResult(null, connection.getNode());
                     } else {
-                        testSearchPhaseResult = new TestSearchPhaseResult(new SearchContextId(UUIDs.randomBase64UUID(),
+                        testSearchPhaseResult = new TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(),
                             contextIdGenerator.incrementAndGet()), connection.getNode());
-                        Set<SearchContextId> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> newConcurrentSet());
+                        Set<ShardSearchContextId> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> newConcurrentSet());
                         ids.add(testSearchPhaseResult.getContextId());
                     }
                     if (randomBoolean()) {
@@ -547,7 +547,7 @@ protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting sha
                     new Thread(() -> {
                         Transport.Connection connection = getConnection(null, shard.currentNodeId());
                         TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(
-                            new SearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
+                            new ShardSearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), connection.getNode());
                         if (shardIt.remaining() > 0) {
                             numFailReplicas.incrementAndGet();
                             listener.onFailure(new RuntimeException());
@@ -625,7 +625,7 @@ public static class TestSearchResponse extends SearchResponse {
 
     public static class TestSearchPhaseResult extends SearchPhaseResult {
         final DiscoveryNode node;
-        TestSearchPhaseResult(SearchContextId contextId, DiscoveryNode node) {
+        TestSearchPhaseResult(ShardSearchContextId contextId, DiscoveryNode node) {
             this.contextId = contextId;
             this.node = node;
         }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index 39c21428cbf7f..63c93b1a1127b 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -57,7 +57,7 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.Suggest;
@@ -256,7 +256,7 @@ private static AtomicArray<SearchPhaseResult> generateQueryResults(int nShards,
             String clusterAlias = randomBoolean() ? null : "remote";
             SearchShardTarget searchShardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex),
                 clusterAlias, OriginalIndices.NONE);
-            QuerySearchResult querySearchResult = new QuerySearchResult(new SearchContextId("", shardIndex), searchShardTarget, null);
+            QuerySearchResult querySearchResult = new QuerySearchResult(new ShardSearchContextId("", shardIndex), searchShardTarget, null);
             final TopDocs topDocs;
             float maxScore = 0;
             if (searchHitsSize == 0) {
@@ -328,7 +328,7 @@ private static AtomicArray<SearchPhaseResult> generateFetchResults(int nShards,
             float maxScore = -1F;
             String clusterAlias = randomBoolean() ? null : "remote";
             SearchShardTarget shardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex), clusterAlias, OriginalIndices.NONE);
-            FetchSearchResult fetchSearchResult = new FetchSearchResult(new SearchContextId("", shardIndex), shardTarget);
+            FetchSearchResult fetchSearchResult = new FetchSearchResult(new ShardSearchContextId("", shardIndex), shardTarget);
             List<SearchHit> searchHits = new ArrayList<>();
             for (ScoreDoc scoreDoc : mergedSearchDocs) {
                 if (scoreDoc.shardIndex == shardIndex) {
@@ -391,7 +391,7 @@ private void consumerTestCase(int numEmptyResponses) {
             numEmptyResponses --;
         }
 
-        QuerySearchResult result = new QuerySearchResult(new SearchContextId("", 0),
+        QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", 0),
             new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), null);
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
@@ -400,7 +400,7 @@ private void consumerTestCase(int numEmptyResponses) {
         result.setShardIndex(0);
         consumer.consumeResult(result);
 
-        result = new QuerySearchResult(new SearchContextId("", 1),
+        result = new QuerySearchResult(new ShardSearchContextId("", 1),
             new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), null);
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
@@ -409,7 +409,7 @@ private void consumerTestCase(int numEmptyResponses) {
         result.setShardIndex(2);
         consumer.consumeResult(result);
 
-        result = new QuerySearchResult(new SearchContextId("", 1),
+        result = new QuerySearchResult(new ShardSearchContextId("", 1),
             new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE), null);
         result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN),
             new DocValueFormat[0]);
@@ -477,7 +477,7 @@ public void testConsumerConcurrently() throws InterruptedException {
             threads[i] = new Thread(() -> {
                 int number = randomIntBetween(1, 1000);
                 max.updateAndGet(prev -> Math.max(prev, number));
-                QuerySearchResult result = new QuerySearchResult(new SearchContextId("", id),
+                QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", id),
                     new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE), null);
                 result.topDocs(new TopDocsAndMaxScore(
                     new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(0, number)}), number),
@@ -521,7 +521,7 @@ public void testConsumerOnlyAggs() {
         for (int i = 0; i < expectedNumResults; i++) {
             int number = randomIntBetween(1, 1000);
             max.updateAndGet(prev -> Math.max(prev, number));
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+            QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", i),
                 new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), number),
                     new DocValueFormat[0]);
@@ -559,7 +559,7 @@ public void testConsumerOnlyHits() {
         for (int i = 0; i < expectedNumResults; i++) {
             int number = randomIntBetween(1, 1000);
             max.updateAndGet(prev -> Math.max(prev, number));
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+            QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", i),
                 new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO),
                     new ScoreDoc[] {new ScoreDoc(0, number)}), number), new DocValueFormat[0]);
@@ -632,7 +632,7 @@ public void testReduceTopNWithFromOffset() {
             searchPhaseController.newSearchPhaseResults(NOOP, request, 4);
         int score = 100;
         for (int i = 0; i < 4; i++) {
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+            QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", i),
                 new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             ScoreDoc[] docs = new ScoreDoc[3];
             for (int j = 0; j < docs.length; j++) {
@@ -674,7 +674,7 @@ public void testConsumerSortByField() {
             max.updateAndGet(prev -> Math.max(prev, number));
             FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, new Object[]{number})};
             TopDocs topDocs = new TopFieldDocs(new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields);
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+            QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", i),
                 new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats);
             result.setShardIndex(i);
@@ -712,7 +712,7 @@ public void testConsumerFieldCollapsing() {
             Object[] values = {randomFrom(collapseValues)};
             FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, values)};
             TopDocs topDocs = new CollapseTopFieldDocs("field", new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields, values);
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+            QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", i),
                 new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats);
             result.setShardIndex(i);
@@ -745,7 +745,7 @@ public void testConsumerSuggestions() {
         int maxScorePhrase = -1;
         int maxScoreCompletion = -1;
         for (int i = 0; i < expectedNumResults; i++) {
-            QuerySearchResult result = new QuerySearchResult(new SearchContextId("", i),
+            QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", i),
                 new SearchShardTarget("node", new ShardId("a", "b", i), null, OriginalIndices.NONE), null);
             List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions =
                 new ArrayList<>();
@@ -872,7 +872,7 @@ public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, Interna
                 threads[i] = new Thread(() -> {
                     int number = randomIntBetween(1, 1000);
                     max.updateAndGet(prev -> Math.max(prev, number));
-                    QuerySearchResult result = new QuerySearchResult(new SearchContextId("", id),
+                    QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId("", id),
                         new SearchShardTarget("node", new ShardId("a", "b", id), null, OriginalIndices.NONE), null);
                     result.topDocs(new TopDocsAndMaxScore(
                             new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[]{new ScoreDoc(0, number)}), number),
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
index 4995d7d8cfaa9..1536627b60be3 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
@@ -37,7 +37,7 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.sort.SortBuilders;
@@ -92,7 +92,7 @@ public void sendExecuteQuery(Transport.Connection connection, ShardSearchRequest
                     assertNotEquals(shardId, (int) request.getBottomSortValues().getFormattedSortValues()[0]);
                     numWithTopDocs.incrementAndGet();
                 }
-                QuerySearchResult queryResult = new QuerySearchResult(new SearchContextId("N/A", 123),
+                QuerySearchResult queryResult = new QuerySearchResult(new ShardSearchContextId("N/A", 123),
                     new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null, OriginalIndices.NONE), null);
                 SortField sortField = new SortField("timestamp", SortField.Type.LONG);
                 queryResult.topDocs(new TopDocsAndMaxScore(new TopFieldDocs(
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
index cbfaf4ad4b888..0ca8c31037583 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
 
@@ -50,11 +50,11 @@ public class SearchScrollAsyncActionTests extends ESTestCase {
     public void testSendRequestsToNodes() throws InterruptedException {
 
         ParsedScrollId scrollId = getParsedScrollId(
-            new SearchContextIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 1)),
-            new SearchContextIdForNode(null, "node2", new SearchContextId(UUIDs.randomBase64UUID(), 2)),
-            new SearchContextIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 17)),
-            new SearchContextIdForNode(null, "node1", new SearchContextId(UUIDs.randomBase64UUID(), 0)),
-            new SearchContextIdForNode(null, "node3", new SearchContextId(UUIDs.randomBase64UUID(), 0)));
+            new SearchContextIdForNode(null, "node1", new ShardSearchContextId(UUIDs.randomBase64UUID(), 1)),
+            new SearchContextIdForNode(null, "node2", new ShardSearchContextId(UUIDs.randomBase64UUID(), 2)),
+            new SearchContextIdForNode(null, "node3", new ShardSearchContextId(UUIDs.randomBase64UUID(), 17)),
+            new SearchContextIdForNode(null, "node1", new ShardSearchContextId(UUIDs.randomBase64UUID(), 0)),
+            new SearchContextIdForNode(null, "node3", new ShardSearchContextId(UUIDs.randomBase64UUID(), 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -119,11 +119,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
     public void testFailNextPhase() throws InterruptedException {
 
         ParsedScrollId scrollId = getParsedScrollId(
-            new SearchContextIdForNode(null, "node1", new SearchContextId("", 1)),
-            new SearchContextIdForNode(null, "node2", new SearchContextId("a", 2)),
-            new SearchContextIdForNode(null, "node3", new SearchContextId("b", 17)),
-            new SearchContextIdForNode(null, "node1", new SearchContextId("c", 0)),
-            new SearchContextIdForNode(null, "node3", new SearchContextId("d", 0)));
+            new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 1)),
+            new SearchContextIdForNode(null, "node2", new ShardSearchContextId("a", 2)),
+            new SearchContextIdForNode(null, "node3", new ShardSearchContextId("b", 17)),
+            new SearchContextIdForNode(null, "node1", new ShardSearchContextId("c", 0)),
+            new SearchContextIdForNode(null, "node3", new ShardSearchContextId("d", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -209,11 +209,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testNodeNotAvailable() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new SearchContextIdForNode(null, "node1", new SearchContextId("", 1)),
-            new SearchContextIdForNode(null, "node2", new SearchContextId("", 2)),
-            new SearchContextIdForNode(null, "node3", new SearchContextId("", 17)),
-            new SearchContextIdForNode(null, "node1", new SearchContextId("", 0)),
-            new SearchContextIdForNode(null, "node3", new SearchContextId("", 0)));
+            new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 1)),
+            new SearchContextIdForNode(null, "node2", new ShardSearchContextId("", 2)),
+            new SearchContextIdForNode(null, "node3", new ShardSearchContextId("", 17)),
+            new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 0)),
+            new SearchContextIdForNode(null, "node3", new ShardSearchContextId("", 0)));
         // node2 is not available
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -289,11 +289,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testShardFailures() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new SearchContextIdForNode(null, "node1", new SearchContextId("", 1)),
-            new SearchContextIdForNode(null, "node2", new SearchContextId("", 2)),
-            new SearchContextIdForNode(null, "node3", new SearchContextId("",17)),
-            new SearchContextIdForNode(null, "node1", new SearchContextId("", 0)),
-            new SearchContextIdForNode(null, "node3", new SearchContextId("", 0)));
+            new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 1)),
+            new SearchContextIdForNode(null, "node2", new ShardSearchContextId("", 2)),
+            new SearchContextIdForNode(null, "node3", new ShardSearchContextId("",17)),
+            new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 0)),
+            new SearchContextIdForNode(null, "node3", new ShardSearchContextId("", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
@@ -367,11 +367,11 @@ protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearch
 
     public void testAllShardsFailed() throws InterruptedException {
         ParsedScrollId scrollId = getParsedScrollId(
-            new SearchContextIdForNode(null, "node1", new SearchContextId("", 1)),
-            new SearchContextIdForNode(null, "node2", new SearchContextId("", 2)),
-            new SearchContextIdForNode(null, "node3", new SearchContextId("", 17)),
-            new SearchContextIdForNode(null, "node1", new SearchContextId("", 0)),
-            new SearchContextIdForNode(null, "node3", new SearchContextId("", 0)));
+            new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 1)),
+            new SearchContextIdForNode(null, "node2", new ShardSearchContextId("", 2)),
+            new SearchContextIdForNode(null, "node3", new ShardSearchContextId("", 17)),
+            new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 0)),
+            new SearchContextIdForNode(null, "node3", new ShardSearchContextId("", 0)));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
             .add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT))
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
index f661cf8e7e8a7..389d8933a23dd 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
@@ -33,7 +33,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -60,7 +60,7 @@ public void testSerialization() throws Exception {
     public void testInternalScrollSearchRequestSerialization() throws IOException {
         SearchScrollRequest searchScrollRequest = createSearchScrollRequest();
         InternalScrollSearchRequest internalScrollSearchRequest =
-            new InternalScrollSearchRequest(searchScrollRequest, new SearchContextId(UUIDs.randomBase64UUID(), randomLong()));
+            new InternalScrollSearchRequest(searchScrollRequest, new ShardSearchContextId(UUIDs.randomBase64UUID(), randomLong()));
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             internalScrollSearchRequest.writeTo(output);
             try (StreamInput in = output.bytes().streamInput()) {
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
index d4de57d3cec2e..a57063cdc4ab1 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
@@ -24,10 +24,12 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 
+import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -42,13 +44,13 @@ AtomicArray<SearchPhaseResult> generateQueryResults() {
         DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("a", 1), node1);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId("a", 1), node1);
         testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), "cluster_x", null));
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("b", 12), node2);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId("b", 12), node2);
         testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), "cluster_y", null));
         SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 =
-            new SearchAsyncActionTests.TestSearchPhaseResult(new SearchContextId("c", 42), node3);
+            new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId("c", 42), node3);
         testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
         array.setOnce(0, testSearchPhaseResult1);
         array.setOnce(1, testSearchPhaseResult2);
@@ -94,23 +96,27 @@ public void testParseScrollId()  {
     public void testEncodeDecodeSearchContext() {
         final AtomicArray<SearchPhaseResult> queryResults = generateQueryResults();
         final Version version = VersionUtils.randomVersion(random());
-        final String readerId = TransportSearchHelper.encodeSearchContextId(queryResults, version);
-        final Map<ShardId, SearchContextIdForNode> contextIds = TransportSearchHelper.decodeSearchContextId(readerId);
-        assertThat(contextIds.keySet(), hasSize(3));
-
-        SearchContextIdForNode node1 = contextIds.get(new ShardId("idx", "uuid1", 2));
+        final Map<String, AliasFilter> aliasFilter = new HashMap<>();
+        for (SearchPhaseResult result : queryResults.asList()) {
+            aliasFilter.put(result.getSearchShardTarget().getShardId().getIndex().getUUID(),  AliasFilter.EMPTY);
+        }
+        final String id = SearchContextId.encode(queryResults.asList(), aliasFilter, version);
+        final SearchContextId context = SearchContextId.decode(id);
+        assertThat(context.shards().keySet(), hasSize(3));
+        assertThat(context.aliasFilter(), equalTo(aliasFilter));
+        SearchContextIdForNode node1 = context.shards().get(new ShardId("idx", "uuid1", 2));
         assertThat(node1.getClusterAlias(), equalTo("cluster_x"));
         assertThat(node1.getNode(), equalTo("node_1"));
         assertThat(node1.getSearchContextId().getId(), equalTo(1L));
         assertThat(node1.getSearchContextId().getReaderId(), equalTo("a"));
 
-        SearchContextIdForNode node2 = contextIds.get(new ShardId("idy", "uuid2", 42));
+        SearchContextIdForNode node2 = context.shards().get(new ShardId("idy", "uuid2", 42));
         assertThat(node2.getClusterAlias(), equalTo("cluster_y"));
         assertThat(node2.getNode(), equalTo("node_2"));
         assertThat(node2.getSearchContextId().getId(), equalTo(12L));
         assertThat(node2.getSearchContextId().getReaderId(), equalTo("b"));
 
-        SearchContextIdForNode node3 = contextIds.get(new ShardId("idy", "uuid2", 43));
+        SearchContextIdForNode node3 = context.shards().get(new ShardId("idy", "uuid2", 43));
         assertThat(node3.getClusterAlias(), nullValue());
         assertThat(node3.getNode(), equalTo("node_3"));
         assertThat(node3.getSearchContextId().getId(), equalTo(42L));
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 4017c9377e969..ee9ebe043760e 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -81,7 +81,7 @@
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.suggest.SuggestBuilder;
@@ -1036,7 +1036,7 @@ public void testLookUpSearchContext() throws Exception {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
         IndexShard indexShard = indexService.getShard(0);
-        List<SearchContextId> contextIds = new ArrayList<>();
+        List<ShardSearchContextId> contextIds = new ArrayList<>();
         int numContexts = randomIntBetween(1, 10);
         CountDownLatch latch = new CountDownLatch(1);
         indexShard.getThreadPool().executor(ThreadPool.Names.SEARCH).execute(() -> {
@@ -1052,17 +1052,17 @@ OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
                 }
                 assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
                 while (contextIds.isEmpty() == false) {
-                    final SearchContextId contextId = randomFrom(contextIds);
-                    assertFalse(searchService.freeReaderContext(new SearchContextId(UUIDs.randomBase64UUID(), contextId.getId())));
+                    final ShardSearchContextId contextId = randomFrom(contextIds);
+                    assertFalse(searchService.freeReaderContext(new ShardSearchContextId(UUIDs.randomBase64UUID(), contextId.getId())));
                     assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
                     if (randomBoolean()) {
                         assertTrue(searchService.freeReaderContext(contextId));
                     } else {
-                        assertTrue(searchService.freeReaderContext((new SearchContextId("", contextId.getId()))));
+                        assertTrue(searchService.freeReaderContext((new ShardSearchContextId("", contextId.getId()))));
                     }
                     contextIds.remove(contextId);
                     assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
-                    assertFalse(searchService.freeReaderContext(new SearchContextId("", contextId.getId())));
+                    assertFalse(searchService.freeReaderContext(new ShardSearchContextId("", contextId.getId())));
                     assertFalse(searchService.freeReaderContext(contextId));
                     assertThat(searchService.getActiveContexts(), equalTo(contextIds.size()));
                 }
@@ -1076,7 +1076,7 @@ OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(true),
     public void testOpenReaderContext() {
         createIndex("index");
         SearchService searchService = getInstanceFromNode(SearchService.class);
-        PlainActionFuture<SearchContextId> future = new PlainActionFuture<>();
+        PlainActionFuture<ShardSearchContextId> future = new PlainActionFuture<>();
         searchService.openReaderContext(new ShardId(resolveIndex("index"), 0), TimeValue.timeValueMinutes(between(1, 10)), future);
         future.actionGet();
         assertThat(searchService.getActiveContexts(), equalTo(1));
diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
index e560fec58171c..0eede324c9437 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
@@ -38,7 +38,7 @@
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregationsTests;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.suggest.SuggestTests;
 import org.elasticsearch.test.ESTestCase;
@@ -60,7 +60,7 @@ private static QuerySearchResult createTestInstance() throws Exception {
         SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(randomBoolean());
         ShardSearchRequest shardSearchRequest = new ShardSearchRequest(OriginalIndicesTests.randomOriginalIndices(), searchRequest,
             shardId, 1, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, randomNonNegativeLong(), null, new String[0]);
-        QuerySearchResult result = new QuerySearchResult(new SearchContextId(UUIDs.base64UUID(), randomLong()),
+        QuerySearchResult result = new QuerySearchResult(new ShardSearchContextId(UUIDs.base64UUID(), randomLong()),
             new SearchShardTarget("node", shardId, null, OriginalIndices.NONE), shardSearchRequest);
         if (randomBoolean()) {
             result.terminatedEarly(randomBoolean());
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index fa7e5ae1b9a47..d565eef6e9a22 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -52,7 +52,7 @@
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
@@ -140,8 +140,8 @@ public Query buildFilteredQuery(Query query) {
     }
 
     @Override
-    public SearchContextId id() {
-        return new SearchContextId("", 0);
+    public ShardSearchContextId id() {
+        return new ShardSearchContextId("", 0);
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
index c7a6c78bd2057..7971334669af7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.internal.ReaderContext;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
@@ -72,7 +72,7 @@ public void validateSearchContext(ReaderContext readerContext, TransportRequest
      * (or lookup) realm. To work around this we compare the username and the originating realm type.
      */
     static void ensureAuthenticatedUserIsSame(Authentication original, Authentication current, AuditTrailService auditTrailService,
-                                              SearchContextId id, String action, TransportRequest request, String requestId,
+                                              ShardSearchContextId id, String action, TransportRequest request, String requestId,
                                               AuthorizationInfo authorizationInfo) {
         // this is really a best effort attempt since we cannot guarantee principal uniqueness
         // and realm names can change between nodes.
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
index 7a380ed4cc2a0..8e8a6c69ab9be 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.search.internal.LegacyReaderContext;
 import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.SearchContextId;
+import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.transport.TransportRequest;
@@ -208,7 +208,7 @@ public void testEnsuredAuthenticatedUserIsSame() {
         Authentication original = new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
         Authentication current =
                 randomBoolean() ? original : new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null);
-        SearchContextId contextId = new SearchContextId(UUIDs.randomBase64UUID(), randomLong());
+        ShardSearchContextId contextId = new ShardSearchContextId(UUIDs.randomBase64UUID(), randomLong());
         final String action = randomAlphaOfLength(4);
         TransportRequest request = Empty.INSTANCE;
         XPackLicenseState licenseState = mock(XPackLicenseState.class);

From 29e304aef822d62aa333cc7a3006c18adbbb09eb Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 2 Jul 2020 18:53:07 -0400
Subject: [PATCH 39/47] fix doc test

---
 docs/reference/search/search_context.asciidoc |  4 +-
 .../action/search/SearchContextIdTests.java   | 70 +++++++++++++++++++
 .../search/TransportSearchHelperTests.java    | 38 +---------
 3 files changed, 73 insertions(+), 39 deletions(-)
 create mode 100644 server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java

diff --git a/docs/reference/search/search_context.asciidoc b/docs/reference/search/search_context.asciidoc
index 8867d62735873..9b50cf29d9600 100644
--- a/docs/reference/search/search_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -31,7 +31,7 @@ POST /_search <1>
         }
     },
     "search_context": {
-	    "id":  "46ToAwEHdHdpdHRlchZSa1dRZDFELVNLbTZkQVJudUtGMFFnAAAWNVAwS09JWTdTRUdibWE1ZC1id0tjQRRURnhPWjNFQmI3bmVOeVZReS1tRAAAAAAAAAAB", <2>
+	    "id":  "y_auAwMDaWR4BXV1aWQxAgEJY2x1c3Rlcl94Bm5vZGVfMQFhAAAAAAAAAAEDaWR5BXV1aWQyKgEJY2x1c3Rlcl95Bm5vZGVfMgFiAAAAAAAAAAwDaWR5BXV1aWQyKwAGbm9kZV8zAWMAAAAAAAAAKgEFdXVpZDIAAA==", <2>
 	    "keep_alive": "1m"  <3>
     }
 }
@@ -90,7 +90,7 @@ explicitly closed as soon as they are no longer used in search requests.
 ---------------------------------------
 DELETE /_search_context
 {
-    "id" : "46ToAwEHdHdpdHRlchZSa1dRZDFELVNLbTZkQVJudUtGMFFnAAAWNVAwS09JWTdTRUdibWE1ZC1id0tjQRRURnhPWjNFQmI3bmVOeVZReS1tRAAAAAAAAAAB"
+    "id" : "46ToAwMDaWR4BXV1aWQxAgEJY2x1c3Rlcl94Bm5vZGVfMQFhAAAAAAAAAAEDaWR5BXV1aWQyKgEJY2x1c3Rlcl95Bm5vZGVfMgFiAAAAAAAAAAwDaWR5BXV1aWQyKwAGbm9kZV8zAWMAAAAAAAAAKgIFdXVpZDIAAAV1dWlkMQAA"
 }
 ---------------------------------------
 // TEST[catch:missing]
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java
new file mode 100644
index 0000000000000..cadc9950b9b8d
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.nullValue;
+
+public class SearchContextIdTests extends ESTestCase {
+
+    public void testEncode() {
+        final AtomicArray<SearchPhaseResult> queryResults = TransportSearchHelperTests.generateQueryResults();
+        final Version version = VersionUtils.randomVersion(random());
+        final Map<String, AliasFilter> aliasFilter = new HashMap<>();
+        for (SearchPhaseResult result : queryResults.asList()) {
+            if (randomBoolean()) {
+                aliasFilter.put(result.getSearchShardTarget().getShardId().getIndex().getUUID(),  AliasFilter.EMPTY);
+            }
+        }
+        final String id = SearchContextId.encode(queryResults.asList(), aliasFilter, version);
+        final SearchContextId context = SearchContextId.decode(id);
+        assertThat(context.shards().keySet(), hasSize(3));
+        assertThat(context.aliasFilter(), equalTo(aliasFilter));
+        SearchContextIdForNode node1 = context.shards().get(new ShardId("idx", "uuid1", 2));
+        assertThat(node1.getClusterAlias(), equalTo("cluster_x"));
+        assertThat(node1.getNode(), equalTo("node_1"));
+        assertThat(node1.getSearchContextId().getId(), equalTo(1L));
+        assertThat(node1.getSearchContextId().getReaderId(), equalTo("a"));
+
+        SearchContextIdForNode node2 = context.shards().get(new ShardId("idy", "uuid2", 42));
+        assertThat(node2.getClusterAlias(), equalTo("cluster_y"));
+        assertThat(node2.getNode(), equalTo("node_2"));
+        assertThat(node2.getSearchContextId().getId(), equalTo(12L));
+        assertThat(node2.getSearchContextId().getReaderId(), equalTo("b"));
+
+        SearchContextIdForNode node3 = context.shards().get(new ShardId("idy", "uuid2", 43));
+        assertThat(node3.getClusterAlias(), nullValue());
+        assertThat(node3.getNode(), equalTo("node_3"));
+        assertThat(node3.getSearchContextId().getId(), equalTo(42L));
+        assertThat(node3.getSearchContextId().getReaderId(), equalTo("c"));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
index a57063cdc4ab1..a4ae931d925b5 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java
@@ -24,21 +24,15 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 
-import java.util.HashMap;
-import java.util.Map;
-
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.nullValue;
 
 public class TransportSearchHelperTests extends ESTestCase {
 
-    AtomicArray<SearchPhaseResult> generateQueryResults() {
+    public static AtomicArray<SearchPhaseResult> generateQueryResults() {
         AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
         DiscoveryNode node1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
@@ -92,34 +86,4 @@ public void testParseScrollId()  {
             assertThat(parseScrollId.getContext()[2].getSearchContextId().getReaderId(), equalTo(""));
         }
     }
-
-    public void testEncodeDecodeSearchContext() {
-        final AtomicArray<SearchPhaseResult> queryResults = generateQueryResults();
-        final Version version = VersionUtils.randomVersion(random());
-        final Map<String, AliasFilter> aliasFilter = new HashMap<>();
-        for (SearchPhaseResult result : queryResults.asList()) {
-            aliasFilter.put(result.getSearchShardTarget().getShardId().getIndex().getUUID(),  AliasFilter.EMPTY);
-        }
-        final String id = SearchContextId.encode(queryResults.asList(), aliasFilter, version);
-        final SearchContextId context = SearchContextId.decode(id);
-        assertThat(context.shards().keySet(), hasSize(3));
-        assertThat(context.aliasFilter(), equalTo(aliasFilter));
-        SearchContextIdForNode node1 = context.shards().get(new ShardId("idx", "uuid1", 2));
-        assertThat(node1.getClusterAlias(), equalTo("cluster_x"));
-        assertThat(node1.getNode(), equalTo("node_1"));
-        assertThat(node1.getSearchContextId().getId(), equalTo(1L));
-        assertThat(node1.getSearchContextId().getReaderId(), equalTo("a"));
-
-        SearchContextIdForNode node2 = context.shards().get(new ShardId("idy", "uuid2", 42));
-        assertThat(node2.getClusterAlias(), equalTo("cluster_y"));
-        assertThat(node2.getNode(), equalTo("node_2"));
-        assertThat(node2.getSearchContextId().getId(), equalTo(12L));
-        assertThat(node2.getSearchContextId().getReaderId(), equalTo("b"));
-
-        SearchContextIdForNode node3 = context.shards().get(new ShardId("idy", "uuid2", 43));
-        assertThat(node3.getClusterAlias(), nullValue());
-        assertThat(node3.getNode(), equalTo("node_3"));
-        assertThat(node3.getSearchContextId().getId(), equalTo(42L));
-        assertThat(node3.getSearchContextId().getReaderId(), equalTo("c"));
-    }
 }

From 1d9bb74f0a303313887b66b6f751c2bc9bd8ce87 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 2 Jul 2020 20:34:48 -0400
Subject: [PATCH 40/47] Add response sample for close api

---
 docs/reference/search/search_context.asciidoc | 21 ++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/docs/reference/search/search_context.asciidoc b/docs/reference/search/search_context.asciidoc
index 9b50cf29d9600..6f49fc203a5f2 100644
--- a/docs/reference/search/search_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -83,14 +83,29 @@ GET /_nodes/stats/indices/search
 
 Search contexts are automatically closed when the `keep_alive` has
 been elapsed. However keeping search contexts has a cost, as discussed in the
-<<search-context-keep-alive,previous section>> so search contexts should be
-explicitly closed as soon as they are no longer used in search requests.
+<<search-context-keep-alive,previous section>>. Search contexts should be closed
+as soon as they are no longer used in search requests.
 
 [source,console]
 ---------------------------------------
 DELETE /_search_context
 {
-    "id" : "46ToAwMDaWR4BXV1aWQxAgEJY2x1c3Rlcl94Bm5vZGVfMQFhAAAAAAAAAAEDaWR5BXV1aWQyKgEJY2x1c3Rlcl95Bm5vZGVfMgFiAAAAAAAAAAwDaWR5BXV1aWQyKwAGbm9kZV8zAWMAAAAAAAAAKgIFdXVpZDIAAAV1dWlkMQAA"
+    "id" : "x9mtAwMDaWR4BXV1aWQxAgAGbm9kZV8xAWEAAAAAAAAAAQNpZHkFdXVpZDIqAAZub2RlXzIBYgAAAAAAAAAMA2lkeQV1dWlkMisABm5vZGVfMwFjAAAAAAAAACoBBXV1aWQyAAA="
 }
 ---------------------------------------
 // TEST[catch:missing]
+
+The API returns the following response:
+
+[source,console-result]
+--------------------------------------------------
+{
+   "succeeded": true, <1>
+   "num_freed": 3     <2>
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"succeeded": true/"succeeded": $body.succeeded/]
+// TESTRESPONSE[s/"num_freed": 3/"num_freed": $body.num_freed/]
+
+<1> If true, all search contexts associated with the given id are successfully closed
+<2> The number of search contexts have been successfully closed

From 52902017590b79a3325c66eef3b9e1b47467a9b9 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 2 Jul 2020 23:42:12 -0400
Subject: [PATCH 41/47] support wildcard

---
 .../test/search/330_search_context.yml        | 48 ++++++++++++++++++-
 .../search/OpenSearchContextRequest.java      | 10 +++-
 2 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml
index 253678dda9db2..0f22a886e4db9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml
@@ -20,9 +20,19 @@ setup:
             id:     172
             body:   { id: 172, foo: bar, age: 24 }
 
+  - do:
+      indices.create:
+        index:  test2
+
+  - do:
+      index:
+        index:  test2
+        id:     45
+        body:   { id: 45, foo: bar, age: 19 }
+
   - do:
       indices.refresh:
-        index: test
+        index: "test*"
 
 ---
 "search context":
@@ -125,3 +135,39 @@ setup:
       close_search_context:
         body:
           id: "$search_context_id"
+
+---
+"Search context with wildcard":
+  - skip:
+      version: " - 7.9.99"
+      reason: "search context is introduced in 8.0"
+  - do:
+      open_search_context:
+        index: "t*"
+        keep_alive: 5m
+  - set: {id: search_context_id}
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          size: 2
+          query:
+            match:
+              foo: bar
+          sort: [{ age: desc }, { id: desc }]
+          search_context:
+            id: "$search_context_id"
+            keep_alive: 1m
+
+  - match: {hits.total: 4 }
+  - length: {hits.hits: 2 }
+  - match: {hits.hits.0._index: test }
+  - match: {hits.hits.0._id: "172" }
+  - match: {hits.hits.1._index: test2 }
+  - match: {hits.hits.1._id: "45" }
+
+  - do:
+      close_search_context:
+        body:
+          id: "$search_context_id"
diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenSearchContextRequest.java b/server/src/main/java/org/elasticsearch/action/search/OpenSearchContextRequest.java
index 5134568c8158a..dea0fbf981332 100644
--- a/server/src/main/java/org/elasticsearch/action/search/OpenSearchContextRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/OpenSearchContextRequest.java
@@ -36,8 +36,8 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public final class OpenSearchContextRequest extends ActionRequest implements IndicesRequest {
-    private final String[] indices;
+public final class OpenSearchContextRequest extends ActionRequest implements IndicesRequest.Replaceable {
+    private String[] indices;
     private final IndicesOptions indicesOptions;
     private final TimeValue keepAlive;
 
@@ -93,6 +93,12 @@ public String[] indices() {
         return indices;
     }
 
+    @Override
+    public OpenSearchContextRequest indices(String... indices) {
+        this.indices = indices;
+        return this;
+    }
+
     @Override
     public IndicesOptions indicesOptions() {
         return indicesOptions;

From 4a38f0d097b1a636021eaf5e1b6b86a92ae8aa1e Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Fri, 3 Jul 2020 23:07:02 -0400
Subject: [PATCH 42/47] Fix AliasFilter serialize

---
 .../search/ClearSearchContextController.java  |  6 +-
 .../action/search/SearchContextId.java        | 62 ++++++-------------
 .../action/search/SearchContextIdForNode.java | 20 +++++-
 .../search/TransportClearScrollAction.java    |  5 +-
 .../TransportCloseSearchContextAction.java    |  8 ++-
 .../action/search/TransportSearchAction.java  |  7 ++-
 .../action/search/SearchContextIdTests.java   | 42 ++++++++++---
 .../snapshots/SnapshotResiliencyTests.java    |  2 +-
 8 files changed, 93 insertions(+), 59 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java b/server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java
index 5733ddceff1f2..cc3a7f2002a77 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearSearchContextController.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportResponse;
@@ -69,12 +70,13 @@ final class ClearSearchContextController implements Runnable {
     }
 
     ClearSearchContextController(CloseSearchContextRequest closeSearchContextRequest, ActionListener<CloseSearchContextResponse> listener,
-                                 DiscoveryNodes nodes, Logger logger, SearchTransportService searchTransportService) {
+                                 DiscoveryNodes nodes, Logger logger, SearchTransportService searchTransportService,
+                                 NamedWriteableRegistry namedWriteableRegistry) {
         this.nodes = nodes;
         this.logger = logger;
         this.searchTransportService = searchTransportService;
         this.listener = listener;
-        final SearchContextId context = SearchContextId.decode(closeSearchContextRequest.getId());
+        final SearchContextId context = SearchContextId.decode(namedWriteableRegistry, closeSearchContextRequest.getId());
         expectedOps = new CountDown(context.shards().size());
         runner = () -> cleanReaderIds(context.shards().values());
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java
index 2250f5415918e..402a9de76e15d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java
@@ -23,14 +23,14 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.ShardSearchContextId;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -58,63 +58,39 @@ public Map<String, AliasFilter> aliasFilter() {
     }
 
     static String encode(List<SearchPhaseResult> searchPhaseResults, Map<String, AliasFilter> aliasFilter, Version version) {
+        final Map<ShardId, SearchContextIdForNode> shards = new HashMap<>();
+        for (SearchPhaseResult searchPhaseResult : searchPhaseResults) {
+            final SearchShardTarget target = searchPhaseResult.getSearchShardTarget();
+            shards.put(target.getShardId(),
+                new SearchContextIdForNode(target.getClusterAlias(), target.getNodeId(), searchPhaseResult.getContextId()));
+        }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
-            out.writeVInt(version.id);
-            out.writeVInt(searchPhaseResults.size());
-            for (SearchPhaseResult searchPhaseResult : searchPhaseResults) {
-                SearchShardTarget target = searchPhaseResult.getSearchShardTarget();
-                ShardId shardId = target.getShardId();
-                // shardId
-                out.writeString(shardId.getIndex().getName());
-                out.writeString(shardId.getIndex().getUUID());
-                out.writeVInt(shardId.getId());
-                // nodeId
-                if (target.getClusterAlias() != null) {
-                    out.writeByte((byte) 1);
-                    out.writeString(target.getClusterAlias());
-                } else {
-                    out.writeByte((byte) 0);
-                }
-                out.writeString(target.getNodeId());
-                // readerId
-                out.writeString(searchPhaseResult.getContextId().getReaderId());
-                out.writeLong(searchPhaseResult.getContextId().getId());
-            }
-            out.writeMap(aliasFilter, StreamOutput::writeString, (outstream, val) -> val.writeTo(outstream));
+            out.setVersion(version);
+            Version.writeVersion(version, out);
+            out.writeMap(shards, (o, k) -> k.writeTo(o), (o, v) -> v.writeTo(o));
+            out.writeMap(aliasFilter, StreamOutput::writeString, (o, v) -> v.writeTo(o));
             return Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(out.bytes()));
         } catch (IOException e) {
             throw new IllegalArgumentException(e);
         }
     }
 
-    static SearchContextId decode(String id) {
+    static SearchContextId decode(NamedWriteableRegistry namedWriteableRegistry, String id) {
         final ByteBuffer byteBuffer;
         try {
             byteBuffer = ByteBuffer.wrap(Base64.getUrlDecoder().decode(id));
         } catch (Exception e) {
             throw new IllegalArgumentException("invalid id: [" + id + "]", e);
         }
-        try (StreamInput in = new ByteBufferStreamInput(byteBuffer)) {
-            Version version = Version.fromId(in.readVInt());
-            final int numOfContexts = in.readVInt();
-            final Map<ShardId, SearchContextIdForNode> shards = new HashMap<>();
-            for (int i = 0; i < numOfContexts; i++) {
-                final ShardId shardId = new ShardId(new Index(in.readString(), in.readString()), in.readVInt());
-                final String clusterAlias;
-                if (in.readByte() == 1) {
-                    clusterAlias = in.readString();
-                } else {
-                    clusterAlias = null;
-                }
-                final String nodeId = in.readString();
-                final ShardSearchContextId contextId = new ShardSearchContextId(in.readString(), in.readLong());
-                shards.put(shardId, new SearchContextIdForNode(clusterAlias, nodeId, contextId));
-            }
-            Map<String, AliasFilter> aliasFilterMap = in.readMap(StreamInput::readString, AliasFilter::new);
+        try (StreamInput in = new NamedWriteableAwareStreamInput(new ByteBufferStreamInput(byteBuffer), namedWriteableRegistry)) {
+            final Version version = Version.readVersion(in);
+            in.setVersion(version);
+            final Map<ShardId, SearchContextIdForNode> shards = in.readMap(ShardId::new, SearchContextIdForNode::new);
+            final Map<String, AliasFilter> aliasFilters = in.readMap(StreamInput::readString, AliasFilter::new);
             if (in.available() > 0) {
                 throw new IllegalArgumentException("Not all bytes were read");
             }
-            return new SearchContextId(Collections.unmodifiableMap(shards), Collections.unmodifiableMap(aliasFilterMap));
+            return new SearchContextId(Collections.unmodifiableMap(shards), Collections.unmodifiableMap(aliasFilters));
         } catch (IOException e) {
             throw new IllegalArgumentException(e);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
index 6f19d056ccc86..2c298c66339f5 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
@@ -20,9 +20,14 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.search.internal.ShardSearchContextId;
 
-final class SearchContextIdForNode {
+import java.io.IOException;
+
+final class SearchContextIdForNode implements Writeable {
     private final String node;
     private final ShardSearchContextId searchContextId;
     private final String clusterAlias;
@@ -33,6 +38,19 @@ final class SearchContextIdForNode {
         this.searchContextId = searchContextId;
     }
 
+    SearchContextIdForNode(StreamInput in) throws IOException {
+        this.node = in.readString();
+        this.clusterAlias = in.readOptionalString();
+        this.searchContextId = new ShardSearchContextId(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(node);
+        out.writeOptionalString(clusterAlias);
+        searchContextId.writeTo(out);
+    }
+
     public String getNode() {
         return node;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
index 016f6bca0a828..d7734d05939d1 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
@@ -31,13 +32,15 @@ public class TransportClearScrollAction extends HandledTransportAction<ClearScro
 
     private final ClusterService clusterService;
     private final SearchTransportService searchTransportService;
+    private final NamedWriteableRegistry namedWriteableRegistry;
 
     @Inject
     public TransportClearScrollAction(TransportService transportService, ClusterService clusterService, ActionFilters actionFilters,
-                                      SearchTransportService searchTransportService) {
+                                      SearchTransportService searchTransportService, NamedWriteableRegistry namedWriteableRegistry) {
         super(ClearScrollAction.NAME, transportService, actionFilters, ClearScrollRequest::new);
         this.clusterService = clusterService;
         this.searchTransportService = searchTransportService;
+        this.namedWriteableRegistry = namedWriteableRegistry;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java
index a668cf61bb0ea..6b7f149796fab 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportCloseSearchContextAction.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
@@ -31,19 +32,22 @@ public class TransportCloseSearchContextAction extends HandledTransportAction<Cl
 
     private final ClusterService clusterService;
     private final SearchTransportService searchTransportService;
+    private final NamedWriteableRegistry namedWriteableRegistry;
 
     @Inject
     public TransportCloseSearchContextAction(TransportService transportService, ClusterService clusterService,
-                                             ActionFilters actionFilters, SearchTransportService searchTransportService) {
+                                             ActionFilters actionFilters, SearchTransportService searchTransportService,
+                                             NamedWriteableRegistry namedWriteableRegistry) {
         super(CloseSearchContextAction.NAME, transportService, actionFilters, CloseSearchContextRequest::new);
         this.clusterService = clusterService;
         this.searchTransportService = searchTransportService;
+        this.namedWriteableRegistry = namedWriteableRegistry;
     }
 
     @Override
     protected void doExecute(Task task, CloseSearchContextRequest request, ActionListener<CloseSearchContextResponse> listener) {
         Runnable runnable = new ClearSearchContextController(
-            request, listener, clusterService.state().nodes(), logger, searchTransportService);
+            request, listener, clusterService.state().nodes(), logger, searchTransportService, namedWriteableRegistry);
         runnable.run();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 009dd03231ff8..4d5abb471be16 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -103,12 +104,13 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
     private final SearchPhaseController searchPhaseController;
     private final SearchService searchService;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
+    private final NamedWriteableRegistry namedWriteableRegistry;
 
     @Inject
     public TransportSearchAction(ThreadPool threadPool, TransportService transportService, SearchService searchService,
                                  SearchTransportService searchTransportService, SearchPhaseController searchPhaseController,
                                  ClusterService clusterService, ActionFilters actionFilters,
-                                 IndexNameExpressionResolver indexNameExpressionResolver) {
+                                 IndexNameExpressionResolver indexNameExpressionResolver, NamedWriteableRegistry namedWriteableRegistry) {
         super(SearchAction.NAME, transportService, actionFilters, (Writeable.Reader<SearchRequest>) SearchRequest::new);
         this.threadPool = threadPool;
         this.searchPhaseController = searchPhaseController;
@@ -118,6 +120,7 @@ public TransportSearchAction(ThreadPool threadPool, TransportService transportSe
         this.clusterService = clusterService;
         this.searchService = searchService;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
+        this.namedWriteableRegistry = namedWriteableRegistry;
     }
 
     private Map<String, AliasFilter> buildPerIndexAliasFilter(SearchRequest request, ClusterState clusterState,
@@ -218,7 +221,7 @@ void executeRequest(Task task, SearchRequest searchRequest,
             final SearchContextId searchContext;
             final Map<String, OriginalIndices> remoteClusterIndices;
             if (searchRequest.searchContextBuilder() != null) {
-                searchContext = SearchContextId.decode(searchRequest.searchContextBuilder().getId());
+                searchContext = SearchContextId.decode(namedWriteableRegistry, searchRequest.searchContextBuilder().getId());
                 remoteClusterIndices = getIndicesFromSearchContexts(searchContext, searchRequest.indicesOptions());
             } else {
                 searchContext = null;
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java
index cadc9950b9b8d..00d65f34ed11e 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java
@@ -20,14 +20,19 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.query.IdsQueryBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -36,19 +41,42 @@
 
 public class SearchContextIdTests extends ESTestCase {
 
+    QueryBuilder randomQueryBuilder() {
+        if (randomBoolean()) {
+            return new TermQueryBuilder(randomAlphaOfLength(10), randomAlphaOfLength(10));
+        } else if (randomBoolean()) {
+            return new MatchAllQueryBuilder();
+        } else {
+            return new IdsQueryBuilder().addIds(randomAlphaOfLength(10));
+        }
+    }
+
     public void testEncode() {
+        final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(List.of(
+            new NamedWriteableRegistry.Entry(QueryBuilder.class, TermQueryBuilder.NAME, TermQueryBuilder::new),
+            new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new),
+            new NamedWriteableRegistry.Entry(QueryBuilder.class, IdsQueryBuilder.NAME, IdsQueryBuilder::new)
+        ));
         final AtomicArray<SearchPhaseResult> queryResults = TransportSearchHelperTests.generateQueryResults();
-        final Version version = VersionUtils.randomVersion(random());
-        final Map<String, AliasFilter> aliasFilter = new HashMap<>();
+        final Version version = Version.CURRENT;
+        final Map<String, AliasFilter> aliasFilters = new HashMap<>();
         for (SearchPhaseResult result : queryResults.asList()) {
+            final AliasFilter aliasFilter;
+            if (randomBoolean()) {
+                aliasFilter = new AliasFilter(randomQueryBuilder());
+            } else if (randomBoolean()) {
+                aliasFilter = new AliasFilter(randomQueryBuilder(), "alias-" + between(1, 10));
+            } else {
+                aliasFilter = AliasFilter.EMPTY;
+            }
             if (randomBoolean()) {
-                aliasFilter.put(result.getSearchShardTarget().getShardId().getIndex().getUUID(),  AliasFilter.EMPTY);
+                aliasFilters.put(result.getSearchShardTarget().getShardId().getIndex().getUUID(), aliasFilter);
             }
         }
-        final String id = SearchContextId.encode(queryResults.asList(), aliasFilter, version);
-        final SearchContextId context = SearchContextId.decode(id);
+        final String id = SearchContextId.encode(queryResults.asList(), aliasFilters, version);
+        final SearchContextId context = SearchContextId.decode(namedWriteableRegistry, id);
         assertThat(context.shards().keySet(), hasSize(3));
-        assertThat(context.aliasFilter(), equalTo(aliasFilter));
+        assertThat(context.aliasFilter(), equalTo(aliasFilters));
         SearchContextIdForNode node1 = context.shards().get(new ShardId("idx", "uuid1", 2));
         assertThat(node1.getClusterAlias(), equalTo("cluster_x"));
         assertThat(node1.getNode(), equalTo("node_1"));
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index 006f153b2afde..49990b452222b 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -1531,7 +1531,7 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi
                 actions.put(SearchAction.INSTANCE,
                     new TransportSearchAction(threadPool, transportService, searchService,
                         searchTransportService, searchPhaseController, clusterService,
-                        actionFilters, indexNameExpressionResolver));
+                        actionFilters, indexNameExpressionResolver, namedWriteableRegistry));
                 actions.put(RestoreSnapshotAction.INSTANCE,
                     new TransportRestoreSnapshotAction(transportService, clusterService, threadPool, restoreService, actionFilters,
                         indexNameExpressionResolver));

From c77d3ca744c55fcb6fc200979fb7f31e65819457 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sat, 4 Jul 2020 16:11:30 -0400
Subject: [PATCH 43/47] fix docs

---
 docs/reference/search/search_context.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/search/search_context.asciidoc b/docs/reference/search/search_context.asciidoc
index 6f49fc203a5f2..f29bae180e825 100644
--- a/docs/reference/search/search_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -31,7 +31,7 @@ POST /_search <1>
         }
     },
     "search_context": {
-	    "id":  "y_auAwMDaWR4BXV1aWQxAgEJY2x1c3Rlcl94Bm5vZGVfMQFhAAAAAAAAAAEDaWR5BXV1aWQyKgEJY2x1c3Rlcl95Bm5vZGVfMgFiAAAAAAAAAAwDaWR5BXV1aWQyKwAGbm9kZV8zAWMAAAAAAAAAKgEFdXVpZDIAAA==", <2>
+	    "id":  "46ToAwMDaWR4BXV1aWQxAgZub2RlXzEAAAAAAAAAAAEBYQNpZHkFdXVpZDIrBm5vZGVfMwAAAAAAAAAAKgFjA2lkeQV1dWlkMioGbm9kZV8yAAAAAAAAAAAMAWICBXV1aWQyAAAFdXVpZDEAAQltYXRjaF9hbGw_gAAAAA==", <2>
 	    "keep_alive": "1m"  <3>
     }
 }
@@ -90,7 +90,7 @@ as soon as they are no longer used in search requests.
 ---------------------------------------
 DELETE /_search_context
 {
-    "id" : "x9mtAwMDaWR4BXV1aWQxAgAGbm9kZV8xAWEAAAAAAAAAAQNpZHkFdXVpZDIqAAZub2RlXzIBYgAAAAAAAAAMA2lkeQV1dWlkMisABm5vZGVfMwFjAAAAAAAAACoBBXV1aWQyAAA="
+    "id" : "46ToAwMDaWR4BXV1aWQxAgZub2RlXzEAAAAAAAAAAAEBYQNpZHkFdXVpZDIrBm5vZGVfMwAAAAAAAAAAKgFjA2lkeQV1dWlkMioGbm9kZV8yAAAAAAAAAAAMAWIBBXV1aWQyAAA="
 }
 ---------------------------------------
 // TEST[catch:missing]

From e2a7fefa3fc08c40c31cddd9dec02d42f35c2362 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sun, 5 Jul 2020 15:51:38 -0400
Subject: [PATCH 44/47] explain refresh in search context

---
 docs/reference/search/search_context.asciidoc | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/docs/reference/search/search_context.asciidoc b/docs/reference/search/search_context.asciidoc
index f29bae180e825..7b51d22a8dc59 100644
--- a/docs/reference/search/search_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -1,15 +1,16 @@
 [[search-context]]
 ==== Search Context
 
-By default, a search request executes against the latest point in time readers of the
-target indices. Sometimes it's more preferred to execute multiple search requests using
-the same point in time readers. For example, the combined result of the initial and
-subsequent search_after requests would be more consistent if they use the same point
-in time readers. This can be done by using a single search context for all search requests.
-
-A search context must be opened in a separate step before being used in subsequent
-search requests. The keep_alive parameter tells Elasticsearch how long it should keep
-the seach context alive, e.g. `?keep_alive=5m`.
+A search request by default executes using the most recent searchers of the target indices.
+In some cases, it's preferred to perform multiple search requests using the same searchers.
+For example, if <<indices-refresh,refreshes>> happen between search_after requests,
+then the results of those requests might not be consistent as changes happening between
+searches are only visible to the more recent searchers. Search context is introduced to
+allow performing multiple search requests using the same searchers.
+
+A search context must be opened explicitly before being used in search requests. The
+keep_alive parameter tells Elasticsearch how long it should keep the seach context alive,
+e.g. `?keep_alive=5m`.
 
 [source,console]
 --------------------------------------------------

From ada43b3f9cd27033143d702d6a833a594bbc7e78 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 6 Jul 2020 21:47:24 -0400
Subject: [PATCH 45/47] rewording search context

---
 docs/reference/search/search_context.asciidoc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/reference/search/search_context.asciidoc b/docs/reference/search/search_context.asciidoc
index 7b51d22a8dc59..08ed632fe54b5 100644
--- a/docs/reference/search/search_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -1,12 +1,12 @@
 [[search-context]]
 ==== Search Context
 
-A search request by default executes using the most recent searchers of the target indices.
-In some cases, it's preferred to perform multiple search requests using the same searchers.
+A search request by default executes against the most recent point in time view
+of the target indices, which is called a search context. In some cases, it's
+preferred to perform multiple search requests using the same search context.
 For example, if <<indices-refresh,refreshes>> happen between search_after requests,
 then the results of those requests might not be consistent as changes happening between
-searches are only visible to the more recent searchers. Search context is introduced to
-allow performing multiple search requests using the same searchers.
+searches are only visible to the more recent search context.
 
 A search context must be opened explicitly before being used in search requests. The
 keep_alive parameter tells Elasticsearch how long it should keep the seach context alive,

From 2f867dcc43193fd76457ff1b7d67fca84c15f711 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sun, 19 Jul 2020 22:12:51 -0400
Subject: [PATCH 46/47] apply doc suggestion

---
 docs/reference/search/search_context.asciidoc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/reference/search/search_context.asciidoc b/docs/reference/search/search_context.asciidoc
index 08ed632fe54b5..ad0523743c5c0 100644
--- a/docs/reference/search/search_context.asciidoc
+++ b/docs/reference/search/search_context.asciidoc
@@ -40,7 +40,8 @@ POST /_search <1>
 // TEST[catch:missing]
 
 <1> A search request with `search_context` must not specify `index`, `routing`,
-and `preferences` as these parameters are copied from the `search_context`.
+and {ref}/search-request-body.html#request-body-search-preference[`preference`]
+as these parameters are copied from the `search_context`.
 <2> The `id` parameter tells Elasticsearch to execute the request using
 the point-in-time readers from this search context id.
 <3> The `keep_alive` parameter tells Elasticsearch how long it should extend
@@ -59,7 +60,7 @@ process all data -- it just needs to be long enough for the next request.
 
 Normally, the background merge process optimizes the index by merging together
 smaller segments to create new, bigger segments. Once the smaller segments are
-no longer needed they are deleted. However, open search contexts prevents the
+no longer needed they are deleted. However, open search contexts prevent the
 old segments from being deleted since they are still in use.
 
 TIP: Keeping older segments alive means that more disk space and file handles

From 840ca7cfc56ab4e3802df0cd7334ff2d038438eb Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Wed, 22 Jul 2020 17:35:20 -0400
Subject: [PATCH 47/47] bump bwc version

---
 .../rest-api-spec/test/search/330_search_context.yml          | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml
index 0f22a886e4db9..cc7d5f4dce7b0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_search_context.yml
@@ -37,7 +37,7 @@ setup:
 ---
 "search context":
   - skip:
-      version: " - 7.9.99"
+      version: " - 7.99.99"
       reason: "search context is introduced in 8.0"
   - do:
       open_search_context:
@@ -139,7 +139,7 @@ setup:
 ---
 "Search context with wildcard":
   - skip:
-      version: " - 7.9.99"
+      version: " - 7.99.99"
       reason: "search context is introduced in 8.0"
   - do:
       open_search_context: