Prevent throttled indices to be searched through wildcards by default (…

…#34354) Today if a wildcard, date-math expression or alias expands/resolves to an index that is search-throttled we still search it. This is likely not the desired behavior since it can unexpectedly slow down searches significantly. This change adds a new indices option that allows `search`, `count` and `msearch` to ignore throttled indices by default. Users can force expansion to throttled indices by using `ignore_throttled=true` on the rest request to expand also to throttled indices. Relates to #34352
elastic · Nov 6, 2018 · 45e16b5 · 45e16b5
1 parent dc313c8
commit 45e16b5
Show file tree

Hide file tree

Showing 28 changed files with 281 additions and 98 deletions.
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
@@ -109,9 +109,8 @@ public void testRankEvalRequest() throws IOException {
 
         // now try this when test2 is closed
         client().performRequest(new Request("POST", "index2/_close"));
-        rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS));
-        response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync,
-                highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
+        rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
+        response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
     }
 
     private static List<RatedRequest> createTestEvaluationSpec() {

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -1067,7 +1067,8 @@ public void testMultiSearch() throws IOException {
             IndicesOptions msearchDefault = new MultiSearchRequest().indicesOptions();
             searchRequest.indicesOptions(IndicesOptions.fromOptions(randomlyGenerated.ignoreUnavailable(),
                     randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(), randomlyGenerated.expandWildcardsClosed(),
-                    msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases()));
+                    msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(),
+                msearchDefault.ignoreThrottled()));
             multiSearchRequest.add(searchRequest);
         }
 

diff --git a/...l/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java b/...l/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java
@@ -38,7 +38,7 @@ private ExplainLifecycleRequest createTestInstance() {
         }
         if (randomBoolean()) {
             IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
-                    randomBoolean(), randomBoolean(), randomBoolean());
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
             request.indicesOptions(indicesOptions);
         }
         return request;
@@ -54,7 +54,7 @@ private ExplainLifecycleRequest mutateInstance(ExplainLifecycleRequest instance)
             break;
         case 1:
             indicesOptions = randomValueOtherThan(indicesOptions, () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(),
-                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
             break;
         default:
             throw new AssertionError("Illegal randomisation branch");

diff --git a/.../java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequestTests.java b/.../java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequestTests.java
@@ -46,7 +46,7 @@ protected RemoveIndexLifecyclePolicyRequest createInstance() {
         if (randomBoolean()) {
             return new RemoveIndexLifecyclePolicyRequest(Arrays.asList(generateRandomStringArray(20, 20, false)),
                     IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(),
-                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
         } else {
             return new RemoveIndexLifecyclePolicyRequest(Arrays.asList(generateRandomStringArray(20, 20, false)));
         }
@@ -57,14 +57,14 @@ private RemoveIndexLifecyclePolicyRequest copyInstance(RemoveIndexLifecyclePolic
                 req.indicesOptions().ignoreUnavailable(), req.indicesOptions().allowNoIndices(),
                 req.indicesOptions().expandWildcardsOpen(), req.indicesOptions().expandWildcardsClosed(),
                 req.indicesOptions().allowAliasesToMultipleIndices(), req.indicesOptions().forbidClosedIndices(),
-                req.indicesOptions().ignoreAliases()));
+                req.indicesOptions().ignoreAliases(), req.indicesOptions().ignoreThrottled()));
     }
 
     private RemoveIndexLifecyclePolicyRequest mutateInstance(RemoveIndexLifecyclePolicyRequest req) {
         if (randomBoolean()) {
             return new RemoveIndexLifecyclePolicyRequest(req.indices(),
                     randomValueOtherThan(req.indicesOptions(), () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(),
-                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())));
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())));
         } else {
             return new RemoveIndexLifecyclePolicyRequest(
                     randomValueOtherThan(req.indices(), () -> Arrays.asList(generateRandomStringArray(20, 20, false))),

diff --git a/...-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/...-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
@@ -46,7 +46,7 @@ public class MultiSearchTemplateRequest extends ActionRequest implements Composi
     private int maxConcurrentSearchRequests = 0;
     private List<SearchTemplateRequest> requests = new ArrayList<>();
 
-    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
+    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
 
     /**
      * Add a search template request to execute. Note, the order is important, the search response will be returned in the

diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
@@ -286,45 +286,45 @@ public void testIndicesOptions() {
         // test that ignore_unavailable=true works but returns one result less
         assertTrue(client().admin().indices().prepareClose("test2").get().isAcknowledged());
 
-        request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
         assertEquals(6, details.getRetrieved());
         assertEquals(5, details.getRelevantRetrieved());
 
         // test that ignore_unavailable=false or default settings throw an IndexClosedException
         assertTrue(client().admin().indices().prepareClose("test2").get().isAcknowledged());
-        request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         assertEquals(1, response.getFailures().size());
         assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class));
 
         // test expand_wildcards
         request = new RankEvalRequest(task, new String[] { "tes*" });
-        request.indicesOptions(IndicesOptions.fromParameters("none", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters("none", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
         assertEquals(0, details.getRetrieved());
 
-        request.indicesOptions(IndicesOptions.fromParameters("open", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters("open", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
         assertEquals(6, details.getRetrieved());
         assertEquals(5, details.getRelevantRetrieved());
 
-        request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         assertEquals(1, response.getFailures().size());
         assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class));
 
         // test allow_no_indices
         request = new RankEvalRequest(task, new String[] { "bad*" });
-        request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
         assertEquals(0, details.getRetrieved());
 
-        request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         assertEquals(1, response.getFailures().size());
         assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexNotFoundException.class));

diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java
@@ -59,7 +59,8 @@ protected RankEvalRequest createTestInstance() {
         }
         RankEvalRequest rankEvalRequest = new RankEvalRequest(RankEvalSpecTests.createTestItem(), indices);
         IndicesOptions indicesOptions = IndicesOptions.fromOptions(
-                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+            randomBoolean());
         rankEvalRequest.indicesOptions(indicesOptions);
         return rankEvalRequest;
     }

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
@@ -20,6 +20,10 @@
           "type" : "boolean",
           "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
         },
+        "ignore_throttled": {
+          "type" : "boolean",
+          "description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled"
+        },
         "allow_no_indices": {
            "type" : "boolean",
            "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -54,6 +54,10 @@
             "type" : "boolean",
             "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
         },
+        "ignore_throttled": {
+          "type" : "boolean",
+          "description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled"
+        },
         "allow_no_indices": {
             "type" : "boolean",
             "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
@@ -20,6 +20,10 @@
           "type" : "boolean",
           "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
         },
+        "ignore_throttled": {
+          "type" : "boolean",
+          "description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled"
+        },
         "allow_no_indices": {
           "type" : "boolean",
           "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
@@ -67,7 +67,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
     // indices options that require every specified index to exist, expand wildcards only to open
     // indices, don't allow that no indices are resolved from wildcard expressions and resolve the
     // expressions only against indices
-    private static final IndicesOptions INDICES_OPTIONS = IndicesOptions.fromOptions(false, false, true, false, true, false, true);
+    private static final IndicesOptions INDICES_OPTIONS = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
 
     public IndicesAliasesRequest() {
     }

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java
@@ -38,7 +38,7 @@ public class DeleteIndexRequest extends AcknowledgedRequest<DeleteIndexRequest>
 
     private String[] indices;
     // Delete index should work by default on both open and closed indices.
-    private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true);
+    private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
 
     public DeleteIndexRequest() {
     }

diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
@@ -58,7 +58,7 @@ public class MultiSearchRequest extends ActionRequest implements CompositeIndice
     private int maxConcurrentSearchRequests = 0;
     private List<SearchRequest> requests = new ArrayList<>();
 
-    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
+    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
 
     /**
      * Add a search request to execute. Note, the order is important, the search response will be returned in the
@@ -288,7 +288,7 @@ public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest,
         }
         return output.toByteArray();
     }
-    
+
     public static void writeSearchRequestParams(SearchRequest request, XContentBuilder xContentBuilder) throws IOException {
         xContentBuilder.startObject();
         if (request.indices() != null) {

diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java
@@ -28,6 +28,7 @@
  */
 public class MultiSearchRequestBuilder extends ActionRequestBuilder<MultiSearchRequest, MultiSearchResponse, MultiSearchRequestBuilder> {
 
+
     public MultiSearchRequestBuilder(ElasticsearchClient client, MultiSearchAction action) {
         super(client, action, new MultiSearchRequest());
     }
@@ -40,7 +41,8 @@ public MultiSearchRequestBuilder(ElasticsearchClient client, MultiSearchAction a
      * will not be used (if set).
      */
     public MultiSearchRequestBuilder add(SearchRequest request) {
-        if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
+        if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed()
+            && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
             request.indicesOptions(request().indicesOptions());
         }
 
@@ -53,7 +55,8 @@ public MultiSearchRequestBuilder add(SearchRequest request) {
      * same order as the search requests.
      */
     public MultiSearchRequestBuilder add(SearchRequestBuilder request) {
-        if (request.request().indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
+        if (request.request().indicesOptions() == SearchRequest.DEFAULT_INDICES_OPTIONS
+            && request().indicesOptions() != SearchRequest.DEFAULT_INDICES_OPTIONS) {
             request.request().indicesOptions(request().indicesOptions());
         }
 

diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -91,7 +91,7 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest
 
     private String[] types = Strings.EMPTY_ARRAY;
 
-    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed();
+    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
 
     private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;