diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 7fe3e08f3afb0..89f81512bc9d2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; @@ -887,6 +888,19 @@ static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) t return request; } + static Request putScript(PutStoredScriptRequest putStoredScriptRequest) throws IOException { + String endpoint = new EndpointBuilder().addPathPartAsIs("_scripts").addPathPart(putStoredScriptRequest.id()).build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + Params params = new Params(request); + params.withTimeout(putStoredScriptRequest.timeout()); + params.withMasterTimeout(putStoredScriptRequest.masterNodeTimeout()); + if (Strings.hasText(putStoredScriptRequest.context())) { + params.putParam("context", putStoredScriptRequest.context()); + } + request.setEntity(createEntity(putStoredScriptRequest, REQUEST_BODY_CONTENT_TYPE)); + return request; + } + static Request analyze(AnalyzeRequest request) throws IOException { EndpointBuilder builder = new EndpointBuilder(); String index = request.index(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 17f8f65943012..687290abe8866 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; @@ -121,36 +122,36 @@ import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedAvg; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedCardinality; +import org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedGeoBounds; import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedGeoCentroid; -import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedMax; -import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedMin; import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentiles; -import org.elasticsearch.search.aggregations.metrics.ParsedHDRPercentileRanks; -import org.elasticsearch.search.aggregations.metrics.ParsedHDRPercentiles; import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentileRanks; import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles; +import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.ParsedAvg; +import org.elasticsearch.search.aggregations.metrics.ParsedCardinality; +import org.elasticsearch.search.aggregations.metrics.ParsedExtendedStats; +import org.elasticsearch.search.aggregations.metrics.ParsedGeoBounds; +import org.elasticsearch.search.aggregations.metrics.ParsedGeoCentroid; +import org.elasticsearch.search.aggregations.metrics.ParsedHDRPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.ParsedHDRPercentiles; +import org.elasticsearch.search.aggregations.metrics.ParsedMax; +import org.elasticsearch.search.aggregations.metrics.ParsedMin; +import org.elasticsearch.search.aggregations.metrics.ParsedScriptedMetric; +import org.elasticsearch.search.aggregations.metrics.ParsedStats; +import org.elasticsearch.search.aggregations.metrics.ParsedSum; import org.elasticsearch.search.aggregations.metrics.ParsedTDigestPercentileRanks; import org.elasticsearch.search.aggregations.metrics.ParsedTDigestPercentiles; -import org.elasticsearch.search.aggregations.metrics.ParsedScriptedMetric; +import org.elasticsearch.search.aggregations.metrics.ParsedTopHits; +import org.elasticsearch.search.aggregations.metrics.ParsedValueCount; import org.elasticsearch.search.aggregations.metrics.ScriptedMetricAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedStats; import org.elasticsearch.search.aggregations.metrics.StatsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedExtendedStats; -import org.elasticsearch.search.aggregations.metrics.ParsedSum; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedTopHits; import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedValueCount; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue; @@ -1050,6 +1051,35 @@ public void deleteScriptAsync(DeleteStoredScriptRequest request, RequestOptions AcknowledgedResponse::fromXContent, listener, emptySet()); } + /** + * Puts an stored script using the Scripting API. + * See Scripting API + * on elastic.co + * @param putStoredScriptRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public AcknowledgedResponse putScript(PutStoredScriptRequest putStoredScriptRequest, + RequestOptions options) throws IOException { + return performRequestAndParseEntity(putStoredScriptRequest, RequestConverters::putScript, options, + AcknowledgedResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously puts an stored script using the Scripting API. + * See Scripting API + * on elastic.co + * @param putStoredScriptRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void putScriptAsync(PutStoredScriptRequest putStoredScriptRequest, RequestOptions options, + ActionListener listener) { + performRequestAsyncAndParseEntity(putStoredScriptRequest, RequestConverters::putScript, options, + AcknowledgedResponse::fromXContent, listener, emptySet()); + } + /** * Asynchronously executes a request using the Field Capabilities API. * See Field Capabilities API diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 840df49b47811..6f48d305a7799 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; @@ -1991,6 +1992,42 @@ public void testGetTemplateRequest() throws Exception { assertThat(request.getEntity(), nullValue()); } + public void testPutScript() throws Exception { + PutStoredScriptRequest putStoredScriptRequest = new PutStoredScriptRequest(); + + String id = randomAlphaOfLengthBetween(5, 10); + putStoredScriptRequest.id(id); + + XContentType xContentType = randomFrom(XContentType.values()); + try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) { + builder.startObject(); + builder.startObject("script") + .field("lang", "painless") + .field("source", "Math.log(_score * 2) + params.multiplier") + .endObject(); + builder.endObject(); + + putStoredScriptRequest.content(BytesReference.bytes(builder), xContentType); + } + + Map expectedParams = new HashMap<>(); + setRandomMasterTimeout(putStoredScriptRequest, expectedParams); + setRandomTimeout(putStoredScriptRequest::timeout, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams); + + if (randomBoolean()) { + String context = randomAlphaOfLengthBetween(5, 10); + putStoredScriptRequest.context(context); + expectedParams.put("context", context); + } + + Request request = RequestConverters.putScript(putStoredScriptRequest); + + assertThat(request.getEndpoint(), equalTo("/_scripts/" + id)); + assertThat(request.getParameters(), equalTo(expectedParams)); + assertNotNull(request.getEntity()); + assertToXContentBody(putStoredScriptRequest, request.getEntity()); + } + public void testAnalyzeRequest() throws Exception { AnalyzeRequest indexAnalyzeRequest = new AnalyzeRequest() .text("Here is some text") diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index b6562cd44cd55..3bd47306e5e10 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -658,7 +658,6 @@ public void testApiNamingConventions() throws Exception { "indices.get_upgrade", "indices.put_alias", "mtermvectors", - "put_script", "reindex_rethrottle", "render_search_template", "scripts_painless_execute", diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java index 1d693eee8396e..b15467d24ba2b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java @@ -1,4 +1,5 @@ -package org.elasticsearch.client;/* +package org.elasticsearch.client; +/* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright @@ -17,27 +18,27 @@ * under the License. */ -import org.apache.http.util.EntityUtils; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.StoredScriptSource; import java.util.Collections; +import java.util.Map; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; public class StoredScriptsIT extends ESRestHighLevelClientTestCase { - final String id = "calculate-score"; + private static final String id = "calculate-score"; public void testGetStoredScript() throws Exception { final StoredScriptSource scriptSource = @@ -45,13 +46,9 @@ public void testGetStoredScript() throws Exception { "Math.log(_score * 2) + params.my_modifier", Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType())); - final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - // TODO: change to HighLevel PutStoredScriptRequest when it will be ready - // so far - using low-level REST API - Request putRequest = new Request("PUT", "/_scripts/calculate-score"); - putRequest.setJsonEntity("{\"script\":" + script + "}"); - Response putResponse = adminClient().performRequest(putRequest); - assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity())); + PutStoredScriptRequest request = + new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource); + assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync)); GetStoredScriptRequest getRequest = new GetStoredScriptRequest("calculate-score"); getRequest.masterNodeTimeout("50s"); @@ -68,22 +65,14 @@ public void testDeleteStoredScript() throws Exception { "Math.log(_score * 2) + params.my_modifier", Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType())); - final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - // TODO: change to HighLevel PutStoredScriptRequest when it will be ready - // so far - using low-level REST API - Request putRequest = new Request("PUT", "/_scripts/" + id); - putRequest.setJsonEntity("{\"script\":" + script + "}"); - Response putResponse = adminClient().performRequest(putRequest); - assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity())); + PutStoredScriptRequest request = + new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource); + assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync)); DeleteStoredScriptRequest deleteRequest = new DeleteStoredScriptRequest(id); deleteRequest.masterNodeTimeout("50s"); deleteRequest.timeout("50s"); - - AcknowledgedResponse deleteResponse = execute(deleteRequest, highLevelClient()::deleteScript, - highLevelClient()::deleteScriptAsync); - - assertThat(deleteResponse.isAcknowledged(), equalTo(true)); + assertAcked(execute(deleteRequest, highLevelClient()::deleteScript, highLevelClient()::deleteScriptAsync)); GetStoredScriptRequest getRequest = new GetStoredScriptRequest(id); @@ -92,4 +81,21 @@ public void testDeleteStoredScript() throws Exception { highLevelClient()::getScriptAsync)); assertThat(statusException.status(), equalTo(RestStatus.NOT_FOUND)); } + + public void testPutScript() throws Exception { + final StoredScriptSource scriptSource = + new StoredScriptSource("painless", + "Math.log(_score * 2) + params.my_modifier", + Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType())); + + PutStoredScriptRequest request = + new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource); + assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync)); + + Map script = getAsMap("/_scripts/" + id); + assertThat(extractValue("_id", script), equalTo(id)); + assertThat(extractValue("found", script), equalTo(true)); + assertThat(extractValue("script.lang", script), equalTo("painless")); + assertThat(extractValue("script.source", script), equalTo("Math.log(_score * 2) + params.my_modifier")); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java index fc38090ef5b5b..c5d53abd978e1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java @@ -17,21 +17,21 @@ * under the License. */ -import org.apache.http.util.EntityUtils; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; -import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.Script; import org.elasticsearch.script.StoredScriptSource; @@ -42,7 +42,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; /** @@ -187,14 +188,124 @@ public void onFailure(Exception e) { assertTrue(latch.await(30L, TimeUnit.SECONDS)); } + public void testPutScript() throws Exception { + RestHighLevelClient client = highLevelClient(); + + { + // tag::put-stored-script-request + PutStoredScriptRequest request = new PutStoredScriptRequest(); + request.id("id"); // <1> + request.content(new BytesArray( + "{\n" + + "\"script\": {\n" + + "\"lang\": \"painless\",\n" + + "\"source\": \"Math.log(_score * 2) + params.multiplier\"" + + "}\n" + + "}\n" + ), XContentType.JSON); // <2> + // end::put-stored-script-request + + // tag::put-stored-script-context + request.context("context"); // <1> + // end::put-stored-script-context + + // tag::put-stored-script-timeout + request.timeout(TimeValue.timeValueMinutes(2)); // <1> + request.timeout("2m"); // <2> + // end::put-stored-script-timeout + + // tag::put-stored-script-masterTimeout + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1> + request.masterNodeTimeout("1m"); // <2> + // end::put-stored-script-masterTimeout + } + + { + PutStoredScriptRequest request = new PutStoredScriptRequest(); + request.id("id"); + + // tag::put-stored-script-content-painless + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.startObject("script"); + { + builder.field("lang", "painless"); + builder.field("source", "Math.log(_score * 2) + params.multiplier"); + } + builder.endObject(); + } + builder.endObject(); + request.content(BytesReference.bytes(builder), XContentType.JSON); // <1> + // end::put-stored-script-content-painless + + + // tag::put-stored-script-execute + AcknowledgedResponse putStoredScriptResponse = client.putScript(request, RequestOptions.DEFAULT); + // end::put-stored-script-execute + + // tag::put-stored-script-response + boolean acknowledged = putStoredScriptResponse.isAcknowledged(); // <1> + // end::put-stored-script-response + + assertTrue(acknowledged); + + // tag::put-stored-script-execute-listener + ActionListener listener = + new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse response) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::put-stored-script-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::put-stored-script-execute-async + client.putScriptAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::put-stored-script-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + + { + PutStoredScriptRequest request = new PutStoredScriptRequest(); + request.id("id"); + + // tag::put-stored-script-content-mustache + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.startObject("script"); + { + builder.field("lang", "mustache"); + builder.field("source", "{\"query\":{\"match\":{\"title\":\"{{query_string}}\"}}}"); + } + builder.endObject(); + } + builder.endObject(); + request.content(BytesReference.bytes(builder), XContentType.JSON); // <1> + // end::put-stored-script-content-mustache + + client.putScript(request, RequestOptions.DEFAULT); + + Map script = getAsMap("/_scripts/id"); + assertThat(extractValue("script.lang", script), equalTo("mustache")); + assertThat(extractValue("script.source", script), equalTo("{\"query\":{\"match\":{\"title\":\"{{query_string}}\"}}}")); + } + } + private void putStoredScript(String id, StoredScriptSource scriptSource) throws IOException { - final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - // TODO: change to HighLevel PutStoredScriptRequest when it will be ready - // so far - using low-level REST API - Request request = new Request("PUT", "/_scripts/" + id); - request.setJsonEntity("{\"script\":" + script + "}"); - Response putResponse = adminClient().performRequest(request); - assertEquals(putResponse.getStatusLine().getReasonPhrase(), 200, putResponse.getStatusLine().getStatusCode()); - assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity())); + PutStoredScriptRequest request = + new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource); + assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync)); } } diff --git a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java index 09d6b9e51a5fc..ffd3a1f6c0c3c 100644 --- a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java +++ b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java @@ -38,6 +38,7 @@ /** * Tests that wait for refresh is fired if the index is closed. */ +@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33533") public class WaitForRefreshAndCloseIT extends ESRestTestCase { @Before public void setupIndex() throws IOException { @@ -76,7 +77,6 @@ public void testUpdateAndThenClose() throws Exception { closeWhileListenerEngaged(start("POST", "/_update", "{\"doc\":{\"name\":\"test\"}}")); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33533") public void testDeleteAndThenClose() throws Exception { Request request = new Request("PUT", docPath()); request.setJsonEntity("{\"test\":\"test\"}"); diff --git a/docs/java-rest/high-level/script/put_script.asciidoc b/docs/java-rest/high-level/script/put_script.asciidoc new file mode 100644 index 0000000000000..acc80e82d11e6 --- /dev/null +++ b/docs/java-rest/high-level/script/put_script.asciidoc @@ -0,0 +1,106 @@ +[[java-rest-high-put-stored-script]] +=== Put Stored Script API + +[[java-rest-high-put-stored-script-request]] +==== Put Stored Script Request + +A `PutStoredScriptRequest` requires an `id` and `content`: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-request] +-------------------------------------------------- +<1> The id of the script +<2> The content of the script + +[[java-rest-high-put-stored-script-content]] +==== Content +The content of a script can be written in different languages and provided in +different ways: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-content-painless] +-------------------------------------------------- +<1> Specify a painless script and provided as `XContentBuilder` object. +Note that the builder needs to be passed as a `BytesReference` object + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-content-mustache] +-------------------------------------------------- +<1> Specify a mustache script and provided as `XContentBuilder` object. +Note that value of source can be directly provided as a JSON string + +==== Optional arguments +The following arguments can optionally be provided: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-context] +-------------------------------------------------- +<1> The context the script should be executed in. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-timeout] +-------------------------------------------------- +<1> Timeout to wait for the all the nodes to acknowledge the script creation as a `TimeValue` +<2> Timeout to wait for the all the nodes to acknowledge the script creation as a `String` + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-masterTimeout] +-------------------------------------------------- +<1> Timeout to connect to the master node as a `TimeValue` +<2> Timeout to connect to the master node as a `String` + +[[java-rest-high-put-stored-script-sync]] +==== Synchronous Execution +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-execute] +-------------------------------------------------- + +[[java-rest-high-put-stored-script-async]] +==== Asynchronous Execution + +The asynchronous execution of a put stored script request requires both the `PutStoredScriptRequest` +instance and an `ActionListener` instance to be passed to the asynchronous method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-execute-async] +-------------------------------------------------- +<1> The `PutStoredScriptRequest` to execute and the `ActionListener` to use when +the execution completes + +[[java-rest-high-put-stored-script-listener]] +===== Action Listener + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method +if the execution successfully completed or using the `onFailure` method if +it failed. + +A typical listener for `AcknowledgedResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is +provided as an argument +<2> Called in case of failure. The raised exception is provided as an argument + +[[java-rest-high-put-stored-script-response]] +==== Put Stored Script Response + +The returned `AcknowledgedResponse` allows to retrieve information about the +executed operation as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-response] +-------------------------------------------------- +<1> Indicates whether all of the nodes have acknowledged the request \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 8d49724353e6f..8d92653ce5702 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -189,9 +189,11 @@ include::tasks/cancel_tasks.asciidoc[] The Java High Level REST Client supports the following Scripts APIs: * <> +* <> * <> include::script/get_script.asciidoc[] +include::script/put_script.asciidoc[] include::script/delete_script.asciidoc[] == Licensing APIs diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java index 7acbff6cb0b93..31a9e595d0b6d 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java @@ -61,12 +61,19 @@ public final class Whitelist { /** The {@link List} of all the whitelisted Painless classes. */ public final List whitelistClasses; + /** The {@link List} of all the whitelisted static Painless methods. */ + public final List whitelistImportedMethods; + + /** The {@link List} of all the whitelisted Painless bindings. */ public final List whitelistBindings; - /** Standard constructor. All values must be not {@code null}. */ - public Whitelist(ClassLoader classLoader, List whitelistClasses, List whitelistBindings) { + /** Standard constructor. All values must be not {@code null}. */ + public Whitelist(ClassLoader classLoader, List whitelistClasses, + List whitelistImportedMethods, List whitelistBindings) { + this.classLoader = Objects.requireNonNull(classLoader); this.whitelistClasses = Collections.unmodifiableList(Objects.requireNonNull(whitelistClasses)); + this.whitelistImportedMethods = Collections.unmodifiableList(Objects.requireNonNull(whitelistImportedMethods)); this.whitelistBindings = Collections.unmodifiableList(Objects.requireNonNull(whitelistBindings)); } } diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java index 0279c82f1b67b..2f5dec769fc2f 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java @@ -133,6 +133,7 @@ public final class WhitelistLoader { */ public static Whitelist loadFromResourceFiles(Class resource, String... filepaths) { List whitelistClasses = new ArrayList<>(); + List whitelistStatics = new ArrayList<>(); List whitelistBindings = new ArrayList<>(); // Execute a single pass through the whitelist text files. This will gather all the @@ -192,18 +193,18 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep whitelistConstructors = new ArrayList<>(); whitelistMethods = new ArrayList<>(); whitelistFields = new ArrayList<>(); - } else if (line.startsWith("static ")) { + } else if (line.startsWith("static_import ")) { // Ensure the final token of the line is '{'. if (line.endsWith("{") == false) { throw new IllegalArgumentException( - "invalid static definition: failed to parse static opening bracket [" + line + "]"); + "invalid static import definition: failed to parse static import opening bracket [" + line + "]"); } if (parseType != null) { - throw new IllegalArgumentException("invalid definition: cannot embed static definition [" + line + "]"); + throw new IllegalArgumentException("invalid definition: cannot embed static import definition [" + line + "]"); } - parseType = "static"; + parseType = "static_import"; // Handle the end of a definition and reset all previously gathered values. // Expects the following format: '}' '\n' @@ -229,9 +230,9 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep // Reset the parseType. parseType = null; - // Handle static definition types. - // Expects the following format: ID ID '(' ( ID ( ',' ID )* )? ')' 'bound_to' ID '\n' - } else if ("static".equals(parseType)) { + // Handle static import definition types. + // Expects the following format: ID ID '(' ( ID ( ',' ID )* )? ')' ( 'from_class' | 'bound_to' ) ID '\n' + } else if ("static_import".equals(parseType)) { // Mark the origin of this parsable object. String origin = "[" + filepath + "]:[" + number + "]"; @@ -240,7 +241,7 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep if (parameterStartIndex == -1) { throw new IllegalArgumentException( - "illegal static definition: start of method parameters not found [" + line + "]"); + "illegal static import definition: start of method parameters not found [" + line + "]"); } String[] tokens = line.substring(0, parameterStartIndex).trim().split("\\s+"); @@ -261,7 +262,7 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep if (parameterEndIndex == -1) { throw new IllegalArgumentException( - "illegal static definition: end of method parameters not found [" + line + "]"); + "illegal static import definition: end of method parameters not found [" + line + "]"); } String[] canonicalTypeNameParameters = @@ -272,39 +273,37 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep canonicalTypeNameParameters = new String[0]; } - // Parse the static type and class. + // Parse the static import type and class. tokens = line.substring(parameterEndIndex + 1).trim().split("\\s+"); - String staticType; + String staticImportType; String targetJavaClassName; // Based on the number of tokens, look up the type and class. if (tokens.length == 2) { - staticType = tokens[0]; + staticImportType = tokens[0]; targetJavaClassName = tokens[1]; } else { - throw new IllegalArgumentException("invalid static definition: unexpected format [" + line + "]"); + throw new IllegalArgumentException("invalid static import definition: unexpected format [" + line + "]"); } - // Check the static type is valid. - if ("bound_to".equals(staticType) == false) { - throw new IllegalArgumentException( - "invalid static definition: unexpected static type [" + staticType + "] [" + line + "]"); + // Add a static import method or binding depending on the static import type. + if ("from_class".equals(staticImportType)) { + whitelistStatics.add(new WhitelistMethod(origin, targetJavaClassName, + methodName, returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters))); + } else if ("bound_to".equals(staticImportType)) { + whitelistBindings.add(new WhitelistBinding(origin, targetJavaClassName, + methodName, returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters))); + } else { + throw new IllegalArgumentException("invalid static import definition: " + + "unexpected static import type [" + staticImportType + "] [" + line + "]"); } - whitelistBindings.add(new WhitelistBinding(origin, targetJavaClassName, - methodName, returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters))); - // Handle class definition types. } else if ("class".equals(parseType)) { // Mark the origin of this parsable object. String origin = "[" + filepath + "]:[" + number + "]"; - // Ensure we have a defined class before adding any constructors, methods, augmented methods, or fields. - if (parseType == null) { - throw new IllegalArgumentException("invalid definition: expected one of ['class', 'static'] [" + line + "]"); - } - // Handle the case for a constructor definition. // Expects the following format: '(' ( ID ( ',' ID )* )? ')' '\n' if (line.startsWith("(")) { @@ -393,7 +392,7 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep ClassLoader loader = AccessController.doPrivileged((PrivilegedAction)resource::getClassLoader); - return new Whitelist(loader, whitelistClasses, whitelistBindings); + return new Whitelist(loader, whitelistClasses, whitelistStatics, whitelistBindings); } private WhitelistLoader() {} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java index 1e94c19f6d90e..28cbb4aee19a6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java @@ -24,6 +24,21 @@ /** Currently just a dummy class for testing a few features not yet exposed by whitelist! */ public class FeatureTest { + /** static method that returns true */ + public static boolean overloadedStatic() { + return true; + } + + /** static method that returns what you ask it */ + public static boolean overloadedStatic(boolean whatToReturn) { + return whatToReturn; + } + + /** static method only whitelisted as a static */ + public static float staticAddFloatsTest(float x, float y) { + return x + y; + } + private int x; private int y; public int z; @@ -58,21 +73,12 @@ public void setY(int y) { this.y = y; } - /** static method that returns true */ - public static boolean overloadedStatic() { - return true; - } - - /** static method that returns what you ask it */ - public static boolean overloadedStatic(boolean whatToReturn) { - return whatToReturn; - } - /** method taking two functions! */ public Object twoFunctionsOfX(Function f, Function g) { return f.apply(g.apply(x)); } + /** method to take in a list */ public void listInput(List list) { } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java index 2d6ed3e361dc3..7be659d11a124 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java @@ -37,16 +37,23 @@ public final class PainlessLookup { private final Map> canonicalClassNamesToClasses; private final Map, PainlessClass> classesToPainlessClasses; + private final Map painlessMethodKeysToImportedPainlessMethods; private final Map painlessMethodKeysToPainlessBindings; PainlessLookup(Map> canonicalClassNamesToClasses, Map, PainlessClass> classesToPainlessClasses, + Map painlessMethodKeysToImportedPainlessMethods, Map painlessMethodKeysToPainlessBindings) { + Objects.requireNonNull(canonicalClassNamesToClasses); Objects.requireNonNull(classesToPainlessClasses); + Objects.requireNonNull(painlessMethodKeysToImportedPainlessMethods); + Objects.requireNonNull(painlessMethodKeysToPainlessBindings); + this.canonicalClassNamesToClasses = Collections.unmodifiableMap(canonicalClassNamesToClasses); this.classesToPainlessClasses = Collections.unmodifiableMap(classesToPainlessClasses); + this.painlessMethodKeysToImportedPainlessMethods = Collections.unmodifiableMap(painlessMethodKeysToImportedPainlessMethods); this.painlessMethodKeysToPainlessBindings = Collections.unmodifiableMap(painlessMethodKeysToPainlessBindings); } @@ -167,6 +174,14 @@ public PainlessField lookupPainlessField(Class targetClass, boolean isStatic, return painlessField; } + public PainlessMethod lookupImportedPainlessMethod(String methodName, int arity) { + Objects.requireNonNull(methodName); + + String painlessMethodKey = buildPainlessMethodKey(methodName, arity); + + return painlessMethodKeysToImportedPainlessMethods.get(painlessMethodKey); + } + public PainlessBinding lookupPainlessBinding(String methodName, int arity) { Objects.requireNonNull(methodName); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java index a64814f866113..b822bd47c7a48 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java @@ -243,6 +243,14 @@ public static PainlessLookup buildFromWhitelists(List whitelists) { } } + for (WhitelistMethod whitelistStatic : whitelist.whitelistImportedMethods) { + origin = whitelistStatic.origin; + painlessLookupBuilder.addImportedPainlessMethod( + whitelist.classLoader, whitelistStatic.augmentedCanonicalClassName, + whitelistStatic.methodName, whitelistStatic.returnCanonicalTypeName, + whitelistStatic.canonicalTypeNameParameters); + } + for (WhitelistBinding whitelistBinding : whitelist.whitelistBindings) { origin = whitelistBinding.origin; painlessLookupBuilder.addPainlessBinding( @@ -261,12 +269,14 @@ public static PainlessLookup buildFromWhitelists(List whitelists) { private final Map> canonicalClassNamesToClasses; private final Map, PainlessClassBuilder> classesToPainlessClassBuilders; + private final Map painlessMethodKeysToImportedPainlessMethods; private final Map painlessMethodKeysToPainlessBindings; public PainlessLookupBuilder() { canonicalClassNamesToClasses = new HashMap<>(); classesToPainlessClassBuilders = new HashMap<>(); + painlessMethodKeysToImportedPainlessMethods = new HashMap<>(); painlessMethodKeysToPainlessBindings = new HashMap<>(); } @@ -513,8 +523,9 @@ public void addPainlessMethod(ClassLoader classLoader, String targetCanonicalCla addPainlessMethod(targetClass, augmentedClass, methodName, returnType, typeParameters); } - public void addPainlessMethod(Class targetClass, Class augmentedClass, String methodName, - Class returnType, List> typeParameters) { + public void addPainlessMethod(Class targetClass, Class augmentedClass, + String methodName, Class returnType, List> typeParameters) { + Objects.requireNonNull(targetClass); Objects.requireNonNull(methodName); Objects.requireNonNull(returnType); @@ -573,6 +584,12 @@ public void addPainlessMethod(Class targetClass, Class augmentedClass, Str } else { try { javaMethod = augmentedClass.getMethod(methodName, javaTypeParameters.toArray(new Class[typeParametersSize])); + + if (Modifier.isStatic(javaMethod.getModifiers()) == false) { + throw new IllegalArgumentException("method [[" + targetCanonicalClassName + "], [" + methodName + "], " + + typesToCanonicalTypeNames(typeParameters) + "] with augmented class " + + "[" + typeToCanonicalTypeName(augmentedClass) + "] must be static"); + } } catch (NoSuchMethodException nsme) { throw new IllegalArgumentException("method reflection object [[" + targetCanonicalClassName + "], " + "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found " + @@ -620,7 +637,7 @@ public void addPainlessMethod(Class targetClass, Class augmentedClass, Str "with the same arity and different return type or type parameters"); } } else { - PainlessMethod painlessMethod = painlessClassBuilder.staticMethods.get(painlessMethodKey); + PainlessMethod painlessMethod = painlessClassBuilder.methods.get(painlessMethodKey); if (painlessMethod == null) { MethodHandle methodHandle; @@ -788,6 +805,146 @@ public void addPainlessField(Class targetClass, String fieldName, Class ty } } + public void addImportedPainlessMethod(ClassLoader classLoader, String targetCanonicalClassName, + String methodName, String returnCanonicalTypeName, List canonicalTypeNameParameters) { + + Objects.requireNonNull(classLoader); + Objects.requireNonNull(targetCanonicalClassName); + Objects.requireNonNull(methodName); + Objects.requireNonNull(returnCanonicalTypeName); + Objects.requireNonNull(canonicalTypeNameParameters); + + Class targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName); + + if (targetClass == null) { + throw new IllegalArgumentException("target class [" + targetCanonicalClassName + "] not found for imported method " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]"); + } + + List> typeParameters = new ArrayList<>(canonicalTypeNameParameters.size()); + + for (String canonicalTypeNameParameter : canonicalTypeNameParameters) { + Class typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter); + + if (typeParameter == null) { + throw new IllegalArgumentException("type parameter [" + canonicalTypeNameParameter + "] not found for imported method " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]"); + } + + typeParameters.add(typeParameter); + } + + Class returnType = canonicalTypeNameToType(returnCanonicalTypeName); + + if (returnType == null) { + throw new IllegalArgumentException("return type [" + returnCanonicalTypeName + "] not found for imported method " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]"); + } + + addImportedPainlessMethod(targetClass, methodName, returnType, typeParameters); + } + + public void addImportedPainlessMethod(Class targetClass, String methodName, Class returnType, List> typeParameters) { + Objects.requireNonNull(targetClass); + Objects.requireNonNull(methodName); + Objects.requireNonNull(returnType); + Objects.requireNonNull(typeParameters); + + if (targetClass == def.class) { + throw new IllegalArgumentException("cannot add imported method from reserved class [" + DEF_CLASS_NAME + "]"); + } + + String targetCanonicalClassName = typeToCanonicalTypeName(targetClass); + + if (METHOD_NAME_PATTERN.matcher(methodName).matches() == false) { + throw new IllegalArgumentException( + "invalid imported method name [" + methodName + "] for target class [" + targetCanonicalClassName + "]."); + } + + PainlessClassBuilder painlessClassBuilder = classesToPainlessClassBuilders.get(targetClass); + + if (painlessClassBuilder == null) { + throw new IllegalArgumentException("target class [" + targetCanonicalClassName + "] not found for imported method " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]"); + } + + int typeParametersSize = typeParameters.size(); + List> javaTypeParameters = new ArrayList<>(typeParametersSize); + + for (Class typeParameter : typeParameters) { + if (isValidType(typeParameter) == false) { + throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] " + + "not found for imported method [[" + targetCanonicalClassName + "], [" + methodName + "], " + + typesToCanonicalTypeNames(typeParameters) + "]"); + } + + javaTypeParameters.add(typeToJavaType(typeParameter)); + } + + if (isValidType(returnType) == false) { + throw new IllegalArgumentException("return type [" + typeToCanonicalTypeName(returnType) + "] not found for imported method " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]"); + } + + Method javaMethod; + + try { + javaMethod = targetClass.getMethod(methodName, javaTypeParameters.toArray(new Class[typeParametersSize])); + } catch (NoSuchMethodException nsme) { + throw new IllegalArgumentException("imported method reflection object [[" + targetCanonicalClassName + "], " + + "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", nsme); + } + + if (javaMethod.getReturnType() != typeToJavaType(returnType)) { + throw new IllegalArgumentException("return type [" + typeToCanonicalTypeName(javaMethod.getReturnType()) + "] " + + "does not match the specified returned type [" + typeToCanonicalTypeName(returnType) + "] " + + "for imported method [[" + targetClass.getCanonicalName() + "], [" + methodName + "], " + + typesToCanonicalTypeNames(typeParameters) + "]"); + } + + if (Modifier.isStatic(javaMethod.getModifiers()) == false) { + throw new IllegalArgumentException("imported method [[" + targetClass.getCanonicalName() + "], [" + methodName + "], " + + typesToCanonicalTypeNames(typeParameters) + "] must be static"); + } + + String painlessMethodKey = buildPainlessMethodKey(methodName, typeParametersSize); + + if (painlessMethodKeysToPainlessBindings.containsKey(painlessMethodKey)) { + throw new IllegalArgumentException("imported method and binding cannot have the same name [" + methodName + "]"); + } + + PainlessMethod importedPainlessMethod = painlessMethodKeysToImportedPainlessMethods.get(painlessMethodKey); + + if (importedPainlessMethod == null) { + MethodHandle methodHandle; + + try { + methodHandle = MethodHandles.publicLookup().in(targetClass).unreflect(javaMethod); + } catch (IllegalAccessException iae) { + throw new IllegalArgumentException("imported method handle [[" + targetClass.getCanonicalName() + "], " + + "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae); + } + + MethodType methodType = methodHandle.type(); + + importedPainlessMethod = painlessMethodCache.computeIfAbsent( + new PainlessMethodCacheKey(targetClass, methodName, returnType, typeParameters), + key -> new PainlessMethod(javaMethod, targetClass, returnType, typeParameters, methodHandle, methodType)); + + painlessMethodKeysToImportedPainlessMethods.put(painlessMethodKey, importedPainlessMethod); + } else if (importedPainlessMethod.returnType == returnType && + importedPainlessMethod.typeParameters.equals(typeParameters) == false) { + throw new IllegalArgumentException("cannot have imported methods " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + + "[" + typeToCanonicalTypeName(returnType) + "], " + + typesToCanonicalTypeNames(typeParameters) + "] and " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + + "[" + typeToCanonicalTypeName(importedPainlessMethod.returnType) + "], " + + typesToCanonicalTypeNames(importedPainlessMethod.typeParameters) + "] " + + "with the same arity and different return type or type parameters"); + } + } + public void addPainlessBinding(ClassLoader classLoader, String targetJavaClassName, String methodName, String returnCanonicalTypeName, List canonicalTypeNameParameters) { @@ -937,6 +1094,11 @@ public void addPainlessBinding(Class targetClass, String methodName, Class } String painlessMethodKey = buildPainlessMethodKey(methodName, constructorTypeParametersSize + methodTypeParametersSize); + + if (painlessMethodKeysToImportedPainlessMethods.containsKey(painlessMethodKey)) { + throw new IllegalArgumentException("binding and imported method cannot have the same name [" + methodName + "]"); + } + PainlessBinding painlessBinding = painlessMethodKeysToPainlessBindings.get(painlessMethodKey); if (painlessBinding == null) { @@ -976,7 +1138,8 @@ public PainlessLookup build() { classesToPainlessClasses.put(painlessClassBuilderEntry.getKey(), painlessClassBuilderEntry.getValue().build()); } - return new PainlessLookup(canonicalClassNamesToClasses, classesToPainlessClasses, painlessMethodKeysToPainlessBindings); + return new PainlessLookup(canonicalClassNamesToClasses, classesToPainlessClasses, + painlessMethodKeysToImportedPainlessMethods, painlessMethodKeysToPainlessBindings); } private void copyPainlessClassMembers() { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java index 8ae6ad9723da4..d161296d90a56 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java @@ -25,6 +25,7 @@ import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; import org.elasticsearch.painless.lookup.PainlessBinding; +import org.elasticsearch.painless.lookup.PainlessMethod; import org.objectweb.asm.Label; import org.objectweb.asm.Type; import org.objectweb.asm.commons.Method; @@ -45,6 +46,7 @@ public final class ECallLocal extends AExpression { private final List arguments; private LocalMethod method = null; + private PainlessMethod imported = null; private PainlessBinding binding = null; public ECallLocal(Location location, String name, List arguments) { @@ -65,16 +67,33 @@ void extractVariables(Set variables) { void analyze(Locals locals) { method = locals.getMethod(name, arguments.size()); - if (method == null) { - binding = locals.getPainlessLookup().lookupPainlessBinding(name, arguments.size()); + imported = locals.getPainlessLookup().lookupImportedPainlessMethod(name, arguments.size()); + + if (imported == null) { + binding = locals.getPainlessLookup().lookupPainlessBinding(name, arguments.size()); - if (binding == null) { - throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + "] arguments.")); + if (binding == null) { + throw createError( + new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + "] arguments.")); + } } } - List> typeParameters = new ArrayList<>(method == null ? binding.typeParameters : method.typeParameters); + List> typeParameters; + + if (method != null) { + typeParameters = new ArrayList<>(method.typeParameters); + actual = method.returnType; + } else if (imported != null) { + typeParameters = new ArrayList<>(imported.typeParameters); + actual = imported.returnType; + } else if (binding != null) { + typeParameters = new ArrayList<>(binding.typeParameters); + actual = binding.returnType; + } else { + throw new IllegalStateException("Illegal tree structure."); + } for (int argument = 0; argument < arguments.size(); ++argument) { AExpression expression = arguments.get(argument); @@ -86,14 +105,26 @@ void analyze(Locals locals) { } statement = true; - actual = method == null ? binding.returnType : method.returnType; } @Override void write(MethodWriter writer, Globals globals) { writer.writeDebugInfo(location); - if (method == null) { + if (method != null) { + for (AExpression argument : arguments) { + argument.write(writer, globals); + } + + writer.invokeStatic(CLASS_TYPE, new Method(method.name, method.methodType.toMethodDescriptorString())); + } else if (imported != null) { + for (AExpression argument : arguments) { + argument.write(writer, globals); + } + + writer.invokeStatic(Type.getType(imported.targetClass), + new Method(imported.javaMethod.getName(), imported.methodType.toMethodDescriptorString())); + } else if (binding != null) { String name = globals.addBinding(binding.javaConstructor.getDeclaringClass()); Type type = Type.getType(binding.javaConstructor.getDeclaringClass()); int javaConstructorParameterCount = binding.javaConstructor.getParameterCount(); @@ -124,11 +155,7 @@ void write(MethodWriter writer, Globals globals) { writer.invokeVirtual(type, Method.getMethod(binding.javaMethod)); } else { - for (AExpression argument : arguments) { - argument.write(writer, globals); - } - - writer.invokeStatic(CLASS_TYPE, new Method(method.name, method.methodType.toMethodDescriptorString())); + throw new IllegalStateException("Illegal tree structure."); } } diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt index 444234384c6d3..81009de9979eb 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt @@ -176,6 +176,7 @@ class org.elasticsearch.painless.FeatureTest no_import { } # for testing -static { +static_import { + float staticAddFloatsTest(float, float) from_class org.elasticsearch.painless.FeatureTest int testAddWithState(int, int, int, double) bound_to org.elasticsearch.painless.BindingTest } \ No newline at end of file diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java index 25866c8d668a3..9863db0b21eac 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java @@ -133,4 +133,8 @@ public void testPublicMemberAccess() { public void testNoSemicolon() { assertEquals(true, exec("def x = true; if (x) return x")); } + + public void testStatic() { + assertEquals(15.5f, exec("staticAddFloatsTest(6.5f, 9.0f)")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java index 96cc296a1af52..963a433f172e8 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java @@ -47,6 +47,8 @@ * Typically just asserts the output of {@code exec()} */ public abstract class ScriptTestCase extends ESTestCase { + private static final PainlessLookup PAINLESS_LOOKUP = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS); + protected PainlessScriptEngine scriptEngine; @Before @@ -92,12 +94,12 @@ public Object exec(String script, Map vars, boolean picky) { public Object exec(String script, Map vars, Map compileParams, Scorer scorer, boolean picky) { // test for ambiguity errors before running the actual script if picky is true if (picky) { - PainlessLookup painlessLookup = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS); - ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, GenericElasticsearchScript.class); + ScriptClassInfo scriptClassInfo = new ScriptClassInfo(PAINLESS_LOOKUP, GenericElasticsearchScript.class); CompilerSettings pickySettings = new CompilerSettings(); pickySettings.setPicky(true); pickySettings.setRegexesEnabled(CompilerSettings.REGEX_ENABLED.get(scriptEngineSettings())); - Walker.buildPainlessTree(scriptClassInfo, new MainMethodReserved(), getTestName(), script, pickySettings, painlessLookup, null); + Walker.buildPainlessTree( + scriptClassInfo, new MainMethodReserved(), getTestName(), script, pickySettings, PAINLESS_LOOKUP, null); } // test actual script execution ExecutableScript.Factory factory = scriptEngine.compile(null, script, ExecutableScript.CONTEXT, compileParams); diff --git a/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java b/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java index 512fc2345549c..9f36a600b68de 100644 --- a/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java +++ b/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.unconfigured_node_name; +import org.elasticsearch.bootstrap.BootstrapInfo; import org.elasticsearch.common.logging.NodeNameInLogsIntegTestCase; import java.io.IOException; @@ -32,6 +33,8 @@ public class NodeNameInLogsIT extends NodeNameInLogsIntegTestCase { @Override protected BufferedReader openReader(Path logFile) throws IOException { + assumeTrue("We log a line without the node name if we can't install the seccomp filters", + BootstrapInfo.isSystemCallFilterInstalled()); return AccessController.doPrivileged((PrivilegedAction) () -> { try { return Files.newBufferedReader(logFile, StandardCharsets.UTF_8); @@ -40,4 +43,11 @@ protected BufferedReader openReader(Path logFile) throws IOException { } }); } + + public void testDummy() { + /* Dummy test case so that when we run this test on a platform that + * does not support our syscall filters and we skip the test above + * we don't fail the entire test run because we skipped all the tests. + */ + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index d02d6272c9514..e7c5a07f56874 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -25,6 +25,8 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.StoredScriptSource; @@ -34,7 +36,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; -public class PutStoredScriptRequest extends AcknowledgedRequest { +public class PutStoredScriptRequest extends AcknowledgedRequest implements ToXContent { private String id; private String context; @@ -160,4 +162,12 @@ public String toString() { (context != null ? ", context [" + context + "]" : "") + ", content [" + source + "]}"; } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("script"); + source.toXContent(builder, params); + + return builder; + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index c3da63886140a..75869b54850d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -296,14 +296,38 @@ private ImmutableOpenMap> findAliases(String[] origi return ImmutableOpenMap.of(); } - boolean matchAllAliases = matchAllAliases(aliases); + String[] patterns = new String[aliases.length]; + boolean[] include = new boolean[aliases.length]; + for (int i = 0; i < aliases.length; i++) { + String alias = aliases[i]; + if (alias.charAt(0) == '-') { + patterns[i] = alias.substring(1); + include[i] = false; + } else { + patterns[i] = alias; + include[i] = true; + } + } + boolean matchAllAliases = patterns.length == 0; ImmutableOpenMap.Builder> mapBuilder = ImmutableOpenMap.builder(); for (String index : concreteIndices) { IndexMetaData indexMetaData = indices.get(index); List filteredValues = new ArrayList<>(); for (ObjectCursor cursor : indexMetaData.getAliases().values()) { AliasMetaData value = cursor.value; - if (matchAllAliases || Regex.simpleMatch(aliases, value.alias())) { + boolean matched = matchAllAliases; + String alias = value.alias(); + for (int i = 0; i < patterns.length; i++) { + if (include[i]) { + if (matched == false) { + String pattern = patterns[i]; + matched = ALL.equals(pattern) || Regex.simpleMatch(pattern, alias); + } + } else if (matched) { + matched = Regex.simpleMatch(patterns[i], alias) == false; + } + } + if (matched) { filteredValues.add(value); } } @@ -317,15 +341,6 @@ private ImmutableOpenMap> findAliases(String[] origi return mapBuilder.build(); } - private static boolean matchAllAliases(final String[] aliases) { - for (String alias : aliases) { - if (alias.equals(ALL)) { - return true; - } - } - return aliases.length == 0; - } - /** * Checks if at least one of the specified aliases exists in the specified concrete indices. Wildcards are supported in the * alias names for partial matches. diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index 4e7e81def8739..e9d805d34c8a1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -27,10 +27,11 @@ import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.RecoverySource.EmptyStoreRecoverySource; +import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -386,7 +387,7 @@ private Builder initializeAsRestore(IndexMetaData indexMetaData, SnapshotRecover if (asNew && ignoreShards.contains(shardNumber)) { // This shards wasn't completely snapshotted - restore it as new shard indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary, - primary ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo)); + primary ? EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo)); } else { indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary, primary ? recoverySource : PeerRecoverySource.INSTANCE, unassignedInfo)); @@ -410,13 +411,13 @@ private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unas final RecoverySource primaryRecoverySource; if (indexMetaData.inSyncAllocationIds(shardNumber).isEmpty() == false) { // we have previous valid copies for this shard. use them for recovery - primaryRecoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE; + primaryRecoverySource = ExistingStoreRecoverySource.INSTANCE; } else if (indexMetaData.getResizeSourceIndex() != null) { // this is a new index but the initial shards should merged from another index primaryRecoverySource = LocalShardsRecoverySource.INSTANCE; } else { // a freshly created index with no restriction - primaryRecoverySource = StoreRecoverySource.EMPTY_STORE_INSTANCE; + primaryRecoverySource = EmptyStoreRecoverySource.INSTANCE; } IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java index 13cb85ea399d5..2502fb0f3cc62 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java @@ -34,7 +34,8 @@ /** * Represents the recovery source of a shard. Available recovery types are: * - * - {@link StoreRecoverySource} recovery from the local store (empty or with existing data) + * - {@link EmptyStoreRecoverySource} recovery from an empty store + * - {@link ExistingStoreRecoverySource} recovery from an existing store * - {@link PeerRecoverySource} recovery from a primary on another node * - {@link SnapshotRecoverySource} recovery from a snapshot * - {@link LocalShardsRecoverySource} recovery from other shards of another index on the same node @@ -59,8 +60,8 @@ public void addAdditionalFields(XContentBuilder builder, ToXContent.Params param public static RecoverySource readFrom(StreamInput in) throws IOException { Type type = Type.values()[in.readByte()]; switch (type) { - case EMPTY_STORE: return StoreRecoverySource.EMPTY_STORE_INSTANCE; - case EXISTING_STORE: return StoreRecoverySource.EXISTING_STORE_INSTANCE; + case EMPTY_STORE: return EmptyStoreRecoverySource.INSTANCE; + case EXISTING_STORE: return new ExistingStoreRecoverySource(in); case PEER: return PeerRecoverySource.INSTANCE; case SNAPSHOT: return new SnapshotRecoverySource(in); case LOCAL_SHARDS: return LocalShardsRecoverySource.INSTANCE; @@ -91,6 +92,10 @@ public enum Type { public abstract Type getType(); + public boolean shouldBootstrapNewHistoryUUID() { + return false; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -107,25 +112,68 @@ public int hashCode() { } /** - * recovery from an existing on-disk store or a fresh copy + * Recovery from a fresh copy */ - public abstract static class StoreRecoverySource extends RecoverySource { - public static final StoreRecoverySource EMPTY_STORE_INSTANCE = new StoreRecoverySource() { - @Override - public Type getType() { - return Type.EMPTY_STORE; + public static final class EmptyStoreRecoverySource extends RecoverySource { + public static final EmptyStoreRecoverySource INSTANCE = new EmptyStoreRecoverySource(); + + @Override + public Type getType() { + return Type.EMPTY_STORE; + } + + @Override + public String toString() { + return "new shard recovery"; + } + } + + /** + * Recovery from an existing on-disk store + */ + public static final class ExistingStoreRecoverySource extends RecoverySource { + public static final ExistingStoreRecoverySource INSTANCE = new ExistingStoreRecoverySource(false); + public static final ExistingStoreRecoverySource FORCE_STALE_PRIMARY_INSTANCE = new ExistingStoreRecoverySource(true); + + private final boolean bootstrapNewHistoryUUID; + + private ExistingStoreRecoverySource(boolean bootstrapNewHistoryUUID) { + this.bootstrapNewHistoryUUID = bootstrapNewHistoryUUID; + } + + private ExistingStoreRecoverySource(StreamInput in) throws IOException { + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + bootstrapNewHistoryUUID = in.readBoolean(); + } else { + bootstrapNewHistoryUUID = false; } - }; - public static final StoreRecoverySource EXISTING_STORE_INSTANCE = new StoreRecoverySource() { - @Override - public Type getType() { - return Type.EXISTING_STORE; + } + + @Override + public void addAdditionalFields(XContentBuilder builder, Params params) throws IOException { + builder.field("bootstrap_new_history_uuid", bootstrapNewHistoryUUID); + } + + @Override + protected void writeAdditionalFields(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeBoolean(bootstrapNewHistoryUUID); } - }; + } + + @Override + public boolean shouldBootstrapNewHistoryUUID() { + return bootstrapNewHistoryUUID; + } + + @Override + public Type getType() { + return Type.EXISTING_STORE; + } @Override public String toString() { - return getType() == Type.EMPTY_STORE ? "new shard recovery" : "existing recovery"; + return "existing store recovery; bootstrap_history_uuid=" + bootstrapNewHistoryUUID; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index 6a9a105b6c432..74341ca271a9c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -19,14 +19,13 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.Index; @@ -318,7 +317,7 @@ public ShardRouting moveToUnassigned(UnassignedInfo unassignedInfo) { final RecoverySource recoverySource; if (active()) { if (primary()) { - recoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE; + recoverySource = ExistingStoreRecoverySource.INSTANCE; } else { recoverySource = PeerRecoverySource.INSTANCE; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java index 66281b73458b3..a42fd2765b598 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java @@ -21,7 +21,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; +import org.elasticsearch.cluster.routing.RecoverySource.EmptyStoreRecoverySource; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; @@ -136,7 +136,7 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain) } initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, unassignedInfoToUpdate, - StoreRecoverySource.EMPTY_STORE_INSTANCE); + EmptyStoreRecoverySource.INSTANCE); return new RerouteExplanation(this, allocation.decision(Decision.YES, name() + " (allocation command)", "ignore deciders")); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java index 11c4420200e33..f4c9aba17d71e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java @@ -129,7 +129,8 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain) "trying to allocate an existing primary shard [" + index + "][" + shardId + "], while no such shard has ever been active"); } - initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting); + initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, null, + RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE); return new RerouteExplanation(this, allocation.decision(Decision.YES, name() + " (allocation command)", "ignore deciders")); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index ae3f90e63e7d1..c4b971e470d66 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -398,6 +398,9 @@ private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRe indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPendingPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); } else if (indexShouldExists) { + if (recoveryState.getRecoverySource().shouldBootstrapNewHistoryUUID()) { + store.bootstrapNewHistory(); + } // since we recover from local, just fill the files and size try { final RecoveryState.Index index = recoveryState.getIndex(); diff --git a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java index f576667f44109..e1a413f6aa9bb 100644 --- a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -//TODO(simonw): can all these classes go into org.elasticsearch.ingest? package org.elasticsearch.ingest; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java index 2ca71fabbc7dc..821c75c2ed7d3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java @@ -20,8 +20,11 @@ package org.elasticsearch.action.admin.cluster.storedscripts; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.StoredScriptSource; import org.elasticsearch.test.ESTestCase; @@ -48,4 +51,30 @@ public void testSerialization() throws IOException { } } } + + public void testToXContent() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()); + builder.startObject(); + builder.startObject("script") + .field("lang", "painless") + .field("source", "Math.log(_score * 2) + params.multiplier") + .endObject(); + builder.endObject(); + + BytesReference expectedRequestBody = BytesReference.bytes(builder); + + PutStoredScriptRequest request = new PutStoredScriptRequest(); + request.id("test1"); + request.content(expectedRequestBody, xContentType); + + XContentBuilder requestBuilder = XContentBuilder.builder(xContentType.xContent()); + requestBuilder.startObject(); + request.toXContent(requestBuilder, ToXContent.EMPTY_PARAMS); + requestBuilder.endObject(); + + BytesReference actualRequestBody = BytesReference.bytes(requestBuilder); + + assertEquals(expectedRequestBody, actualRequestBody); + } } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index 1f7f6f4249b0c..95282e358e144 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -377,7 +377,7 @@ static GroupShardsIterator getShardsIter(String index, Orig ArrayList unassigned = new ArrayList<>(); ShardRouting routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), true, - RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")); + RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")); routing = routing.initialize(primaryNode.getId(), i + "p", 0); routing.started(); started.add(routing); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index 9d82e9e1cdca5..da50e99705dfb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -109,6 +109,38 @@ public void testFindAliases() { } } + public void testFindAliasWithExclusion() { + MetaData metaData = MetaData.builder().put( + IndexMetaData.builder("index") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetaData.builder("alias1").build()) + .putAlias(AliasMetaData.builder("alias2").build()) + ).build(); + List aliases = + metaData.findAliases(new GetAliasesRequest().aliases("*", "-alias1"), new String[] {"index"}).get("index"); + assertThat(aliases.size(), equalTo(1)); + assertThat(aliases.get(0).alias(), equalTo("alias2")); + } + + public void testFindAliasWithExclusionAndOverride() { + MetaData metaData = MetaData.builder().put( + IndexMetaData.builder("index") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetaData.builder("aa").build()) + .putAlias(AliasMetaData.builder("ab").build()) + .putAlias(AliasMetaData.builder("bb").build()) + ).build(); + List aliases = + metaData.findAliases(new GetAliasesRequest().aliases("a*", "-*b", "b*"), new String[] {"index"}).get("index"); + assertThat(aliases.size(), equalTo(2)); + assertThat(aliases.get(0).alias(), equalTo("aa")); + assertThat(aliases.get(1).alias(), equalTo("bb")); + } + public void testIndexAndAliasWithSameName() { IndexMetaData.Builder builder = IndexMetaData.builder("index") .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java index c1861572d8352..86dbeabd1d73e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java @@ -19,7 +19,7 @@ package org.elasticsearch.cluster.routing; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; +import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentFactory; @@ -37,7 +37,7 @@ public class AllocationIdTests extends ESTestCase { public void testShardToStarted() { logger.info("-- create unassigned shard"); - ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); assertThat(shard.allocationId(), nullValue()); logger.info("-- initialize the shard"); @@ -57,7 +57,7 @@ public void testShardToStarted() { public void testSuccessfulRelocation() { logger.info("-- build started shard"); - ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); shard = shard.initialize("node1", null, -1); shard = shard.moveToStarted(); @@ -80,7 +80,7 @@ public void testSuccessfulRelocation() { public void testCancelRelocation() { logger.info("-- build started shard"); - ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); shard = shard.initialize("node1", null, -1); shard = shard.moveToStarted(); @@ -100,7 +100,7 @@ public void testCancelRelocation() { public void testMoveToUnassigned() { logger.info("-- build started shard"); - ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); shard = shard.initialize("node1", null, -1); shard = shard.moveToStarted(); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java index f2571fce3391d..66eabd4cbd921 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java @@ -77,7 +77,7 @@ public void testIterate() { public ShardRouting newRouting(Index index, int id, boolean started) { ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, id), true, - RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); shardRouting = ShardRoutingHelper.initialize(shardRouting, "some node"); if (started) { shardRouting = ShardRoutingHelper.moveToStarted(shardRouting); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java index 9786c0eaf5290..9b2db5b34b1da 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.gateway.GatewayAllocator; +import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; @@ -55,6 +56,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -64,6 +66,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.isIn; import static org.hamcrest.Matchers.not; @@ -83,18 +86,9 @@ protected Settings nodeSettings(int nodeOrdinal) { .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false).build(); } - private void createStaleReplicaScenario() throws Exception { - logger.info("--> starting 3 nodes, 1 master, 2 data"); - String master = internalCluster().startMasterOnlyNode(Settings.EMPTY); - internalCluster().startDataOnlyNodes(2); - - assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() - .put("index.number_of_shards", 1).put("index.number_of_replicas", 1)).get()); - ensureGreen(); - logger.info("--> indexing..."); + private void createStaleReplicaScenario(String master) throws Exception { client().prepareIndex("test", "type1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); refresh(); - ClusterState state = client().admin().cluster().prepareState().all().get().getState(); List shards = state.routingTable().allShards("test"); assertThat(shards.size(), equalTo(2)); @@ -140,7 +134,13 @@ private void createStaleReplicaScenario() throws Exception { } public void testDoNotAllowStaleReplicasToBePromotedToPrimary() throws Exception { - createStaleReplicaScenario(); + logger.info("--> starting 3 nodes, 1 master, 2 data"); + String master = internalCluster().startMasterOnlyNode(Settings.EMPTY); + internalCluster().startDataOnlyNodes(2); + assertAcked(client().admin().indices().prepareCreate("test") + .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 1)).get()); + ensureGreen(); + createStaleReplicaScenario(master); logger.info("--> starting node that reuses data folder with the up-to-date primary shard"); internalCluster().startDataOnlyNode(Settings.EMPTY); @@ -176,9 +176,17 @@ public void testFailedAllocationOfStalePrimaryToDataNodeWithNoData() throws Exce } public void testForceStaleReplicaToBePromotedToPrimary() throws Exception { - boolean useStaleReplica = randomBoolean(); // if true, use stale replica, otherwise a completely empty copy - createStaleReplicaScenario(); + logger.info("--> starting 3 nodes, 1 master, 2 data"); + String master = internalCluster().startMasterOnlyNode(Settings.EMPTY); + internalCluster().startDataOnlyNodes(2); + assertAcked(client().admin().indices().prepareCreate("test") + .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 1)).get()); + ensureGreen(); + Set historyUUIDs = Arrays.stream(client().admin().indices().prepareStats("test").clear().get().getShards()) + .map(shard -> shard.getCommitStats().getUserData().get(Engine.HISTORY_UUID_KEY)).collect(Collectors.toSet()); + createStaleReplicaScenario(master); + boolean useStaleReplica = randomBoolean(); // if true, use stale replica, otherwise a completely empty copy logger.info("--> explicitly promote old primary shard"); final String idxName = "test"; ImmutableOpenIntMap> storeStatuses = client().admin().indices().prepareShardStores(idxName).get().getStoreStatuses().get(idxName); @@ -213,6 +221,11 @@ public void testForceStaleReplicaToBePromotedToPrimary() throws Exception { ClusterState state = client().admin().cluster().prepareState().get().getState(); assertEquals(Collections.singleton(state.routingTable().index(idxName).shard(0).primary.allocationId().getId()), state.metaData().index(idxName).inSyncAllocationIds(0)); + + Set newHistoryUUIds = Arrays.stream(client().admin().indices().prepareStats("test").clear().get().getShards()) + .map(shard -> shard.getCommitStats().getUserData().get(Engine.HISTORY_UUID_KEY)).collect(Collectors.toSet()); + assertThat(newHistoryUUIds, everyItem(not(isIn(historyUUIDs)))); + assertThat(newHistoryUUIds, hasSize(1)); } public void testForcePrimaryShardIfAllocationDecidersSayNoAfterIndexCreation() throws ExecutionException, InterruptedException { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java index d32ebe62ec1aa..01586d9c49575 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java @@ -399,7 +399,7 @@ private void addInSyncAllocationIds(Index index, IndexMetaData.Builder indexMeta final boolean primary = randomBoolean(); final ShardRouting unassigned = ShardRouting.newUnassigned(new ShardId(index, shard), primary, primary ? - RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : + RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test") ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index da0e0a9b0bcfa..ce53c14807c22 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -29,9 +29,9 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RecoverySource.EmptyStoreRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -69,7 +69,7 @@ public void testCanAllocateUsesMaxAvailableSpace() { final Index index = metaData.index("test").getIndex(); - ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(), @@ -125,22 +125,22 @@ public void testCanRemainUsesLeastAvailableSpace() { .build(); final IndexMetaData indexMetaData = metaData.index("test"); - ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 0), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); test_0 = ShardRoutingHelper.initialize(test_0, node_0.getId()); test_0 = ShardRoutingHelper.moveToStarted(test_0); shardRoutingMap.put(test_0, "/node0/least"); - ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 1), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 1), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); test_1 = ShardRoutingHelper.initialize(test_1, node_1.getId()); test_1 = ShardRoutingHelper.moveToStarted(test_1); shardRoutingMap.put(test_1, "/node1/least"); - ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 2), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 2), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); test_2 = ShardRoutingHelper.initialize(test_2, node_1.getId()); test_2 = ShardRoutingHelper.moveToStarted(test_2); shardRoutingMap.put(test_2, "/node1/most"); - ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 3), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 3), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); test_3 = ShardRoutingHelper.initialize(test_3, node_1.getId()); test_3 = ShardRoutingHelper.moveToStarted(test_3); // Intentionally not in the shardRoutingMap. We want to test what happens when we don't know where it is. diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 182747e7dda5d..87edfcfccb150 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -637,7 +637,7 @@ private static ShardRouting getInitializingShardRouting(ShardRouting existingSha existingShardRouting.currentNodeId(), null, existingShardRouting.primary(), ShardRoutingState.INITIALIZING, existingShardRouting.allocationId()); shardRouting = shardRouting.updateUnassigned(new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "fake recovery"), - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE); + RecoverySource.ExistingStoreRecoverySource.INSTANCE); return shardRouting; } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 713bc04634b0a..4ed74388f0e1e 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -178,6 +178,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -843,7 +844,7 @@ public void testGlobalCheckpointSync() throws IOException { randomAlphaOfLength(8), true, ShardRoutingState.INITIALIZING, - RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE); + RecoverySource.EmptyStoreRecoverySource.INSTANCE); final Settings settings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2) @@ -1199,7 +1200,7 @@ public void testShardStats() throws IOException { public void testShardStatsWithFailures() throws IOException { allowShardFailures(); final ShardId shardId = new ShardId("index", "_na_", 0); - final ShardRouting shardRouting = newShardRouting(shardId, "node", true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, ShardRoutingState.INITIALIZING); + final ShardRouting shardRouting = newShardRouting(shardId, "node", true, RecoverySource.EmptyStoreRecoverySource.INSTANCE, ShardRoutingState.INITIALIZING); final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir()); @@ -1659,7 +1660,7 @@ public void testRecoverFromStoreWithOutOfOrderDelete() throws IOException { final ShardRouting replicaRouting = shard.routingEntry(); IndexShard newShard = reinitShard(shard, newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE)); + RecoverySource.ExistingStoreRecoverySource.INSTANCE)); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue(newShard.recoverFromStore()); @@ -1684,6 +1685,7 @@ public void testRecoverFromStore() throws IOException { flushShard(shard); translogOps = 0; } + String historyUUID = shard.getHistoryUUID(); IndexShard newShard = reinitShard(shard); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); @@ -1698,6 +1700,29 @@ public void testRecoverFromStore() throws IOException { assertThat(newShard.getReplicationTracker().getTrackedLocalCheckpointForShard(newShard.routingEntry().allocationId().getId()) .getLocalCheckpoint(), equalTo(totalOps - 1L)); assertDocCount(newShard, totalOps); + assertThat(newShard.getHistoryUUID(), equalTo(historyUUID)); + closeShards(newShard); + } + + public void testRecoverFromStalePrimaryForceNewHistoryUUID() throws IOException { + final IndexShard shard = newStartedShard(true); + int totalOps = randomInt(10); + for (int i = 0; i < totalOps; i++) { + indexDoc(shard, "_doc", Integer.toString(i)); + } + if (randomBoolean()) { + shard.updateLocalCheckpointForShard(shard.shardRouting.allocationId().getId(), totalOps - 1); + flushShard(shard); + } + String historyUUID = shard.getHistoryUUID(); + IndexShard newShard = reinitShard(shard, newShardRouting(shard.shardId(), shard.shardRouting.currentNodeId(), true, + ShardRoutingState.INITIALIZING, RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE)); + DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); + newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); + assertTrue(newShard.recoverFromStore()); + IndexShardTestCase.updateRoutingEntry(newShard, newShard.routingEntry().moveToStarted()); + assertDocCount(newShard, totalOps); + assertThat(newShard.getHistoryUUID(), not(equalTo(historyUUID))); closeShards(newShard); } @@ -1734,7 +1759,7 @@ public void testRecoverFromStoreWithNoOps() throws IOException { final ShardRouting primaryShardRouting = shard.routingEntry(); IndexShard newShard = reinitShard(otherShard, ShardRoutingHelper.initWithSameId(primaryShardRouting, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE)); + RecoverySource.ExistingStoreRecoverySource.INSTANCE)); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue(newShard.recoverFromStore()); @@ -1760,7 +1785,7 @@ public void testRecoverFromStoreWithNoOps() throws IOException { for (int i = 0; i < 2; i++) { newShard = reinitShard(newShard, ShardRoutingHelper.initWithSameId(primaryShardRouting, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE)); + RecoverySource.ExistingStoreRecoverySource.INSTANCE)); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue(newShard.recoverFromStore()); try (Translog.Snapshot snapshot = getTranslog(newShard).newSnapshot()) { @@ -1778,7 +1803,7 @@ public void testRecoverFromCleanStore() throws IOException { } final ShardRouting shardRouting = shard.routingEntry(); IndexShard newShard = reinitShard(shard, - ShardRoutingHelper.initWithSameId(shardRouting, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE) + ShardRoutingHelper.initWithSameId(shardRouting, RecoverySource.EmptyStoreRecoverySource.INSTANCE) ); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); @@ -1827,7 +1852,7 @@ public void testFailIfIndexNotPresentInRecoverFromStore() throws Exception { } newShard = reinitShard(newShard, - ShardRoutingHelper.initWithSameId(routing, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE)); + ShardRoutingHelper.initWithSameId(routing, RecoverySource.EmptyStoreRecoverySource.INSTANCE)); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore()); @@ -1865,7 +1890,7 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { final ShardRouting replicaRouting = shard.routingEntry(); IndexShard newShard = reinitShard(shard, newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE)); + RecoverySource.ExistingStoreRecoverySource.INSTANCE)); newShard.pendingPrimaryTerm++; newShard.operationPrimaryTerm++; DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); @@ -1905,7 +1930,7 @@ public void testRestoreShard() throws IOException { assertDocs(target, "1"); flushShard(source); // only flush source ShardRouting routing = ShardRoutingHelper.initWithSameId(target.routingEntry(), - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE); + RecoverySource.ExistingStoreRecoverySource.INSTANCE); final Snapshot snapshot = new Snapshot("foo", new SnapshotId("bar", UUIDs.randomBase64UUID())); routing = ShardRoutingHelper.newWithRestoreSource(routing, new RecoverySource.SnapshotRecoverySource(snapshot, Version.CURRENT, "test")); @@ -1974,7 +1999,7 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { }; closeShards(shard); IndexShard newShard = newShard( - ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE), + ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE), shard.shardPath(), shard.indexSettings().getIndexMetaData(), null, @@ -2127,7 +2152,7 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { closeShards(shard); IndexShard newShard = newShard( - ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE), + ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE), shard.shardPath(), shard.indexSettings().getIndexMetaData(), null, @@ -2625,7 +2650,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO assertThat("corruption marker should not be there", corruptedMarkerCount.get(), equalTo(0)); final ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(indexShard.routingEntry(), - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE + RecoverySource.ExistingStoreRecoverySource.INSTANCE ); // start shard and perform index check on startup. It enforce shard to fail due to corrupted index files final IndexMetaData indexMetaData = IndexMetaData.builder(indexShard.indexSettings().getIndexMetaData()) @@ -2666,7 +2691,7 @@ public void testShardDoesNotStartIfCorruptedMarkerIsPresent() throws Exception { final ShardPath shardPath = indexShard.shardPath(); final ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(indexShard.routingEntry(), - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE + RecoverySource.ExistingStoreRecoverySource.INSTANCE ); final IndexMetaData indexMetaData = indexShard.indexSettings().getIndexMetaData(); @@ -2751,7 +2776,7 @@ public void testReadSnapshotAndCheckIndexConcurrently() throws Exception { closeShards(indexShard); final ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(indexShard.routingEntry(), - isPrimary ? RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE + isPrimary ? RecoverySource.ExistingStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE ); final IndexMetaData indexMetaData = IndexMetaData.builder(indexShard.indexSettings().getIndexMetaData()) .settings(Settings.builder() @@ -3261,7 +3286,7 @@ public void testFlushOnInactive() throws Exception { .settings(settings) .primaryTerm(0, 1).build(); ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(metaData.getIndex(), 0), "n1", true, ShardRoutingState - .INITIALIZING, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE); + .INITIALIZING, RecoverySource.EmptyStoreRecoverySource.INSTANCE); final ShardId shardId = shardRouting.shardId(); final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir()); ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java index aa06f9e9b7dfe..769cdfc8a9b53 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java @@ -128,7 +128,7 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem String nodeId = newRouting.currentNodeId(); UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom"); newRouting = newRouting.moveToUnassigned(unassignedInfo) - .updateUnassigned(unassignedInfo, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE); + .updateUnassigned(unassignedInfo, RecoverySource.EmptyStoreRecoverySource.INSTANCE); newRouting = ShardRoutingHelper.initialize(newRouting, nodeId); IndexShard shard = index.createShard(newRouting, s -> {}); IndexShardTestCase.updateRoutingEntry(shard, newRouting); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 213aa644665dd..6a6970675eb9b 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -36,7 +36,6 @@ import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -186,7 +185,7 @@ public void testGatewayRecovery() throws Exception { RecoveryState recoveryState = recoveryStates.get(0); - assertRecoveryState(recoveryState, 0, StoreRecoverySource.EXISTING_STORE_INSTANCE, true, Stage.DONE, null, node); + assertRecoveryState(recoveryState, 0, RecoverySource.ExistingStoreRecoverySource.INSTANCE, true, Stage.DONE, null, node); validateIndexRecoveryState(recoveryState.getIndex()); } @@ -239,7 +238,7 @@ public void testReplicaRecovery() throws Exception { // validate node A recovery RecoveryState nodeARecoveryState = nodeAResponses.get(0); - assertRecoveryState(nodeARecoveryState, 0, StoreRecoverySource.EMPTY_STORE_INSTANCE, true, Stage.DONE, null, nodeA); + assertRecoveryState(nodeARecoveryState, 0, RecoverySource.EmptyStoreRecoverySource.INSTANCE, true, Stage.DONE, null, nodeA); validateIndexRecoveryState(nodeARecoveryState.getIndex()); // validate node B recovery @@ -295,7 +294,7 @@ public void testRerouteRecovery() throws Exception { List nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates); assertThat(nodeBRecoveryStates.size(), equalTo(1)); - assertRecoveryState(nodeARecoveryStates.get(0), 0, StoreRecoverySource.EMPTY_STORE_INSTANCE, true, Stage.DONE, null, nodeA); + assertRecoveryState(nodeARecoveryStates.get(0), 0, RecoverySource.EmptyStoreRecoverySource.INSTANCE, true, Stage.DONE, null, nodeA); validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex()); assertOnGoingRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, nodeA, nodeB); diff --git a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java index f3a11a86e54e5..9111658e49ca8 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java @@ -92,12 +92,6 @@ public void testReadBooleanPropertyInvalidType() { } } - // TODO(talevy): Issue with generics. This test should fail, "int" is of type List - public void testOptional_InvalidType() { - List val = ConfigurationUtils.readList(null, null, config, "int"); - assertThat(val, equalTo(Collections.singletonList(2))); - } - public void testReadStringOrIntProperty() { String val1 = ConfigurationUtils.readStringOrIntProperty(null, null, config, "foo", null); String val2 = ConfigurationUtils.readStringOrIntProperty(null, null, config, "num", null); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java index 65139109a83a2..6e5d862372ac6 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java @@ -60,7 +60,6 @@ public class IngestClientIT extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { - // TODO: Remove this method once gets in: https://github.com/elastic/elasticsearch/issues/16019 if (nodeOrdinal % 2 == 0) { return Settings.builder().put("node.ingest", false).put(super.nodeSettings(nodeOrdinal)).build(); } diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index fa7de2d629112..ba3fa84a19641 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.TestUtil; import org.elasticsearch.cluster.metadata.RepositoryMetaData; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingHelper; import org.elasticsearch.common.UUIDs; @@ -49,7 +50,6 @@ import java.util.Arrays; import java.util.List; -import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; import static org.hamcrest.Matchers.containsString; /** @@ -99,7 +99,8 @@ public void testRestoreSnapshotWithExistingFiles() throws IOException { } // build a new shard using the same store directory as the closed shard - ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(shard.routingEntry(), EXISTING_STORE_INSTANCE); + ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(shard.routingEntry(), + RecoverySource.ExistingStoreRecoverySource.INSTANCE); shard = newShard( shardRouting, shard.shardPath(), diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java index cd592c9ed1e9c..4535bf7a91b0d 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java @@ -30,8 +30,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Table; @@ -143,7 +143,7 @@ private IndicesStatsResponse randomIndicesStatsResponse(final Index[] indices) { boolean primary = (i == primaryIdx); Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i)); ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, primary, - primary ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE, + primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null) ); shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java index 2291c3d39e200..c91c04884c5a7 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java @@ -88,8 +88,8 @@ private static RecoverySource buildRecoveryTarget(boolean primary, ShardRoutingS case UNASSIGNED: case INITIALIZING: if (primary) { - return ESTestCase.randomFrom(RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE); + return ESTestCase.randomFrom(RecoverySource.EmptyStoreRecoverySource.INSTANCE, + RecoverySource.ExistingStoreRecoverySource.INSTANCE); } else { return RecoverySource.PeerRecoverySource.INSTANCE; } @@ -130,8 +130,8 @@ private static UnassignedInfo buildUnassignedInfo(ShardRoutingState state) { } public static RecoverySource randomRecoverySource() { - return ESTestCase.randomFrom(RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE, + return ESTestCase.randomFrom(RecoverySource.EmptyStoreRecoverySource.INSTANCE, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, RecoverySource.PeerRecoverySource.INSTANCE, RecoverySource.LocalShardsRecoverySource.INSTANCE, new RecoverySource.SnapshotRecoverySource( diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index f2afdff9c3a3a..8717d7ba146fb 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -172,7 +172,7 @@ protected ReplicationGroup(final IndexMetaData indexMetaData) throws IOException private ShardRouting createShardRouting(String nodeId, boolean primary) { return TestShardRouting.newShardRouting(shardId, nodeId, primary, ShardRoutingState.INITIALIZING, - primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); + primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); } protected EngineFactory getEngineFactory(ShardRouting routing) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index 53576a1d80a70..9082b4153b0bf 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -193,7 +193,7 @@ protected IndexShard newShard(final boolean primary, final Settings settings) th */ protected IndexShard newShard(boolean primary, Settings settings, EngineFactory engineFactory) throws IOException { final RecoverySource recoverySource = - primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE; + primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE; final ShardRouting shardRouting = TestShardRouting.newShardRouting( new ShardId("index", "_na_", 0), randomAlphaOfLength(10), primary, ShardRoutingState.INITIALIZING, recoverySource); @@ -244,7 +244,7 @@ protected IndexShard newShard( protected IndexShard newShard(ShardId shardId, boolean primary, IndexingOperationListener... listeners) throws IOException { ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(5), primary, ShardRoutingState.INITIALIZING, - primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); + primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); return newShard(shardRouting, Settings.EMPTY, new InternalEngineFactory(), listeners); } @@ -272,7 +272,7 @@ protected IndexShard newShard(ShardId shardId, boolean primary, String nodeId, I protected IndexShard newShard(ShardId shardId, boolean primary, String nodeId, IndexMetaData indexMetaData, @Nullable IndexSearcherWrapper searcherWrapper, Runnable globalCheckpointSyncer) throws IOException { ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, nodeId, primary, ShardRoutingState.INITIALIZING, - primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); + primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); return newShard(shardRouting, indexMetaData, searcherWrapper, new InternalEngineFactory(), globalCheckpointSyncer); } @@ -371,7 +371,7 @@ protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMe protected IndexShard reinitShard(IndexShard current, IndexingOperationListener... listeners) throws IOException { final ShardRouting shardRouting = current.routingEntry(); return reinitShard(current, ShardRoutingHelper.initWithSameId(shardRouting, - shardRouting.primary() ? RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE + shardRouting.primary() ? RecoverySource.ExistingStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE ), listeners); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java b/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java index a53ba046d32c3..895bd7ec77a2b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java +++ b/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java @@ -85,7 +85,7 @@ public AbstractEventExpectation(String name, String logger, Level level, String @Override public void match(LogEvent event) { - if (event.getLevel().equals(level) && event.getLoggerName().equals(logger)) { + if (event.getLevel().equals(level) && event.getLoggerName().equals(logger) && innerMatch(event)) { if (Regex.isSimpleMatchPattern(message)) { if (Regex.simpleMatch(message, event.getMessage().getFormattedMessage())) { saw = true; @@ -97,6 +97,11 @@ public void match(LogEvent event) { } } } + + public boolean innerMatch(final LogEvent event) { + return true; + } + } public static class UnseenEventExpectation extends AbstractEventExpectation { @@ -123,6 +128,32 @@ public void assertMatched() { } } + public static class ExceptionSeenEventExpectation extends SeenEventExpectation { + + private final Class clazz; + private final String exceptionMessage; + + public ExceptionSeenEventExpectation( + final String name, + final String logger, + final Level level, + final String message, + final Class clazz, + final String exceptionMessage) { + super(name, logger, level, message); + this.clazz = clazz; + this.exceptionMessage = exceptionMessage; + } + + @Override + public boolean innerMatch(final LogEvent event) { + return event.getThrown() != null + && event.getThrown().getClass() == clazz + && event.getThrown().getMessage().equals(exceptionMessage); + } + + } + public static class PatternSeenEventExcpectation implements LoggingExpectation { protected final String name; diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/build.gradle b/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle similarity index 55% rename from x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/build.gradle rename to x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle index 1566333e60848..845c9df533dba 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/build.gradle +++ b/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle @@ -20,7 +20,24 @@ leaderClusterTestRunner { systemProperty 'tests.is_leader_cluster', 'true' } +task writeJavaPolicy { + doLast { + final File tmp = file("${buildDir}/tmp") + if (tmp.exists() == false && tmp.mkdirs() == false) { + throw new GradleException("failed to create temporary directory [${tmp}]") + } + final File javaPolicy = file("${tmp}/java.policy") + javaPolicy.write( + [ + "grant {", + " permission java.io.FilePermission \"${-> followClusterTest.getNodes().get(0).homeDir}/logs/${-> followClusterTest.getNodes().get(0).clusterName}.log\", \"read\";", + "};" + ].join("\n")) + } +} + task followClusterTest(type: RestIntegTestTask) {} +followClusterTest.dependsOn writeJavaPolicy followClusterTestCluster { dependsOn leaderClusterTestRunner @@ -31,8 +48,10 @@ followClusterTestCluster { } followClusterTestRunner { + systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy" systemProperty 'tests.is_leader_cluster', 'false' systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}" + systemProperty 'log', "${-> followClusterTest.getNodes().get(0).homeDir}/logs/${-> followClusterTest.getNodes().get(0).clusterName}.log" finalizedBy 'leaderClusterTestCluster#stop' } diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java b/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java similarity index 50% rename from x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java rename to x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java index 06d9f91c7abb7..c52a4a9b59d78 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java @@ -9,11 +9,16 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.test.rest.ESRestTestCase; +import java.nio.file.Files; +import java.util.Iterator; +import java.util.List; import java.util.Locale; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; public class CcrMultiClusterLicenseIT extends ESRestTestCase { @@ -29,7 +34,7 @@ public void testFollowIndex() { if (runningAgainstLeaderCluster == false) { final Request request = new Request("POST", "/follower/_ccr/follow"); request.setJsonEntity("{\"leader_index\": \"leader_cluster:leader\"}"); - assertLicenseIncompatible(request); + assertNonCompliantLicense(request); } } @@ -37,11 +42,44 @@ public void testCreateAndFollowIndex() { if (runningAgainstLeaderCluster == false) { final Request request = new Request("POST", "/follower/_ccr/create_and_follow"); request.setJsonEntity("{\"leader_index\": \"leader_cluster:leader\"}"); - assertLicenseIncompatible(request); + assertNonCompliantLicense(request); } } - private static void assertLicenseIncompatible(final Request request) { + public void testAutoFollow() throws Exception { + if (runningAgainstLeaderCluster == false) { + final Request request = new Request("PUT", "/_ccr/_auto_follow/leader_cluster"); + request.setJsonEntity("{\"leader_index_patterns\":[\"*\"]}"); + client().performRequest(request); + + // parse the logs and ensure that the auto-coordinator skipped coordination on the leader cluster + assertBusy(() -> { + final List lines = Files.readAllLines(PathUtils.get(System.getProperty("log"))); + + final Iterator it = lines.iterator(); + + boolean warn = false; + while (it.hasNext()) { + final String line = it.next(); + if (line.matches(".*\\[WARN\\s*\\]\\[o\\.e\\.x\\.c\\.a\\.AutoFollowCoordinator\\s*\\] \\[node-0\\] " + + "failure occurred during auto-follower coordination")) { + warn = true; + break; + } + } + assertTrue(warn); + assertTrue(it.hasNext()); + final String lineAfterWarn = it.next(); + assertThat( + lineAfterWarn, + equalTo("org.elasticsearch.ElasticsearchStatusException: " + + "can not fetch remote cluster state as the remote cluster [leader_cluster] is not licensed for [ccr]; " + + "the license mode [BASIC] on cluster [leader_cluster] does not enable [ccr]")); + }); + } + } + + private static void assertNonCompliantLicense(final Request request) { final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request)); final String expected = String.format( Locale.ROOT, diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index cd0561b1c0c60..353a66db26339 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -126,7 +126,7 @@ public Collection createComponents( return Arrays.asList( ccrLicenseChecker, - new AutoFollowCoordinator(settings, client, threadPool, clusterService) + new AutoFollowCoordinator(settings, client, threadPool, clusterService, ccrLicenseChecker) ); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java index cefa490f4f7e2..f9a5d8fe83035 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java @@ -23,6 +23,7 @@ import java.util.Objects; import java.util.function.BooleanSupplier; import java.util.function.Consumer; +import java.util.function.Function; /** * Encapsulates licensing checking for CCR. @@ -58,14 +59,13 @@ public boolean isCcrAllowed() { /** * Fetches the leader index metadata from the remote cluster. Before fetching the index metadata, the remote cluster is checked for - * license compatibility with CCR. If the remote cluster is not licensed for CCR, the {@link ActionListener#onFailure(Exception)} method - * of the specified listener is invoked. Otherwise, the specified consumer is invoked with the leader index metadata fetched from the - * remote cluster. + * license compatibility with CCR. If the remote cluster is not licensed for CCR, the {@code onFailure} consumer is is invoked. + * Otherwise, the specified consumer is invoked with the leader index metadata fetched from the remote cluster. * * @param client the client * @param clusterAlias the remote cluster alias * @param leaderIndex the name of the leader index - * @param listener the listener + * @param onFailure the failure consumer * @param leaderIndexMetadataConsumer the leader index metadata consumer * @param the type of response the listener is waiting for */ @@ -73,8 +73,75 @@ public void checkRemoteClusterLicenseAndFetchLeaderIndexMetadata( final Client client, final String clusterAlias, final String leaderIndex, - final ActionListener listener, + final Consumer onFailure, final Consumer leaderIndexMetadataConsumer) { + + final ClusterStateRequest request = new ClusterStateRequest(); + request.clear(); + request.metaData(true); + request.indices(leaderIndex); + checkRemoteClusterLicenseAndFetchClusterState( + client, + clusterAlias, + request, + onFailure, + leaderClusterState -> leaderIndexMetadataConsumer.accept(leaderClusterState.getMetaData().index(leaderIndex)), + licenseCheck -> indexMetadataNonCompliantRemoteLicense(leaderIndex, licenseCheck), + e -> indexMetadataUnknownRemoteLicense(leaderIndex, clusterAlias, e)); + } + + /** + * Fetches the leader cluster state from the remote cluster by the specified cluster state request. Before fetching the cluster state, + * the remote cluster is checked for license compliance with CCR. If the remote cluster is not licensed for CCR, + * the {@code onFailure} consumer is invoked. Otherwise, the specified consumer is invoked with the leader cluster state fetched from + * the remote cluster. + * + * @param client the client + * @param clusterAlias the remote cluster alias + * @param request the cluster state request + * @param onFailure the failure consumer + * @param leaderClusterStateConsumer the leader cluster state consumer + * @param the type of response the listener is waiting for + */ + public void checkRemoteClusterLicenseAndFetchClusterState( + final Client client, + final String clusterAlias, + final ClusterStateRequest request, + final Consumer onFailure, + final Consumer leaderClusterStateConsumer) { + checkRemoteClusterLicenseAndFetchClusterState( + client, + clusterAlias, + request, + onFailure, + leaderClusterStateConsumer, + CcrLicenseChecker::clusterStateNonCompliantRemoteLicense, + e -> clusterStateUnknownRemoteLicense(clusterAlias, e)); + } + + /** + * Fetches the leader cluster state from the remote cluster by the specified cluster state request. Before fetching the cluster state, + * the remote cluster is checked for license compliance with CCR. If the remote cluster is not licensed for CCR, + * the {@code onFailure} consumer is invoked. Otherwise, the specified consumer is invoked with the leader cluster state fetched from + * the remote cluster. + * + * @param client the client + * @param clusterAlias the remote cluster alias + * @param request the cluster state request + * @param onFailure the failure consumer + * @param leaderClusterStateConsumer the leader cluster state consumer + * @param nonCompliantLicense the supplier for when the license state of the remote cluster is non-compliant + * @param unknownLicense the supplier for when the license state of the remote cluster is unknown due to failure + * @param the type of response the listener is waiting for + */ + private void checkRemoteClusterLicenseAndFetchClusterState( + final Client client, + final String clusterAlias, + final ClusterStateRequest request, + final Consumer onFailure, + final Consumer leaderClusterStateConsumer, + final Function nonCompliantLicense, + final Function unknownLicense) { // we have to check the license on the remote cluster new RemoteClusterLicenseChecker(client, XPackLicenseState::isCcrAllowedForOperationMode).checkRemoteClusterLicenses( Collections.singletonList(clusterAlias), @@ -83,35 +150,25 @@ public void checkRemoteClusterLicenseAndFetchLeaderIndexMetadata( @Override public void onResponse(final RemoteClusterLicenseChecker.LicenseCheck licenseCheck) { if (licenseCheck.isSuccess()) { - final Client remoteClient = client.getRemoteClusterClient(clusterAlias); - final ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); - clusterStateRequest.clear(); - clusterStateRequest.metaData(true); - clusterStateRequest.indices(leaderIndex); - final ActionListener clusterStateListener = ActionListener.wrap( - r -> { - final ClusterState remoteClusterState = r.getState(); - final IndexMetaData leaderIndexMetadata = - remoteClusterState.getMetaData().index(leaderIndex); - leaderIndexMetadataConsumer.accept(leaderIndexMetadata); - }, - listener::onFailure); + final Client leaderClient = client.getRemoteClusterClient(clusterAlias); + final ActionListener clusterStateListener = + ActionListener.wrap(s -> leaderClusterStateConsumer.accept(s.getState()), onFailure); // following an index in remote cluster, so use remote client to fetch leader index metadata - remoteClient.admin().cluster().state(clusterStateRequest, clusterStateListener); + leaderClient.admin().cluster().state(request, clusterStateListener); } else { - listener.onFailure(incompatibleRemoteLicense(leaderIndex, licenseCheck)); + onFailure.accept(nonCompliantLicense.apply(licenseCheck)); } } @Override public void onFailure(final Exception e) { - listener.onFailure(unknownRemoteLicense(leaderIndex, clusterAlias, e)); + onFailure.accept(unknownLicense.apply(e)); } }); } - private static ElasticsearchStatusException incompatibleRemoteLicense( + private static ElasticsearchStatusException indexMetadataNonCompliantRemoteLicense( final String leaderIndex, final RemoteClusterLicenseChecker.LicenseCheck licenseCheck) { final String clusterAlias = licenseCheck.remoteClusterLicenseInfo().clusterAlias(); final String message = String.format( @@ -127,7 +184,21 @@ private static ElasticsearchStatusException incompatibleRemoteLicense( return new ElasticsearchStatusException(message, RestStatus.BAD_REQUEST); } - private static ElasticsearchStatusException unknownRemoteLicense( + private static ElasticsearchStatusException clusterStateNonCompliantRemoteLicense( + final RemoteClusterLicenseChecker.LicenseCheck licenseCheck) { + final String clusterAlias = licenseCheck.remoteClusterLicenseInfo().clusterAlias(); + final String message = String.format( + Locale.ROOT, + "can not fetch remote cluster state as the remote cluster [%s] is not licensed for [ccr]; %s", + clusterAlias, + RemoteClusterLicenseChecker.buildErrorMessage( + "ccr", + licenseCheck.remoteClusterLicenseInfo(), + RemoteClusterLicenseChecker::isLicensePlatinumOrTrial)); + return new ElasticsearchStatusException(message, RestStatus.BAD_REQUEST); + } + + private static ElasticsearchStatusException indexMetadataUnknownRemoteLicense( final String leaderIndex, final String clusterAlias, final Exception cause) { final String message = String.format( Locale.ROOT, @@ -138,4 +209,11 @@ private static ElasticsearchStatusException unknownRemoteLicense( return new ElasticsearchStatusException(message, RestStatus.BAD_REQUEST, cause); } + private static ElasticsearchStatusException clusterStateUnknownRemoteLicense(final String clusterAlias, final Exception cause) { + final String message = String.format( + Locale.ROOT, + "can not fetch remote cluster state as the license state of the remote cluster [%s] could not be determined", clusterAlias); + return new ElasticsearchStatusException(message, RestStatus.BAD_REQUEST, cause); + } + } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java index 234fe32cdd0ee..639cd4d5782ab 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java @@ -21,7 +21,9 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.index.Index; +import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; @@ -30,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -47,22 +50,32 @@ public class AutoFollowCoordinator implements ClusterStateApplier { private final TimeValue pollInterval; private final ThreadPool threadPool; private final ClusterService clusterService; + private final CcrLicenseChecker ccrLicenseChecker; private volatile boolean localNodeMaster = false; - public AutoFollowCoordinator(Settings settings, - Client client, - ThreadPool threadPool, - ClusterService clusterService) { + public AutoFollowCoordinator( + Settings settings, + Client client, + ThreadPool threadPool, + ClusterService clusterService, + CcrLicenseChecker ccrLicenseChecker) { this.client = client; this.threadPool = threadPool; this.clusterService = clusterService; + this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker, "ccrLicenseChecker"); this.pollInterval = CcrSettings.CCR_AUTO_FOLLOW_POLL_INTERVAL.get(settings); clusterService.addStateApplier(this); } private void doAutoFollow() { + if (ccrLicenseChecker.isCcrAllowed() == false) { + // TODO: set non-compliant status on auto-follow coordination that can be viewed via a stats API + LOGGER.warn("skipping auto-follower coordination", LicenseUtils.newComplianceException("ccr")); + threadPool.schedule(pollInterval, ThreadPool.Names.SAME, this::doAutoFollow); + return; + } if (localNodeMaster == false) { return; } @@ -80,23 +93,32 @@ private void doAutoFollow() { Consumer handler = e -> { if (e != null) { - LOGGER.warn("Failure occurred during auto following indices", e); + LOGGER.warn("failure occurred during auto-follower coordination", e); } threadPool.schedule(pollInterval, ThreadPool.Names.SAME, this::doAutoFollow); }; - AutoFollower operation = new AutoFollower(client, handler, followerClusterState) { + AutoFollower operation = new AutoFollower(handler, followerClusterState) { @Override - void getLeaderClusterState(Client leaderClient, BiConsumer handler) { - ClusterStateRequest request = new ClusterStateRequest(); + void getLeaderClusterState(final String leaderClusterAlias, final BiConsumer handler) { + final ClusterStateRequest request = new ClusterStateRequest(); request.clear(); request.metaData(true); - leaderClient.admin().cluster().state(request, - ActionListener.wrap( - r -> handler.accept(r.getState(), null), - e -> handler.accept(null, e) - ) - ); + + if ("_local_".equals(leaderClusterAlias)) { + client.admin().cluster().state( + request, ActionListener.wrap(r -> handler.accept(r.getState(), null), e -> handler.accept(null, e))); + } else { + final Client leaderClient = client.getRemoteClusterClient(leaderClusterAlias); + // TODO: set non-compliant status on auto-follow coordination that can be viewed via a stats API + ccrLicenseChecker.checkRemoteClusterLicenseAndFetchClusterState( + leaderClient, + leaderClusterAlias, + request, + e -> handler.accept(null, e), + leaderClusterState -> handler.accept(leaderClusterState, null)); + } + } @Override @@ -143,7 +165,6 @@ public void applyClusterState(ClusterChangedEvent event) { abstract static class AutoFollower { - private final Client client; private final Consumer handler; private final ClusterState followerClusterState; private final AutoFollowMetadata autoFollowMetadata; @@ -151,8 +172,7 @@ abstract static class AutoFollower { private final CountDown autoFollowPatternsCountDown; private final AtomicReference autoFollowPatternsErrorHolder = new AtomicReference<>(); - AutoFollower(Client client, Consumer handler, ClusterState followerClusterState) { - this.client = client; + AutoFollower(final Consumer handler, final ClusterState followerClusterState) { this.handler = handler; this.followerClusterState = followerClusterState; this.autoFollowMetadata = followerClusterState.getMetaData().custom(AutoFollowMetadata.TYPE); @@ -163,10 +183,9 @@ void autoFollowIndices() { for (Map.Entry entry : autoFollowMetadata.getPatterns().entrySet()) { String clusterAlias = entry.getKey(); AutoFollowPattern autoFollowPattern = entry.getValue(); - Client leaderClient = clusterAlias.equals("_local_") ? client : client.getRemoteClusterClient(clusterAlias); List followedIndices = autoFollowMetadata.getFollowedLeaderIndexUUIDs().get(clusterAlias); - getLeaderClusterState(leaderClient, (leaderClusterState, e) -> { + getLeaderClusterState(clusterAlias, (leaderClusterState, e) -> { if (leaderClusterState != null) { assert e == null; handleClusterAlias(clusterAlias, autoFollowPattern, followedIndices, leaderClusterState); @@ -289,18 +308,17 @@ static Function recordLeaderIndexAsFollowFunction(St }; } - // abstract methods to make unit testing possible: - - abstract void getLeaderClusterState(Client leaderClient, - BiConsumer handler); + /** + * Fetch the cluster state from the leader with the specified cluster alias + * + * @param leaderClusterAlias the cluster alias of the leader + * @param handler the callback to invoke + */ + abstract void getLeaderClusterState(String leaderClusterAlias, BiConsumer handler); - abstract void createAndFollow(FollowIndexAction.Request followRequest, - Runnable successHandler, - Consumer failureHandler); + abstract void createAndFollow(FollowIndexAction.Request followRequest, Runnable successHandler, Consumer failureHandler); - abstract void updateAutoFollowMetadata(Function updateFunction, - Consumer handler); + abstract void updateAutoFollowMetadata(Function updateFunction, Consumer handler); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java index 2e36bca293225..1e14eb8979fb7 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java @@ -221,21 +221,21 @@ protected Response newResponse() { @Override protected void masterOperation( final Request request, final ClusterState state, final ActionListener listener) throws Exception { - if (ccrLicenseChecker.isCcrAllowed()) { - final String[] indices = new String[]{request.getFollowRequest().getLeaderIndex()}; - final Map> remoteClusterIndices = remoteClusterService.groupClusterIndices(indices, s -> false); - if (remoteClusterIndices.containsKey(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { - createFollowerIndexAndFollowLocalIndex(request, state, listener); - } else { - assert remoteClusterIndices.size() == 1; - final Map.Entry> entry = remoteClusterIndices.entrySet().iterator().next(); - assert entry.getValue().size() == 1; - final String clusterAlias = entry.getKey(); - final String leaderIndex = entry.getValue().get(0); - createFollowerIndexAndFollowRemoteIndex(request, clusterAlias, leaderIndex, listener); - } - } else { + if (ccrLicenseChecker.isCcrAllowed() == false) { listener.onFailure(LicenseUtils.newComplianceException("ccr")); + return; + } + final String[] indices = new String[]{request.getFollowRequest().getLeaderIndex()}; + final Map> remoteClusterIndices = remoteClusterService.groupClusterIndices(indices, s -> false); + if (remoteClusterIndices.containsKey(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + createFollowerIndexAndFollowLocalIndex(request, state, listener); + } else { + assert remoteClusterIndices.size() == 1; + final Map.Entry> entry = remoteClusterIndices.entrySet().iterator().next(); + assert entry.getValue().size() == 1; + final String clusterAlias = entry.getKey(); + final String leaderIndex = entry.getValue().get(0); + createFollowerIndexAndFollowRemoteIndex(request, clusterAlias, leaderIndex, listener); } } @@ -255,7 +255,7 @@ private void createFollowerIndexAndFollowRemoteIndex( client, clusterAlias, leaderIndex, - listener, + listener::onFailure, leaderIndexMetaData -> createFollowerIndex(leaderIndexMetaData, request, listener)); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java index 17b7bbe674b38..498224551106d 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java @@ -328,21 +328,21 @@ public TransportAction( protected void doExecute(final Task task, final Request request, final ActionListener listener) { - if (ccrLicenseChecker.isCcrAllowed()) { - final String[] indices = new String[]{request.leaderIndex}; - final Map> remoteClusterIndices = remoteClusterService.groupClusterIndices(indices, s -> false); - if (remoteClusterIndices.containsKey(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { - followLocalIndex(request, listener); - } else { - assert remoteClusterIndices.size() == 1; - final Map.Entry> entry = remoteClusterIndices.entrySet().iterator().next(); - assert entry.getValue().size() == 1; - final String clusterAlias = entry.getKey(); - final String leaderIndex = entry.getValue().get(0); - followRemoteIndex(request, clusterAlias, leaderIndex, listener); - } - } else { + if (ccrLicenseChecker.isCcrAllowed() == false) { listener.onFailure(LicenseUtils.newComplianceException("ccr")); + return; + } + final String[] indices = new String[]{request.leaderIndex}; + final Map> remoteClusterIndices = remoteClusterService.groupClusterIndices(indices, s -> false); + if (remoteClusterIndices.containsKey(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + followLocalIndex(request, listener); + } else { + assert remoteClusterIndices.size() == 1; + final Map.Entry> entry = remoteClusterIndices.entrySet().iterator().next(); + assert entry.getValue().size() == 1; + final String clusterAlias = entry.getKey(); + final String leaderIndex = entry.getValue().get(0); + followRemoteIndex(request, clusterAlias, leaderIndex, listener); } } @@ -370,7 +370,7 @@ private void followRemoteIndex( client, clusterAlias, leaderIndex, - listener, + listener::onFailure, leaderIndexMetadata -> { try { start(request, clusterAlias, leaderIndexMetadata, followerIndexMetadata, listener); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java index 33873201f5fb3..3b5d0ac53cf81 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java @@ -65,11 +65,11 @@ protected void doExecute( final Task task, final CcrStatsAction.TasksRequest request, final ActionListener listener) { - if (ccrLicenseChecker.isCcrAllowed()) { - super.doExecute(task, request, listener); - } else { + if (ccrLicenseChecker.isCcrAllowed() == false) { listener.onFailure(LicenseUtils.newComplianceException("ccr")); + return; } + super.doExecute(task, request, listener); } @Override diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java index 3d3e342c0cd3e..a4ff9511cfbd8 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java @@ -21,8 +21,10 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; @@ -30,20 +32,29 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; public class TransportPutAutoFollowPatternAction extends TransportMasterNodeAction { private final Client client; + private final CcrLicenseChecker ccrLicenseChecker; @Inject - public TransportPutAutoFollowPatternAction(Settings settings, TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, ActionFilters actionFilters, Client client, - IndexNameExpressionResolver indexNameExpressionResolver) { + public TransportPutAutoFollowPatternAction( + final Settings settings, + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final Client client, + final IndexNameExpressionResolver indexNameExpressionResolver, + final CcrLicenseChecker ccrLicenseChecker) { super(settings, PutAutoFollowPatternAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, PutAutoFollowPatternAction.Request::new); this.client = client; + this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker, "ccrLicenseChecker"); } @Override @@ -60,6 +71,10 @@ protected AcknowledgedResponse newResponse() { protected void masterOperation(PutAutoFollowPatternAction.Request request, ClusterState state, ActionListener listener) throws Exception { + if (ccrLicenseChecker.isCcrAllowed() == false) { + listener.onFailure(LicenseUtils.newComplianceException("ccr")); + return; + } final Client leaderClient; if (request.getLeaderClusterAlias().equals("_local_")) { leaderClient = client; @@ -71,22 +86,26 @@ protected void masterOperation(PutAutoFollowPatternAction.Request request, clusterStateRequest.clear(); clusterStateRequest.metaData(true); - leaderClient.admin().cluster().state(clusterStateRequest, ActionListener.wrap(clusterStateResponse -> { - final ClusterState leaderClusterState = clusterStateResponse.getState(); - clusterService.submitStateUpdateTask("put-auto-follow-pattern-" + request.getLeaderClusterAlias(), - new AckedClusterStateUpdateTask(request, listener) { - - @Override - protected AcknowledgedResponse newResponse(boolean acknowledged) { - return new AcknowledgedResponse(acknowledged); - } - - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - return innerPut(request, currentState, leaderClusterState); - } - }); - }, listener::onFailure)); + leaderClient.admin().cluster().state( + clusterStateRequest, + ActionListener.wrap( + clusterStateResponse -> { + final ClusterState leaderClusterState = clusterStateResponse.getState(); + clusterService.submitStateUpdateTask("put-auto-follow-pattern-" + request.getLeaderClusterAlias(), + new AckedClusterStateUpdateTask(request, listener) { + + @Override + protected AcknowledgedResponse newResponse(boolean acknowledged) { + return new AcknowledgedResponse(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return innerPut(request, currentState, leaderClusterState); + } + }); + }, + listener::onFailure)); } static ClusterState innerPut(PutAutoFollowPatternAction.Request request, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java index 675758903bf27..06cafc4777a49 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java @@ -6,15 +6,22 @@ package org.elasticsearch.xpack.ccr; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.MockLogAppender; +import org.elasticsearch.xpack.ccr.action.AutoFollowCoordinator; import org.elasticsearch.xpack.ccr.action.CcrStatsAction; import org.elasticsearch.xpack.ccr.action.CreateAndFollowIndexAction; import org.elasticsearch.xpack.ccr.action.FollowIndexAction; +import org.elasticsearch.xpack.ccr.action.PutAutoFollowPatternAction; import org.elasticsearch.xpack.ccr.action.ShardFollowNodeTask; import java.util.Collection; @@ -28,10 +35,10 @@ public class CcrLicenseIT extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return Collections.singletonList(IncompatibleLicenseLocalStateCcr.class); + return Collections.singletonList(NonCompliantLicenseLocalStateCcr.class); } - public void testThatFollowingIndexIsUnavailableWithIncompatibleLicense() throws InterruptedException { + public void testThatFollowingIndexIsUnavailableWithNonCompliantLicense() throws InterruptedException { final FollowIndexAction.Request followRequest = getFollowRequest(); final CountDownLatch latch = new CountDownLatch(1); client().execute( @@ -40,19 +47,20 @@ public void testThatFollowingIndexIsUnavailableWithIncompatibleLicense() throws new ActionListener() { @Override public void onResponse(final AcknowledgedResponse response) { + latch.countDown(); fail(); } @Override public void onFailure(final Exception e) { - assertIncompatibleLicense(e); + assertNonCompliantLicense(e); latch.countDown(); } }); latch.await(); } - public void testThatCreateAndFollowingIndexIsUnavailableWithIncompatibleLicense() throws InterruptedException { + public void testThatCreateAndFollowingIndexIsUnavailableWithNonCompliantLicense() throws InterruptedException { final FollowIndexAction.Request followRequest = getFollowRequest(); final CreateAndFollowIndexAction.Request createAndFollowRequest = new CreateAndFollowIndexAction.Request(followRequest); final CountDownLatch latch = new CountDownLatch(1); @@ -62,29 +70,31 @@ public void testThatCreateAndFollowingIndexIsUnavailableWithIncompatibleLicense( new ActionListener() { @Override public void onResponse(final CreateAndFollowIndexAction.Response response) { + latch.countDown(); fail(); } @Override public void onFailure(final Exception e) { - assertIncompatibleLicense(e); + assertNonCompliantLicense(e); latch.countDown(); } }); latch.await(); } - public void testThatCcrStatsAreUnavailableWithIncompatibleLicense() throws InterruptedException { + public void testThatCcrStatsAreUnavailableWithNonCompliantLicense() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(1); client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.TasksRequest(), new ActionListener() { @Override public void onResponse(final CcrStatsAction.TasksResponse tasksResponse) { + latch.countDown(); fail(); } @Override public void onFailure(final Exception e) { - assertIncompatibleLicense(e); + assertNonCompliantLicense(e); latch.countDown(); } }); @@ -92,7 +102,52 @@ public void onFailure(final Exception e) { latch.await(); } - private void assertIncompatibleLicense(final Exception e) { + public void testThatPutAutoFollowPatternsIsUnavailableWithNonCompliantLicense() throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + final PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + request.setLeaderClusterAlias("leader"); + request.setLeaderIndexPatterns(Collections.singletonList("*")); + client().execute( + PutAutoFollowPatternAction.INSTANCE, + request, + new ActionListener() { + @Override + public void onResponse(final AcknowledgedResponse response) { + latch.countDown(); + fail(); + } + + @Override + public void onFailure(final Exception e) { + assertNonCompliantLicense(e); + latch.countDown(); + } + }); + latch.await(); + } + + public void testAutoFollowCoordinatorLogsSkippingAutoFollowCoordinationWithNonCompliantLicense() throws Exception { + final Logger logger = LogManager.getLogger(AutoFollowCoordinator.class); + final MockLogAppender appender = new MockLogAppender(); + appender.start(); + appender.addExpectation( + new MockLogAppender.ExceptionSeenEventExpectation( + getTestName(), + logger.getName(), + Level.WARN, + "skipping auto-follower coordination", + ElasticsearchSecurityException.class, + "current license is non-compliant for [ccr]")); + Loggers.addAppender(logger, appender); + try { + assertBusy(appender::assertAllExpectationsMatched); + } finally { + Loggers.removeAppender(logger, appender); + appender.stop(); + } + } + + private void assertNonCompliantLicense(final Exception e) { assertThat(e, instanceOf(ElasticsearchSecurityException.class)); assertThat(e.getMessage(), equalTo("current license is non-compliant for [ccr]")); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IncompatibleLicenseLocalStateCcr.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/NonCompliantLicenseLocalStateCcr.java similarity index 80% rename from x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IncompatibleLicenseLocalStateCcr.java rename to x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/NonCompliantLicenseLocalStateCcr.java index c4b765d3c65ea..f960668a7dff1 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IncompatibleLicenseLocalStateCcr.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/NonCompliantLicenseLocalStateCcr.java @@ -12,16 +12,16 @@ import java.nio.file.Path; -public class IncompatibleLicenseLocalStateCcr extends LocalStateCompositeXPackPlugin { +public class NonCompliantLicenseLocalStateCcr extends LocalStateCompositeXPackPlugin { - public IncompatibleLicenseLocalStateCcr(final Settings settings, final Path configPath) throws Exception { + public NonCompliantLicenseLocalStateCcr(final Settings settings, final Path configPath) throws Exception { super(settings, configPath); plugins.add(new Ccr(settings, new CcrLicenseChecker(() -> false)) { @Override protected XPackLicenseState getLicenseState() { - return IncompatibleLicenseLocalStateCcr.this.getLicenseState(); + return NonCompliantLicenseLocalStateCcr.this.getLicenseState(); } }); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java index dd1376a4d7a73..2ef841292322a 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java @@ -66,9 +66,9 @@ public void testAutoFollower() { invoked[0] = true; assertThat(e, nullValue()); }; - AutoFollower autoFollower = new AutoFollower(client, handler, currentState) { + AutoFollower autoFollower = new AutoFollower(handler, currentState) { @Override - void getLeaderClusterState(Client leaderClient, BiConsumer handler) { + void getLeaderClusterState(String leaderClusterAlias, BiConsumer handler) { handler.accept(leaderState, null); } @@ -113,9 +113,9 @@ public void testAutoFollowerClusterStateApiFailure() { invoked[0] = true; assertThat(e, sameInstance(failure)); }; - AutoFollower autoFollower = new AutoFollower(client, handler, followerState) { + AutoFollower autoFollower = new AutoFollower(handler, followerState) { @Override - void getLeaderClusterState(Client leaderClient, BiConsumer handler) { + void getLeaderClusterState(String leaderClusterAlias, BiConsumer handler) { handler.accept(null, failure); } @@ -161,9 +161,9 @@ public void testAutoFollowerUpdateClusterStateFailure() { invoked[0] = true; assertThat(e, sameInstance(failure)); }; - AutoFollower autoFollower = new AutoFollower(client, handler, followerState) { + AutoFollower autoFollower = new AutoFollower(handler, followerState) { @Override - void getLeaderClusterState(Client leaderClient, BiConsumer handler) { + void getLeaderClusterState(String leaderClusterAlias, BiConsumer handler) { handler.accept(leaderState, null); } @@ -211,9 +211,9 @@ public void testAutoFollowerCreateAndFollowApiCallFailure() { invoked[0] = true; assertThat(e, sameInstance(failure)); }; - AutoFollower autoFollower = new AutoFollower(client, handler, followerState) { + AutoFollower autoFollower = new AutoFollower(handler, followerState) { @Override - void getLeaderClusterState(Client leaderClient, BiConsumer handler) { + void getLeaderClusterState(String leaderClusterAlias, BiConsumer handler) { handler.accept(leaderState, null); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index 5bf8fb6956bfe..bef7705e83533 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -532,7 +532,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { } else { Index index = new Index(indexToRemove, "_uuid"); ShardId shardId = new ShardId(index, 0); - ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, + ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); shardRouting = shardRouting.initialize("node_id", null, 0L); routingTable.add(IndexRoutingTable.builder(index) @@ -656,7 +656,7 @@ private void addJobAndIndices(MetaData.Builder metaData, RoutingTable.Builder ro metaData.put(indexMetaData); Index index = new Index(indexName, "_uuid"); ShardId shardId = new ShardId(index, 0); - ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, + ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); shardRouting = shardRouting.initialize("node_id", null, 0L); shardRouting = shardRouting.moveToStarted(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java index 3a6082c6cf057..4b8ad1d08aed3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java @@ -349,7 +349,7 @@ private static RoutingTable generateRoutingTable(IndexMetaData indexMetaData, Li true, ShardRoutingState.RELOCATING); } else { shardRouting = ShardRouting.newUnassigned(shardId, true, - RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, + RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index c7ddb3c4d2427..61224ac0fd735 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.bytes.BytesReference; @@ -60,7 +61,6 @@ import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; -import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -289,7 +289,8 @@ public void testToXContent() throws IOException { final ShardId shardId = new ShardId("_index", "_index_id", 7); final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message"); - final ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, EXISTING_STORE_INSTANCE, unassignedInfo); + final ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, unassignedInfo); final ShardStats mockShardStats = mock(ShardStats.class); when(mockShardStats.getShardRouting()).thenReturn(shardRouting); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index 4000969187548..4e5271c520a81 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -55,7 +56,6 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; -import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.contains; @@ -267,8 +267,8 @@ private ClusterState getClusterStateWithSecurityIndex() { } Index index = new Index(securityIndexName, UUID.randomUUID().toString()); - ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, EXISTING_STORE_INSTANCE, - new UnassignedInfo(Reason.INDEX_CREATED, "")); + ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(Reason.INDEX_CREATED, "")); IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(new ShardId(index, 0)) .addShard(shardRouting.initialize(randomAlphaOfLength(8), null, shardRouting.getExpectedShardSize()).moveToStarted()) .build(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index c3a6d7e920d1a..76e84f83137e3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -50,7 +51,6 @@ import org.hamcrest.Matchers; import org.junit.Before; -import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_PATTERN; @@ -106,8 +106,8 @@ public void testIndexWithoutPrimaryShards() throws IOException { final ClusterState.Builder clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME); Index index = new Index(INDEX_NAME, UUID.randomUUID().toString()); - ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, EXISTING_STORE_INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); + ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); String nodeId = ESTestCase.randomAlphaOfLength(8); IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(new ShardId(index, 0)) .addShard(shardRouting.initialize(nodeId, null, shardRouting.getExpectedShardSize()) @@ -165,7 +165,8 @@ public void testIndexHealthChangeListeners() throws Exception { clusterStateBuilder.routingTable(RoutingTable.builder() .add(IndexRoutingTable.builder(prevIndex) .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId(prevIndex, 0)) - .addShard(ShardRouting.newUnassigned(new ShardId(prevIndex, 0), true, EXISTING_STORE_INSTANCE, + .addShard(ShardRouting.newUnassigned(new ShardId(prevIndex, 0), true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) .initialize(UUIDs.randomBase64UUID(random()), null, 0L) .moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, ""))) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java index aa4982cce3f84..12474b7a04d59 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java @@ -5,21 +5,17 @@ */ package org.elasticsearch.xpack.security.test; -import org.elasticsearch.core.internal.io.IOUtils; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; @@ -39,9 +35,7 @@ import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import static java.nio.file.StandardOpenOption.WRITE; -import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; -import static org.junit.Assert.assertEquals; public class SecurityTestUtils { @@ -74,7 +68,7 @@ public static String writeFile(Path folder, String name, String content) { public static RoutingTable buildIndexRoutingTable(String indexName) { Index index = new Index(indexName, UUID.randomUUID().toString()); - ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, EXISTING_STORE_INSTANCE, + ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); String nodeId = ESTestCase.randomAlphaOfLength(8); IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(new ShardId(index, 0)) @@ -95,60 +89,4 @@ public static MetaData addAliasToMetaData(MetaData metaData, String indexName) { metaDataBuilder.put(IndexMetaData.builder(indexMetaData).putAlias(aliasMetaData)); return metaDataBuilder.build(); } - - public static ClusterIndexHealth getClusterIndexHealth(ClusterHealthStatus status) { - IndexMetaData metaData = IndexMetaData.builder("foo").settings(Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .build()) - .build(); - final IndexRoutingTable routingTable; - switch (status) { - case RED: - routingTable = IndexRoutingTable.builder(metaData.getIndex()) - .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId(metaData.getIndex(), 0)) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), true, EXISTING_STORE_INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L)) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), false, - RecoverySource.PeerRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L)) - .build()) - .build(); - break; - case YELLOW: - routingTable = IndexRoutingTable.builder(metaData.getIndex()) - .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId(metaData.getIndex(), 0)) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), true, EXISTING_STORE_INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L).moveToStarted()) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), false, - RecoverySource.PeerRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L)) - .build()) - .build(); - break; - case GREEN: - routingTable = IndexRoutingTable.builder(metaData.getIndex()) - .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId(metaData.getIndex(), 0)) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), true, EXISTING_STORE_INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L).moveToStarted()) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), false, - RecoverySource.PeerRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L).moveToStarted()) - .build()) - .build(); - break; - default: - throw new IllegalStateException("unknown status: " + status); - } - ClusterIndexHealth health = new ClusterIndexHealth(metaData, routingTable); - assertEquals(status, health.getStatus()); - return health; - } }