From 213af317beec30f74bf53e04bfd7a9449b98215e Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 13 Jun 2018 11:15:34 +0200 Subject: [PATCH 1/4] Support for remote path in reindex api Closes #22913 --- docs/reference/docs/reindex.asciidoc | 10 +-- .../index/reindex/RestReindexAction.java | 9 ++- .../index/reindex/TransportReindexAction.java | 62 ++++++++++--------- ...ReindexFromRemoteBuildRestClientTests.java | 26 ++++---- .../index/reindex/RestReindexActionTests.java | 15 +++++ .../index/reindex/remote/RemoteInfoTests.java | 16 +++-- .../index/reindex/RemoteInfo.java | 28 ++++++++- 7 files changed, 110 insertions(+), 56 deletions(-) diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index f05acab559ce1..bdbffb0a08d5d 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -422,11 +422,11 @@ POST _reindex // TEST[s/"username": "user",//] // TEST[s/"password": "pass"//] -The `host` parameter must contain a scheme, host, and port (e.g. -`https://otherhost:9200`). The `username` and `password` parameters are -optional, and when they are present `_reindex` will connect to the remote -Elasticsearch node using basic auth. Be sure to use `https` when using -basic auth or the password will be sent in plain text. +The `host` parameter must contain a scheme, host, port (e.g. +`https://otherhost:9200`) and optional path (e.g. `https://otherhost:9200/proxy`). +The `username` and `password` parameters are optional, and when they are present `_reindex` +will connect to the remote Elasticsearch node using basic auth. Be sure to use `https` when +using basic auth or the password will be sent in plain text. Remote hosts have to be explicitly whitelisted in elasticsearch.yaml using the `reindex.remote.whitelist` property. It can be set to a comma delimited list diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java index f1ac681b59fdf..ba133390f084b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java @@ -57,7 +57,7 @@ */ public class RestReindexAction extends AbstractBaseReindexRestHandler { static final ObjectParser PARSER = new ObjectParser<>("reindex"); - private static final Pattern HOST_PATTERN = Pattern.compile("(?[^:]+)://(?[^:]+):(?\\d+)"); + private static final Pattern HOST_PATTERN = Pattern.compile("(?[^:]+)://(?[^:]+):(?\\d+)(?(/.*)?)"); static { ObjectParser.Parser sourceParser = (parser, request, context) -> { @@ -139,10 +139,12 @@ static RemoteInfo buildRemoteInfo(Map source) throws IOException String hostInRequest = requireNonNull(extractString(remote, "host"), "[host] must be specified to reindex from a remote cluster"); Matcher hostMatcher = HOST_PATTERN.matcher(hostInRequest); if (false == hostMatcher.matches()) { - throw new IllegalArgumentException("[host] must be of the form [scheme]://[host]:[port] but was [" + hostInRequest + "]"); + throw new IllegalArgumentException("[host] must be of the form [scheme]://[host]:[port](/[pathPrefix])? but was [" + + hostInRequest + "]"); } String scheme = hostMatcher.group("scheme"); String host = hostMatcher.group("host"); + String pathPrefix = hostMatcher.group("pathPrefix"); int port = Integer.parseInt(hostMatcher.group("port")); Map headers = extractStringStringMap(remote, "headers"); TimeValue socketTimeout = extractTimeValue(remote, "socket_timeout", RemoteInfo.DEFAULT_SOCKET_TIMEOUT); @@ -151,7 +153,8 @@ static RemoteInfo buildRemoteInfo(Map source) throws IOException throw new IllegalArgumentException( "Unsupported fields in [remote]: [" + Strings.collectionToCommaDelimitedString(remote.keySet()) + "]"); } - return new RemoteInfo(scheme, host, port, queryForRemote(source), username, password, headers, socketTimeout, connectTimeout); + return new RemoteInfo(scheme, host, port, pathPrefix, queryForRemote(source), + username, password, headers, socketTimeout, connectTimeout); } /** diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 6f839558e03cc..3af2e038e662f 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -37,6 +37,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse.Failure; +import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure; @@ -206,34 +207,39 @@ static RestClient buildRestClient(RemoteInfo remoteInfo, long taskId, List header : remoteInfo.getHeaders().entrySet()) { clientHeaders[i++] = new BasicHeader(header.getKey(), header.getValue()); } - return RestClient.builder(new HttpHost(remoteInfo.getHost(), remoteInfo.getPort(), remoteInfo.getScheme())) - .setDefaultHeaders(clientHeaders) - .setRequestConfigCallback(c -> { - c.setConnectTimeout(Math.toIntExact(remoteInfo.getConnectTimeout().millis())); - c.setSocketTimeout(Math.toIntExact(remoteInfo.getSocketTimeout().millis())); - return c; - }) - .setHttpClientConfigCallback(c -> { - // Enable basic auth if it is configured - if (remoteInfo.getUsername() != null) { - UsernamePasswordCredentials creds = new UsernamePasswordCredentials(remoteInfo.getUsername(), - remoteInfo.getPassword()); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, creds); - c.setDefaultCredentialsProvider(credentialsProvider); - } - // Stick the task id in the thread name so we can track down tasks from stack traces - AtomicInteger threads = new AtomicInteger(); - c.setThreadFactory(r -> { - String name = "es-client-" + taskId + "-" + threads.getAndIncrement(); - Thread t = new Thread(r, name); - threadCollector.add(t); - return t; - }); - // Limit ourselves to one reactor thread because for now the search process is single threaded. - c.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build()); - return c; - }).build(); + final RestClientBuilder builder = + RestClient.builder(new HttpHost(remoteInfo.getHost(), remoteInfo.getPort(), remoteInfo.getScheme())) + .setDefaultHeaders(clientHeaders) + .setRequestConfigCallback(c -> { + c.setConnectTimeout(Math.toIntExact(remoteInfo.getConnectTimeout().millis())); + c.setSocketTimeout(Math.toIntExact(remoteInfo.getSocketTimeout().millis())); + return c; + }) + .setHttpClientConfigCallback(c -> { + // Enable basic auth if it is configured + if (remoteInfo.getUsername() != null) { + UsernamePasswordCredentials creds = new UsernamePasswordCredentials(remoteInfo.getUsername(), + remoteInfo.getPassword()); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, creds); + c.setDefaultCredentialsProvider(credentialsProvider); + } + // Stick the task id in the thread name so we can track down tasks from stack traces + AtomicInteger threads = new AtomicInteger(); + c.setThreadFactory(r -> { + String name = "es-client-" + taskId + "-" + threads.getAndIncrement(); + Thread t = new Thread(r, name); + threadCollector.add(t); + return t; + }); + // Limit ourselves to one reactor thread because for now the search process is single threaded. + c.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build()); + return c; + }); + if (Strings.hasLength(remoteInfo.getPathPrefix())) { + builder.setPathPrefix(remoteInfo.getPathPrefix()); + } + return builder.build(); } /** diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java index 14e3142d226c9..beb21734e6f33 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java @@ -34,20 +34,22 @@ public class ReindexFromRemoteBuildRestClientTests extends RestClientBuilderTestCase { public void testBuildRestClient() throws Exception { - RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, new BytesArray("ignored"), null, null, emptyMap(), + for(final String path: new String[]{"", null, "path"}) { + RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, path, new BytesArray("ignored"), null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); - long taskId = randomLong(); - List threads = synchronizedList(new ArrayList<>()); - RestClient client = TransportReindexAction.buildRestClient(remoteInfo, taskId, threads); - try { - assertBusy(() -> assertThat(threads, hasSize(2))); - int i = 0; - for (Thread thread : threads) { - assertEquals("es-client-" + taskId + "-" + i, thread.getName()); - i++; + long taskId = randomLong(); + List threads = synchronizedList(new ArrayList<>()); + RestClient client = TransportReindexAction.buildRestClient(remoteInfo, taskId, threads); + try { + assertBusy(() -> assertThat(threads, hasSize(2))); + int i = 0; + for (Thread thread : threads) { + assertEquals("es-client-" + taskId + "-" + i, thread.getName()); + i++; + } + } finally { + client.close(); } - } finally { - client.close(); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java index 88fa31f423a21..a517867228a3b 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java @@ -89,6 +89,7 @@ public void testBuildRemoteInfoWithAllHostParts() throws IOException { assertEquals("http", info.getScheme()); assertEquals("example.com", info.getHost()); assertEquals(9200, info.getPort()); + assertEquals("", info.getPathPrefix()); assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); // Didn't set the timeout so we should get the default assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); // Didn't set the timeout so we should get the default @@ -96,8 +97,22 @@ public void testBuildRemoteInfoWithAllHostParts() throws IOException { assertEquals("https", info.getScheme()); assertEquals("other.example.com", info.getHost()); assertEquals(9201, info.getPort()); + assertEquals("", info.getPathPrefix()); assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); + + info = buildRemoteInfoHostTestCase("https://other.example.com:9201/proxy-path/"); + assertEquals("https", info.getScheme()); + assertEquals("other.example.com", info.getHost()); + assertEquals(9201, info.getPort()); + assertEquals("/proxy-path/", info.getPathPrefix()); + assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); + assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); + + final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> buildRemoteInfoHostTestCase("https")); + assertEquals("[host] must be of the form [scheme]://[host]:[port](/[pathPrefix])? but was [https]", + exception.getMessage()); } public void testReindexFromRemoteRequestParsing() throws IOException { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java index d6ab599b43c2d..de0ade9c47cc3 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java @@ -26,17 +26,21 @@ import static java.util.Collections.emptyMap; public class RemoteInfoTests extends ESTestCase { - private RemoteInfo newRemoteInfo(String scheme, String username, String password) { - return new RemoteInfo(scheme, "testhost", 12344, new BytesArray("testquery"), username, password, emptyMap(), + private RemoteInfo newRemoteInfo(String scheme, String prefixPath, String username, String password) { + return new RemoteInfo(scheme, "testhost", 12344, prefixPath, new BytesArray("testquery"), username, password, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); } public void testToString() { - assertEquals("host=testhost port=12344 query=testquery", newRemoteInfo("http", null, null).toString()); - assertEquals("host=testhost port=12344 query=testquery username=testuser", newRemoteInfo("http", "testuser", null).toString()); + assertEquals("host=testhost port=12344 query=testquery", + newRemoteInfo("http", null, null, null).toString()); + assertEquals("host=testhost port=12344 query=testquery username=testuser", + newRemoteInfo("http", null, "testuser", null).toString()); assertEquals("host=testhost port=12344 query=testquery username=testuser password=<<>>", - newRemoteInfo("http", "testuser", "testpass").toString()); + newRemoteInfo("http", null, "testuser", "testpass").toString()); assertEquals("scheme=https host=testhost port=12344 query=testquery username=testuser password=<<>>", - newRemoteInfo("https", "testuser", "testpass").toString()); + newRemoteInfo("https", null, "testuser", "testpass").toString()); + assertEquals("scheme=https host=testhost port=12344 pathPrefix=prxy query=testquery username=testuser password=<<>>", + newRemoteInfo("https", "prxy", "testuser", "testpass").toString()); } } diff --git a/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java b/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java index 8e7a990902631..16a907a00ad83 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java @@ -48,6 +48,7 @@ public class RemoteInfo implements Writeable { private final String scheme; private final String host; private final int port; + private final String pathPrefix; private final BytesReference query; private final String username; private final String password; @@ -62,10 +63,16 @@ public class RemoteInfo implements Writeable { private final TimeValue connectTimeout; public RemoteInfo(String scheme, String host, int port, BytesReference query, String username, String password, - Map headers, TimeValue socketTimeout, TimeValue connectTimeout) { + Map headers, TimeValue socketTimeout, TimeValue connectTimeout) { + this(scheme, host, port, null, query, username, password, headers, socketTimeout, connectTimeout); + } + + public RemoteInfo(String scheme, String host, int port, String pathPrefix, BytesReference query, String username, String password, + Map headers, TimeValue socketTimeout, TimeValue connectTimeout) { this.scheme = requireNonNull(scheme, "[scheme] must be specified to reindex from a remote cluster"); this.host = requireNonNull(host, "[host] must be specified to reindex from a remote cluster"); this.port = port; + this.pathPrefix = pathPrefix; this.query = requireNonNull(query, "[query] must be specified to reindex from a remote cluster"); this.username = username; this.password = password; @@ -97,6 +104,11 @@ public RemoteInfo(StreamInput in) throws IOException { socketTimeout = DEFAULT_SOCKET_TIMEOUT; connectTimeout = DEFAULT_CONNECT_TIMEOUT; } + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + pathPrefix = in.readOptionalString(); + } else { + pathPrefix = null; + } } @Override @@ -116,6 +128,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeTimeValue(socketTimeout); out.writeTimeValue(connectTimeout); } + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeOptionalString(pathPrefix); + } } public String getScheme() { @@ -130,6 +145,11 @@ public int getPort() { return port; } + @Nullable + public String getPathPrefix() { + return pathPrefix; + } + public BytesReference getQuery() { return query; } @@ -169,7 +189,11 @@ public String toString() { // http is the default so it isn't worth taking up space if it is the scheme b.append("scheme=").append(scheme).append(' '); } - b.append("host=").append(host).append(" port=").append(port).append(" query=").append(query.utf8ToString()); + b.append("host=").append(host).append(" port=").append(port); + if (pathPrefix != null) { + b.append(" pathPrefix=").append(pathPrefix); + } + b.append(" query=").append(query.utf8ToString()); if (username != null) { b.append(" username=").append(username); } From 123fe86fdbb38c87124eed8e85027549f3db8630 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 13 Jun 2018 17:14:59 +0200 Subject: [PATCH 2/4] simplified pathPrefix regexp --- .../org/elasticsearch/index/reindex/RestReindexAction.java | 2 +- .../elasticsearch/index/reindex/RestReindexActionTests.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java index ba133390f084b..a5520c90b0ff5 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java @@ -57,7 +57,7 @@ */ public class RestReindexAction extends AbstractBaseReindexRestHandler { static final ObjectParser PARSER = new ObjectParser<>("reindex"); - private static final Pattern HOST_PATTERN = Pattern.compile("(?[^:]+)://(?[^:]+):(?\\d+)(?(/.*)?)"); + private static final Pattern HOST_PATTERN = Pattern.compile("(?[^:]+)://(?[^:]+):(?\\d+)(?/.*)?"); static { ObjectParser.Parser sourceParser = (parser, request, context) -> { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java index a517867228a3b..e85542b8a1717 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java @@ -89,7 +89,7 @@ public void testBuildRemoteInfoWithAllHostParts() throws IOException { assertEquals("http", info.getScheme()); assertEquals("example.com", info.getHost()); assertEquals(9200, info.getPort()); - assertEquals("", info.getPathPrefix()); + assertNull(info.getPathPrefix()); assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); // Didn't set the timeout so we should get the default assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); // Didn't set the timeout so we should get the default @@ -97,7 +97,7 @@ public void testBuildRemoteInfoWithAllHostParts() throws IOException { assertEquals("https", info.getScheme()); assertEquals("other.example.com", info.getHost()); assertEquals(9201, info.getPort()); - assertEquals("", info.getPathPrefix()); + assertNull(info.getPathPrefix()); assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); From ca6ee27b3e38f740822774c25f649ab181339a04 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 13 Jun 2018 17:16:07 +0200 Subject: [PATCH 3/4] don't overload RemoteInfo ctor --- .../ReindexFromRemoteBuildRestClientTests.java | 2 +- .../reindex/ReindexFromRemoteWhitelistTests.java | 4 ++-- .../reindex/ReindexFromRemoteWithAuthTests.java | 5 +++-- .../reindex/ReindexSourceTargetValidationTests.java | 4 ++-- .../org/elasticsearch/index/reindex/RetryTests.java | 6 ++++-- .../elasticsearch/index/reindex/RoundTripTests.java | 5 +++-- .../org/elasticsearch/index/reindex/RemoteInfo.java | 5 ----- .../index/reindex/ReindexRequestTests.java | 12 +++++++----- 8 files changed, 22 insertions(+), 21 deletions(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java index beb21734e6f33..7ed0ff9035b0c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java @@ -59,7 +59,7 @@ public void testHeaders() throws Exception { for (int i = 0; i < numHeaders; ++i) { headers.put("header" + i, Integer.toString(i)); } - RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, new BytesArray("ignored"), null, null, + RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, null, new BytesArray("ignored"), null, null, headers, RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); long taskId = randomLong(); List threads = synchronizedList(new ArrayList<>()); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java index 128cd4043e283..e32370b166546 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java @@ -49,7 +49,7 @@ public void testLocalRequestWithWhitelist() { * Build a {@link RemoteInfo}, defaulting values that we don't care about in this test to values that don't hurt anything. */ private RemoteInfo newRemoteInfo(String host, int port) { - return new RemoteInfo(randomAlphaOfLength(5), host, port, new BytesArray("test"), null, null, emptyMap(), + return new RemoteInfo(randomAlphaOfLength(5), host, port, null, new BytesArray("test"), null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); } @@ -63,7 +63,7 @@ public void testWhitelistedRemote() { public void testWhitelistedByPrefix() { checkRemoteWhitelist(buildRemoteWhitelist(singletonList("*.example.com:9200")), - new RemoteInfo(randomAlphaOfLength(5), "es.example.com", 9200, new BytesArray("test"), null, null, emptyMap(), + new RemoteInfo(randomAlphaOfLength(5), "es.example.com", 9200, null, new BytesArray("test"), null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT)); checkRemoteWhitelist(buildRemoteWhitelist(singletonList("*.example.com:9200")), newRemoteInfo("6e134134a1.us-east-1.aws.example.com", 9200)); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java index 72ba651dff9ae..dc6d5eac58897 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java @@ -104,8 +104,9 @@ public void fetchTransportAddress() { * Build a {@link RemoteInfo}, defaulting values that we don't care about in this test to values that don't hurt anything. */ private RemoteInfo newRemoteInfo(String username, String password, Map headers) { - return new RemoteInfo("http", address.getAddress(), address.getPort(), new BytesArray("{\"match_all\":{}}"), username, password, - headers, RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); + return new RemoteInfo("http", address.getAddress(), address.getPort(), null, + new BytesArray("{\"match_all\":{}}"), username, password, headers, + RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); } public void testReindexFromRemoteWithAuthentication() throws Exception { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java index 28b9febe1c289..4784d7f5fe546 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java @@ -88,10 +88,10 @@ public void testTargetIsAlias() { public void testRemoteInfoSkipsValidation() { // The index doesn't have to exist - succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, new BytesArray("test"), null, null, emptyMap(), + succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, null, new BytesArray("test"), null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT), "does_not_exist", "target"); // And it doesn't matter if they are the same index. They are considered to be different because the remote one is, well, remote. - succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, new BytesArray("test"), null, null, emptyMap(), + succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, null, new BytesArray("test"), null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT), "target", "target"); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java index 1107a36086927..9ea2a24bfb136 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java @@ -124,8 +124,10 @@ public void testReindexFromRemote() throws Exception { assertNotNull(masterNode); TransportAddress address = masterNode.getHttp().getAddress().publishAddress(); - RemoteInfo remote = new RemoteInfo("http", address.getAddress(), address.getPort(), new BytesArray("{\"match_all\":{}}"), null, - null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); + RemoteInfo remote = + new RemoteInfo("http", address.getAddress(), address.getPort(), null, + new BytesArray("{\"match_all\":{}}"), null, null, emptyMap(), + RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); ReindexRequestBuilder request = new ReindexRequestBuilder(client, ReindexAction.INSTANCE).source("source").destination("dest") .setRemoteInfo(remote); return request; diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java index 946ab030c8285..2dc4b59e8d9f9 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java @@ -63,8 +63,9 @@ public void testReindexRequest() throws IOException { } TimeValue socketTimeout = parseTimeValue(randomPositiveTimeValue(), "socketTimeout"); TimeValue connectTimeout = parseTimeValue(randomPositiveTimeValue(), "connectTimeout"); - reindex.setRemoteInfo(new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), port, query, username, password, headers, - socketTimeout, connectTimeout)); + reindex.setRemoteInfo( + new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), port, null, + query, username, password, headers, socketTimeout, connectTimeout)); } ReindexRequest tripped = new ReindexRequest(); roundTrip(reindex, tripped); diff --git a/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java b/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java index 16a907a00ad83..494658821cfd5 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java @@ -62,11 +62,6 @@ public class RemoteInfo implements Writeable { */ private final TimeValue connectTimeout; - public RemoteInfo(String scheme, String host, int port, BytesReference query, String username, String password, - Map headers, TimeValue socketTimeout, TimeValue connectTimeout) { - this(scheme, host, port, null, query, username, password, headers, socketTimeout, connectTimeout); - } - public RemoteInfo(String scheme, String host, int port, String pathPrefix, BytesReference query, String username, String password, Map headers, TimeValue socketTimeout, TimeValue connectTimeout) { this.scheme = requireNonNull(scheme, "[scheme] must be specified to reindex from a remote cluster"); diff --git a/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java index 9f4b20ff35ba3..6c1988a1440e9 100644 --- a/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java @@ -37,8 +37,9 @@ public class ReindexRequestTests extends AbstractBulkByScrollRequestTestCase Date: Thu, 14 Jun 2018 09:02:55 +0200 Subject: [PATCH 4/4] handle a single '/' in path prefix --- .../index/reindex/TransportReindexAction.java | 2 +- .../reindex/ReindexFromRemoteBuildRestClientTests.java | 2 +- .../index/reindex/RestReindexActionTests.java | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 3af2e038e662f..5e0ad0fd3fdd8 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -236,7 +236,7 @@ static RestClient buildRestClient(RemoteInfo remoteInfo, long taskId, List