From e7e1b773eaa70a3ac4bd458d5e02d7b793b63777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 28 Apr 2022 19:44:36 +0200 Subject: [PATCH 01/12] Run more search rest tests in a CCS setup Currently we only test a small subset of cross cluster search in rest tests in the 'multi-cluster-search' qa module. In order to increase test coverage for basic CCS setups, this change adds a new qa modula that uses a subset of existing search rest tests tests and runs them in a CCS scenario. Relates to #84481 --- qa/ccs-common-rest/build.gradle | 98 +++++++++ .../rest/yaml/CcsCommonYamlTestSuiteIT.java | 188 ++++++++++++++++++ .../rest/yaml/ClientYamlTestCandidate.java | 4 + .../yaml/ClientYamlTestExecutionContext.java | 6 +- .../rest/yaml/ESClientYamlSuiteTestCase.java | 11 +- 5 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 qa/ccs-common-rest/build.gradle create mode 100644 qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle new file mode 100644 index 0000000000000..8ac621afaf626 --- /dev/null +++ b/qa/ccs-common-rest/build.gradle @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + + +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask + + +apply plugin: 'elasticsearch.internal-testclusters' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-resources' + +restResources { + restApi { + include '_common', 'bulk', 'count', 'cluster', 'index', 'indices', 'field_caps', 'msearch', + 'search', 'async_search', 'graph', '*_point_in_time', "info" + } + restTests { + includeCore 'field_caps', 'search', 'suggest' + } +} + +def writeCluster = testClusters.register("ccs-write") { + numberOfNodes = 2 + setting 'node.roles', '[data,ingest,master]' +} + +def searchCluster = testClusters.register("ccs-search") { + setting 'node.roles', '[data,ingest,master,remote_cluster_client]' + setting 'cluster.remote.connections_per_cluster', '1' + setting 'cluster.remote.remote_cluster.seeds', + { "\"${writeCluster.get().getAllTransportPortURI().get(0)}\"" } +} + +testClusters.configureEach { + setting 'xpack.security.enabled', 'false' + requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") +} + +tasks.register('startWriteCluster', DefaultTestClustersTask) { + useCluster writeCluster + doLast { + clusters.each { c -> + print "Writer cluster transport uri for ccs configuration is: " + println c.getAllTransportPortURI().get(0) + } + } +} + +tasks.register("ccs-write", RestIntegTestTask) { + mustRunAfter("precommit") + dependsOn startWriteCluster + + useCluster writeCluster + useCluster searchCluster + + systemProperty 'tests.rest.suite', + [ + 'field_caps', + 'search', + 'search.aggregation', + 'search.highlight', + 'search.inner_hits', + 'suggest', + ].join(',') + + systemProperty 'tests.rest.blacklist', + [ + // TODO look into fixing these + 'search/70_response_filtering/Search with response filtering', // makes assertions on "_clusters" section + 'search/150_rewrite_on_coordinator/Ensure that we fetch the document only once', // terms lookup query with index + 'search/170_terms_query/Terms Query with No.of terms exceeding index.max_terms_count should FAIL', // terms lookup query with index + 'search/350_point_in_time/basic', // [indices] cannot be used with point in time + 'search/350_point_in_time/point-in-time with slicing', // [indices] cannot be used with point in time + 'search/350_point_in_time/wildcard', // [indices] cannot be used with point in time + 'search.aggregation/220_filters_bucket/cache busting', // node_selector? + 'search.aggregation/220_filters_bucket/cache hits', // node_selector? + 'search.aggregation/50_filter/Standard queries get cached', + 'search.aggregation/50_filter/Terms lookup gets cached', // terms lookup by "index" doesn't seem to work correctly + 'search.aggregation/70_adjacency_matrix/Terms lookup' // terms lookup by "index" doesn't seem to work correctly + ].join(',') + + + doFirst { + // Getting the endpoints causes a wait for the cluster + println "Writer cluster endpoints are: ${-> writeCluster.get().allHttpSocketURI.join(",")}" + println "Search cluster endpoints are: ${-> searchCluster.get().allHttpSocketURI.join(",")}" + nonInputProperties.systemProperty('tests.rest.cluster', writeCluster.map(c -> c.allHttpSocketURI.join(","))) + nonInputProperties.systemProperty('tests.rest.search_cluster', searchCluster.map(c -> c.allHttpSocketURI.join(","))) + } +} + diff --git a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java new file mode 100644 index 0000000000000..cacc142c6ea19 --- /dev/null +++ b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test.rest.yaml; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.tests.util.TimeUnits; +import org.elasticsearch.Version; +import org.elasticsearch.client.NodeSelector; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.test.rest.yaml.section.MatchAssertion; +import org.junit.AfterClass; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.unmodifiableList; + +@TimeoutSuite(millis = 15 * TimeUnits.MINUTE) // to account for slow as hell VMs +public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + private static final Logger logger = LogManager.getLogger(CcsCommonYamlTestSuiteIT.class); + private static RestClient searchClient; + private static RestClient adminSearchClient; + private static List clusterHosts; + private static ClientYamlTestClient searchYamlTestClient; + + private static final String REMOTE_CLUSTER_NAME = "remote_cluster"; + + @Before + public void initSearchClient() throws IOException { + if (searchClient == null) { + assert adminSearchClient == null; + assert clusterHosts == null; + + final String cluster = System.getProperty("tests.rest.search_cluster"); + assertNotNull("[tests.rest.search_cluster] is not configured", cluster); + String[] stringUrls = cluster.split(","); + List hosts = new ArrayList<>(stringUrls.length); + for (String stringUrl : stringUrls) { + int portSeparator = stringUrl.lastIndexOf(':'); + if (portSeparator < 0) { + throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]"); + } + String host = stringUrl.substring(0, portSeparator); + int port = Integer.parseInt(stringUrl.substring(portSeparator + 1)); + hosts.add(buildHttpHost(host, port)); + } + clusterHosts = unmodifiableList(hosts); + logger.info("initializing REST search clients against {}", clusterHosts); + searchClient = buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[clusterHosts.size()])); + adminSearchClient = buildClient(restAdminSettings(), clusterHosts.toArray(new HttpHost[clusterHosts.size()])); + + Tuple versionVersionTuple = readVersionsFromCatNodes(adminSearchClient); + final Version esVersion = versionVersionTuple.v1(); + final Version masterVersion = versionVersionTuple.v2(); + final String os = readOsFromNodesInfo(adminSearchClient); + + searchYamlTestClient = new ClientYamlTestClient( + getRestSpec(), + searchClient, + hosts, + esVersion, + masterVersion, + os, + this::getClientBuilderWithSniffedHosts + ) { + public ClientYamlTestResponse callApi( + String apiName, + Map params, + HttpEntity entity, + Map headers, + NodeSelector nodeSelector + ) throws IOException { + // on request, we need to replace index specifications by prefixing the remote cluster + String originalIndices = params.get("index"); + String expandedIndices = REMOTE_CLUSTER_NAME + ":*"; + if (originalIndices != null && (originalIndices.isEmpty() == false)) { + String[] indices = originalIndices.split(","); + List newIndices = new ArrayList<>(); + for (String indexName : indices) { + newIndices.add(REMOTE_CLUSTER_NAME + ":" + indexName); + } + expandedIndices = String.join(",", newIndices); + } + params.put("index", String.join(",", expandedIndices)); + ClientYamlTestResponse clientYamlTestResponse = super.callApi(apiName, params, entity, headers, nodeSelector); + return clientYamlTestResponse; + }; + }; + + // check that we have an established CCS connection + Request request = new Request("GET", "_remote/info"); + Response response = adminSearchClient.performRequest(request); + assertOK(response); + ObjectPath responseObject = ObjectPath.createFromResponse(response); + assertNotNull(responseObject.evaluate(REMOTE_CLUSTER_NAME)); + logger.info("Established connection to remote cluster [" + REMOTE_CLUSTER_NAME + "]"); + } + + assert searchClient != null; + assert adminSearchClient != null; + assert clusterHosts != null; + } + + public CcsCommonYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) throws IOException { + super(rewrite(testCandidate)); + } + + private static ClientYamlTestCandidate rewrite(ClientYamlTestCandidate clientYamlTestCandidate) { + ClientYamlTestSection testSection = clientYamlTestCandidate.getTestSection(); + List executableSections = testSection.getExecutableSections(); + List modifiedExecutableSections = new ArrayList<>(); + for (ExecutableSection section : executableSections) { + if (section instanceof MatchAssertion) { + MatchAssertion matchSection = (MatchAssertion) section; + if (matchSection.getField().endsWith("_index")) { + String modifiedExpectedValue = REMOTE_CLUSTER_NAME + ":" + matchSection.getExpectedValue(); + modifiedExecutableSections.add( + new MatchAssertion(matchSection.getLocation(), matchSection.getField(), modifiedExpectedValue) + ); + } + } else { + modifiedExecutableSections.add(section); + } + } + return new ClientYamlTestCandidate( + clientYamlTestCandidate.getRestTestSuite(), + new ClientYamlTestSection( + testSection.getLocation(), + testSection.getName(), + testSection.getSkipSection(), + modifiedExecutableSections + ) + ); + }; + + @ParametersFactory + public static Iterable parameters() throws Exception { + return createParameters(); + } + + @Override + protected ClientYamlTestExecutionContext createRestTestExecutionContext( + ClientYamlTestCandidate clientYamlTestCandidate, + ClientYamlTestClient clientYamlTestClient + ) { + return new ClientYamlTestExecutionContext(clientYamlTestCandidate, clientYamlTestClient, randomizeContentType()) { + protected ClientYamlTestClient clientYamlTestClient(String apiName) { + if (apiName.equals("search") || apiName.equals("field_caps")) { + return searchYamlTestClient; + } else { + return super.clientYamlTestClient(apiName); + } + } + }; + } + + @AfterClass + public static void closeSearchClients() throws IOException { + try { + IOUtils.close(searchClient, adminSearchClient); + } finally { + clusterHosts = null; + } + } +} diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestCandidate.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestCandidate.java index 23cfe5af582aa..d357d703e8922 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestCandidate.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestCandidate.java @@ -45,6 +45,10 @@ public SetupSection getSetupSection() { return restTestSuite.getSetupSection(); } + public ClientYamlTestSuite getRestTestSuite() { + return restTestSuite; + } + public TeardownSection getTeardownSection() { return restTestSuite.getTeardownSection(); } diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index 9885b7c2e8710..64ec7e2a99d83 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -183,7 +183,11 @@ ClientYamlTestResponse callApiInternal( Map headers, NodeSelector nodeSelector ) throws IOException { - return clientYamlTestClient.callApi(apiName, params, entity, headers, nodeSelector); + return clientYamlTestClient(apiName).callApi(apiName, params, entity, headers, nodeSelector); + } + + protected ClientYamlTestClient clientYamlTestClient(String apiName) { + return clientYamlTestClient; } /** diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 657a9fc74c5c9..b7e6fb0d9ea8c 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -108,6 +108,8 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase { private final ClientYamlTestCandidate testCandidate; + private static ClientYamlSuiteRestSpec restSpecification; + protected ESClientYamlSuiteTestCase(ClientYamlTestCandidate testCandidate) { this.testCandidate = testCandidate; } @@ -126,6 +128,7 @@ public void initAndResetContext() throws Exception { assert blacklistPathMatchers == null; final ClientYamlSuiteRestSpec restSpec = ClientYamlSuiteRestSpec.load(SPEC_PATH); validateSpec(restSpec); + restSpecification = restSpec; final List hosts = getClusterHosts(); Tuple versionVersionTuple = readVersionsFromCatNodes(adminClient()); final Version esVersion = versionVersionTuple.v1(); @@ -317,6 +320,10 @@ protected ClientYamlTestExecutionContext getAdminExecutionContext() { return adminExecutionContext; } + static ClientYamlSuiteRestSpec getRestSpec() { + return restSpecification; + } + private static void validateSpec(ClientYamlSuiteRestSpec restSpec) { boolean validateSpec = RandomizedTest.systemPropertyAsBoolean(REST_TESTS_VALIDATE_SPEC, true); if (validateSpec) { @@ -341,7 +348,7 @@ private static void validateSpec(ClientYamlSuiteRestSpec restSpec) { } } - private Tuple readVersionsFromCatNodes(RestClient restClient) throws IOException { + Tuple readVersionsFromCatNodes(RestClient restClient) throws IOException { // we simply go to the _cat/nodes API and parse all versions in the cluster final Request request = new Request("GET", "/_cat/nodes"); request.addParameter("h", "version,master"); @@ -370,7 +377,7 @@ private Tuple readVersionsFromCatNodes(RestClient restClient) return new Tuple<>(version, masterVersion); } - private String readOsFromNodesInfo(RestClient restClient) throws IOException { + String readOsFromNodesInfo(RestClient restClient) throws IOException { final Request request = new Request("GET", "/_nodes/os"); Response response = restClient.performRequest(request); ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response); From 9f00ee17c61df1e4ebf8a357296fbccfe554d2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 11 May 2022 18:01:21 +0200 Subject: [PATCH 02/12] Adding msearch tests --- qa/ccs-common-rest/build.gradle | 3 ++- .../rest/yaml/CcsCommonYamlTestSuiteIT.java | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index 8ac621afaf626..8a9eada68faa9 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -22,7 +22,7 @@ restResources { 'search', 'async_search', 'graph', '*_point_in_time', "info" } restTests { - includeCore 'field_caps', 'search', 'suggest' + includeCore 'field_caps', 'msearch', 'search', 'suggest' } } @@ -63,6 +63,7 @@ tasks.register("ccs-write", RestIntegTestTask) { systemProperty 'tests.rest.suite', [ 'field_caps', + "msearch", 'search', 'search.aggregation', 'search.highlight', diff --git a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index cacc142c6ea19..ad946549e3871 100644 --- a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import org.elasticsearch.test.rest.yaml.section.DoSection; import org.elasticsearch.test.rest.yaml.section.ExecutableSection; import org.elasticsearch.test.rest.yaml.section.MatchAssertion; import org.junit.AfterClass; @@ -34,6 +35,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import static java.util.Collections.unmodifiableList; @@ -46,7 +48,10 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { private static List clusterHosts; private static ClientYamlTestClient searchYamlTestClient; + // the remote cluster is the one we write index operations etc... to private static final String REMOTE_CLUSTER_NAME = "remote_cluster"; + // the following are the CCS api calls that we run against the "search" cluster in this test setup + private static final Set CCS_APIS = Set.of("search", "field_caps", "msearch"); @Before public void initSearchClient() throws IOException { @@ -141,6 +146,17 @@ private static ClientYamlTestCandidate rewrite(ClientYamlTestCandidate clientYam new MatchAssertion(matchSection.getLocation(), matchSection.getField(), modifiedExpectedValue) ); } + } else if (section instanceof DoSection && ((DoSection) section).getApiCallSection().getApi().equals("msearch")) { + // modify "msearch" body sections so the "index" part is targeting the remote cluster + DoSection doSection = ((DoSection) section); + List> bodies = doSection.getApiCallSection().getBodies(); + for (Map body : bodies) { + if (body.containsKey("index")) { + String modifiedIndex = REMOTE_CLUSTER_NAME + ":" + body.get("index"); + body.put("index", modifiedIndex); + } + } + modifiedExecutableSections.add(section); } else { modifiedExecutableSections.add(section); } @@ -168,7 +184,7 @@ protected ClientYamlTestExecutionContext createRestTestExecutionContext( ) { return new ClientYamlTestExecutionContext(clientYamlTestCandidate, clientYamlTestClient, randomizeContentType()) { protected ClientYamlTestClient clientYamlTestClient(String apiName) { - if (apiName.equals("search") || apiName.equals("field_caps")) { + if (CCS_APIS.contains(apiName)) { return searchYamlTestClient; } else { return super.clientYamlTestClient(apiName); From 2ca02053108f4cffe55a7d50334451b023a31061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 11 May 2022 19:14:14 +0200 Subject: [PATCH 03/12] Adding scroll --- qa/ccs-common-rest/build.gradle | 6 +++-- .../rest/yaml/CcsCommonYamlTestSuiteIT.java | 25 ++++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index 8a9eada68faa9..2c73947733e78 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -19,10 +19,10 @@ apply plugin: 'elasticsearch.rest-resources' restResources { restApi { include '_common', 'bulk', 'count', 'cluster', 'index', 'indices', 'field_caps', 'msearch', - 'search', 'async_search', 'graph', '*_point_in_time', "info" + 'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', "clear_scroll" } restTests { - includeCore 'field_caps', 'msearch', 'search', 'suggest' + includeCore 'field_caps', 'msearch', 'search', 'suggest', 'scroll' } } @@ -64,6 +64,7 @@ tasks.register("ccs-write", RestIntegTestTask) { [ 'field_caps', "msearch", + "scroll", 'search', 'search.aggregation', 'search.highlight', @@ -79,6 +80,7 @@ tasks.register("ccs-write", RestIntegTestTask) { 'search/170_terms_query/Terms Query with No.of terms exceeding index.max_terms_count should FAIL', // terms lookup query with index 'search/350_point_in_time/basic', // [indices] cannot be used with point in time 'search/350_point_in_time/point-in-time with slicing', // [indices] cannot be used with point in time + 'search/350_point_in_time/msearch', // [indices] cannot be used with point in time 'search/350_point_in_time/wildcard', // [indices] cannot be used with point in time 'search.aggregation/220_filters_bucket/cache busting', // node_selector? 'search.aggregation/220_filters_bucket/cache hits', // node_selector? diff --git a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index ad946549e3871..b8089522dfead 100644 --- a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -51,7 +51,7 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { // the remote cluster is the one we write index operations etc... to private static final String REMOTE_CLUSTER_NAME = "remote_cluster"; // the following are the CCS api calls that we run against the "search" cluster in this test setup - private static final Set CCS_APIS = Set.of("search", "field_caps", "msearch"); + private static final Set CCS_APIS = Set.of("search", "field_caps", "msearch", "scroll", "clear_scroll"); @Before public void initSearchClient() throws IOException { @@ -99,19 +99,20 @@ public ClientYamlTestResponse callApi( NodeSelector nodeSelector ) throws IOException { // on request, we need to replace index specifications by prefixing the remote cluster - String originalIndices = params.get("index"); - String expandedIndices = REMOTE_CLUSTER_NAME + ":*"; - if (originalIndices != null && (originalIndices.isEmpty() == false)) { - String[] indices = originalIndices.split(","); - List newIndices = new ArrayList<>(); - for (String indexName : indices) { - newIndices.add(REMOTE_CLUSTER_NAME + ":" + indexName); + if (apiName.equals("scroll") == false && apiName.equals("clear_scroll") == false) { + String originalIndices = params.get("index"); + String expandedIndices = REMOTE_CLUSTER_NAME + ":*"; + if (originalIndices != null && (originalIndices.isEmpty() == false)) { + String[] indices = originalIndices.split(","); + List newIndices = new ArrayList<>(); + for (String indexName : indices) { + newIndices.add(REMOTE_CLUSTER_NAME + ":" + indexName); + } + expandedIndices = String.join(",", newIndices); } - expandedIndices = String.join(",", newIndices); + params.put("index", String.join(",", expandedIndices)); } - params.put("index", String.join(",", expandedIndices)); - ClientYamlTestResponse clientYamlTestResponse = super.callApi(apiName, params, entity, headers, nodeSelector); - return clientYamlTestResponse; + return super.callApi(apiName, params, entity, headers, nodeSelector); }; }; From f4eae856c5fb68c61d1fee0ac39e6feabcef7c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 16 May 2022 15:22:24 +0200 Subject: [PATCH 04/12] Add indices.resolve_index to tests --- qa/ccs-common-rest/build.gradle | 19 ++++--- .../rest/yaml/CcsCommonYamlTestSuiteIT.java | 56 +++++++++++++------ 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index 2c73947733e78..fed853c4b7e0e 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -22,7 +22,7 @@ restResources { 'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', "clear_scroll" } restTests { - includeCore 'field_caps', 'msearch', 'search', 'suggest', 'scroll' + includeCore 'field_caps', 'msearch', 'search', 'suggest', 'scroll', "indices.resolve_index" } } @@ -62,14 +62,15 @@ tasks.register("ccs-write", RestIntegTestTask) { systemProperty 'tests.rest.suite', [ - 'field_caps', - "msearch", - "scroll", - 'search', - 'search.aggregation', - 'search.highlight', - 'search.inner_hits', - 'suggest', +// 'field_caps', + 'indices.resolve_index', +// "msearch", +// "scroll", +// 'search', +// 'search.aggregation', +// 'search.highlight', +// 'search.inner_hits', +// 'suggest', ].join(',') systemProperty 'tests.rest.blacklist', diff --git a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index b8089522dfead..5ab0abfce30e2 100644 --- a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -51,7 +51,14 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { // the remote cluster is the one we write index operations etc... to private static final String REMOTE_CLUSTER_NAME = "remote_cluster"; // the following are the CCS api calls that we run against the "search" cluster in this test setup - private static final Set CCS_APIS = Set.of("search", "field_caps", "msearch", "scroll", "clear_scroll"); + private static final Set CCS_APIS = Set.of( + "search", + "field_caps", + "msearch", + "scroll", + "clear_scroll", + "indices.resolve_index" + ); @Before public void initSearchClient() throws IOException { @@ -100,7 +107,12 @@ public ClientYamlTestResponse callApi( ) throws IOException { // on request, we need to replace index specifications by prefixing the remote cluster if (apiName.equals("scroll") == false && apiName.equals("clear_scroll") == false) { - String originalIndices = params.get("index"); + String parameterName = "index"; + if (apiName.equals("indices.resolve_index")) { + // in this api the index parameter is called "name" + parameterName = "name"; + } + String originalIndices = params.get(parameterName); String expandedIndices = REMOTE_CLUSTER_NAME + ":*"; if (originalIndices != null && (originalIndices.isEmpty() == false)) { String[] indices = originalIndices.split(","); @@ -110,9 +122,11 @@ public ClientYamlTestResponse callApi( } expandedIndices = String.join(",", newIndices); } - params.put("index", String.join(",", expandedIndices)); + params.put(parameterName, String.join(",", expandedIndices)); } - return super.callApi(apiName, params, entity, headers, nodeSelector); + ClientYamlTestResponse clientYamlTestResponse = super.callApi(apiName, params, entity, headers, nodeSelector); + logger.info(clientYamlTestResponse.getBodyAsString()); + return clientYamlTestResponse; }; }; @@ -138,29 +152,35 @@ private static ClientYamlTestCandidate rewrite(ClientYamlTestCandidate clientYam ClientYamlTestSection testSection = clientYamlTestCandidate.getTestSection(); List executableSections = testSection.getExecutableSections(); List modifiedExecutableSections = new ArrayList<>(); + String lastAPIDoSection = ""; for (ExecutableSection section : executableSections) { + ExecutableSection rewrittenSection = section; if (section instanceof MatchAssertion) { MatchAssertion matchSection = (MatchAssertion) section; if (matchSection.getField().endsWith("_index")) { String modifiedExpectedValue = REMOTE_CLUSTER_NAME + ":" + matchSection.getExpectedValue(); - modifiedExecutableSections.add( - new MatchAssertion(matchSection.getLocation(), matchSection.getField(), modifiedExpectedValue) - ); + rewrittenSection = new MatchAssertion(matchSection.getLocation(), matchSection.getField(), modifiedExpectedValue); + } + if (lastAPIDoSection.equals("indices.resolve_index") && matchSection.getField().endsWith("name")) { + // modify " indices.resolve_index" expected index names + String modifiedExpectedValue = REMOTE_CLUSTER_NAME + ":" + matchSection.getExpectedValue(); + rewrittenSection = new MatchAssertion(matchSection.getLocation(), matchSection.getField(), modifiedExpectedValue); } - } else if (section instanceof DoSection && ((DoSection) section).getApiCallSection().getApi().equals("msearch")) { - // modify "msearch" body sections so the "index" part is targeting the remote cluster - DoSection doSection = ((DoSection) section); - List> bodies = doSection.getApiCallSection().getBodies(); - for (Map body : bodies) { - if (body.containsKey("index")) { - String modifiedIndex = REMOTE_CLUSTER_NAME + ":" + body.get("index"); - body.put("index", modifiedIndex); + } else if (section instanceof DoSection) { + lastAPIDoSection = ((DoSection) section).getApiCallSection().getApi(); + if (lastAPIDoSection.equals("msearch")) { + // modify "msearch" body sections so the "index" part is targeting the remote cluster + DoSection doSection = ((DoSection) section); + List> bodies = doSection.getApiCallSection().getBodies(); + for (Map body : bodies) { + if (body.containsKey("index")) { + String modifiedIndex = REMOTE_CLUSTER_NAME + ":" + body.get("index"); + body.put("index", modifiedIndex); + } } } - modifiedExecutableSections.add(section); - } else { - modifiedExecutableSections.add(section); } + modifiedExecutableSections.add(rewrittenSection); } return new ClientYamlTestCandidate( clientYamlTestCandidate.getRestTestSuite(), From 20e0c4af5ce57ae77ae70264764ce633df6d7a9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 16 May 2022 16:41:21 +0200 Subject: [PATCH 05/12] fix some tests that passed due to a wrong rewrite --- qa/ccs-common-rest/build.gradle | 16 ++++---- .../rest/yaml/CcsCommonYamlTestSuiteIT.java | 37 +++++++++++++------ 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index fed853c4b7e0e..5a85ce2461680 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -62,15 +62,15 @@ tasks.register("ccs-write", RestIntegTestTask) { systemProperty 'tests.rest.suite', [ -// 'field_caps', + 'field_caps', 'indices.resolve_index', -// "msearch", -// "scroll", -// 'search', -// 'search.aggregation', -// 'search.highlight', -// 'search.inner_hits', -// 'suggest', + "msearch", + "scroll", + 'search', + 'search.aggregation', + 'search.highlight', + 'search.inner_hits', + 'suggest', ].join(',') systemProperty 'tests.rest.blacklist', diff --git a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index 5ab0abfce30e2..28deb712136f4 100644 --- a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -124,9 +124,7 @@ public ClientYamlTestResponse callApi( } params.put(parameterName, String.join(",", expandedIndices)); } - ClientYamlTestResponse clientYamlTestResponse = super.callApi(apiName, params, entity, headers, nodeSelector); - logger.info(clientYamlTestResponse.getBodyAsString()); - return clientYamlTestResponse; + return super.callApi(apiName, params, entity, headers, nodeSelector); }; }; @@ -155,17 +153,19 @@ private static ClientYamlTestCandidate rewrite(ClientYamlTestCandidate clientYam String lastAPIDoSection = ""; for (ExecutableSection section : executableSections) { ExecutableSection rewrittenSection = section; - if (section instanceof MatchAssertion) { - MatchAssertion matchSection = (MatchAssertion) section; - if (matchSection.getField().endsWith("_index")) { - String modifiedExpectedValue = REMOTE_CLUSTER_NAME + ":" + matchSection.getExpectedValue(); - rewrittenSection = new MatchAssertion(matchSection.getLocation(), matchSection.getField(), modifiedExpectedValue); + if (section instanceof MatchAssertion matchSection) { + Object modifiedExpectedValue = ((MatchAssertion) section).getExpectedValue(); + if (matchSection.getField().endsWith("_index") || matchSection.getField().contains("fields._index")) { + modifiedExpectedValue = rewriteExpectedIndexValue(matchSection.getExpectedValue()); } if (lastAPIDoSection.equals("indices.resolve_index") && matchSection.getField().endsWith("name")) { // modify " indices.resolve_index" expected index names - String modifiedExpectedValue = REMOTE_CLUSTER_NAME + ":" + matchSection.getExpectedValue(); - rewrittenSection = new MatchAssertion(matchSection.getLocation(), matchSection.getField(), modifiedExpectedValue); + modifiedExpectedValue = rewriteExpectedIndexValue(matchSection.getExpectedValue()); } + if (lastAPIDoSection.equals("field_caps") && matchSection.getField().endsWith("indices")) { + modifiedExpectedValue = rewriteExpectedIndexValue(matchSection.getExpectedValue()); + } + rewrittenSection = new MatchAssertion(matchSection.getLocation(), matchSection.getField(), modifiedExpectedValue); } else if (section instanceof DoSection) { lastAPIDoSection = ((DoSection) section).getApiCallSection().getApi(); if (lastAPIDoSection.equals("msearch")) { @@ -191,7 +191,22 @@ private static ClientYamlTestCandidate rewrite(ClientYamlTestCandidate clientYam modifiedExecutableSections ) ); - }; + } + + /** + * add the remote cluster prefix to either a single index name or a list of expected index names + */ + private static Object rewriteExpectedIndexValue(Object expectedValue) { + if (expectedValue instanceof String) { + return REMOTE_CLUSTER_NAME + ":" + expectedValue; + } + if (expectedValue instanceof List) { + @SuppressWarnings("unchecked") + List expectedValues = (List) expectedValue; + return expectedValues.stream().map(s -> REMOTE_CLUSTER_NAME + ":" + s).toList(); + } + throw new IllegalArgumentException("Either String or List expected"); + } @ParametersFactory public static Iterable parameters() throws Exception { From d8b0ffaa0bab2f44b3f85f4c7997921b2d02323f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 18 May 2022 11:00:07 +0200 Subject: [PATCH 06/12] Comments for gradle build file --- qa/ccs-common-rest/build.gradle | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index 5a85ce2461680..4b0477cc25223 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -16,6 +16,14 @@ apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-resources' +// This subproject copies a subset of the rest-api-spec rest tests and runs them in a slightly +// modified way on two clusters connected via CCS. All operations except searches and a few other +// APIs that support CCS are run agains the "write" cluster, so all indexed data will live there. +// All search requests however are run against the other cluster connected via CCS to the "write" +// cluster. The test runner modifies index names on these API calls to route to the remote cluster +// and also modifies certain "match" sections to expect index names with the remote cluser prefix +// on the fly while running these tests. + restResources { restApi { include '_common', 'bulk', 'count', 'cluster', 'index', 'indices', 'field_caps', 'msearch', @@ -43,6 +51,8 @@ testClusters.configureEach { requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") } +// the following task is needed to make sure the write cluster is running before the search cluster +// gets configured with the remotes cluster seed tasks.register('startWriteCluster', DefaultTestClustersTask) { useCluster writeCluster doLast { @@ -60,6 +70,7 @@ tasks.register("ccs-write", RestIntegTestTask) { useCluster writeCluster useCluster searchCluster + // subset of test suit that are run in this module systemProperty 'tests.rest.suite', [ 'field_caps', @@ -92,7 +103,6 @@ tasks.register("ccs-write", RestIntegTestTask) { doFirst { - // Getting the endpoints causes a wait for the cluster println "Writer cluster endpoints are: ${-> writeCluster.get().allHttpSocketURI.join(",")}" println "Search cluster endpoints are: ${-> searchCluster.get().allHttpSocketURI.join(",")}" nonInputProperties.systemProperty('tests.rest.cluster', writeCluster.map(c -> c.allHttpSocketURI.join(","))) From 8b885b162ead37858a13f0b4ea288f8962456167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 18 May 2022 11:09:48 +0200 Subject: [PATCH 07/12] Comments for test runner --- .../rest/yaml/CcsCommonYamlTestSuiteIT.java | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index 28deb712136f4..faea2e2a87b2e 100644 --- a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -39,6 +39,15 @@ import static java.util.Collections.unmodifiableList; +/** + * This runner executes test suits against two clusters (a "write" (the remote) cluster and a + * "search" cluster) connected via CCS. + * The test runner maintains an additional client to the one provided by ESClientYamlSuiteTestCase + * That client instance (and a corresponding client only used for administration) is running all API calls + * defined in CCS_APIS against the "search" cluster, while all other operations like indexing are performed + * using the client running against the "write" cluster. + * + */ @TimeoutSuite(millis = 15 * TimeUnits.MINUTE) // to account for slow as hell VMs public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { @@ -50,7 +59,7 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { // the remote cluster is the one we write index operations etc... to private static final String REMOTE_CLUSTER_NAME = "remote_cluster"; - // the following are the CCS api calls that we run against the "search" cluster in this test setup + // the CCS api calls that we run against the "search" cluster in this test setup private static final Set CCS_APIS = Set.of( "search", "field_caps", @@ -60,6 +69,9 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { "indices.resolve_index" ); + /** + * initialize the search client and an additional administration client and check for an established connection + */ @Before public void initSearchClient() throws IOException { if (searchClient == null) { @@ -98,6 +110,8 @@ public void initSearchClient() throws IOException { os, this::getClientBuilderWithSniffedHosts ) { + // we overwrite this method so the search client can modify the index names by prefixing them with the + // remote cluster name before sending the requests public ClientYamlTestResponse callApi( String apiName, Map params, @@ -109,7 +123,7 @@ public ClientYamlTestResponse callApi( if (apiName.equals("scroll") == false && apiName.equals("clear_scroll") == false) { String parameterName = "index"; if (apiName.equals("indices.resolve_index")) { - // in this api the index parameter is called "name" + // in this specific api, the index parameter is called "name" parameterName = "name"; } String originalIndices = params.get(parameterName); @@ -146,6 +160,10 @@ public CcsCommonYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) throws IO super(rewrite(testCandidate)); } + /** + * we need to rewrite a few "match" sections in order to change the expected index name values + * to include the remote cluster prefix + */ private static ClientYamlTestCandidate rewrite(ClientYamlTestCandidate clientYamlTestCandidate) { ClientYamlTestSection testSection = clientYamlTestCandidate.getTestSection(); List executableSections = testSection.getExecutableSections(); @@ -218,6 +236,7 @@ protected ClientYamlTestExecutionContext createRestTestExecutionContext( ClientYamlTestCandidate clientYamlTestCandidate, ClientYamlTestClient clientYamlTestClient ) { + // depending on the API called, we either return the client running against the "write" or the "search" cluster here return new ClientYamlTestExecutionContext(clientYamlTestCandidate, clientYamlTestClient, randomizeContentType()) { protected ClientYamlTestClient clientYamlTestClient(String apiName) { if (CCS_APIS.contains(apiName)) { From 254f06fc062a9fa0b396869cc2e167d22973aeb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 19 May 2022 15:14:39 +0200 Subject: [PATCH 08/12] fix typos --- qa/ccs-common-rest/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index 4b0477cc25223..3e9662e65e707 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -18,10 +18,10 @@ apply plugin: 'elasticsearch.rest-resources' // This subproject copies a subset of the rest-api-spec rest tests and runs them in a slightly // modified way on two clusters connected via CCS. All operations except searches and a few other -// APIs that support CCS are run agains the "write" cluster, so all indexed data will live there. +// APIs that support CCS are run against the "write" cluster, so all indexed data will live there. // All search requests however are run against the other cluster connected via CCS to the "write" // cluster. The test runner modifies index names on these API calls to route to the remote cluster -// and also modifies certain "match" sections to expect index names with the remote cluser prefix +// and also modifies certain "match" sections to expect index names with the remote cluster prefix // on the fly while running these tests. restResources { @@ -70,7 +70,7 @@ tasks.register("ccs-write", RestIntegTestTask) { useCluster writeCluster useCluster searchCluster - // subset of test suit that are run in this module + // subset of test suite that are run in this module systemProperty 'tests.rest.suite', [ 'field_caps', From 43b08a031ef61996bd62f2ce11ec53587eca2b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 19 May 2022 15:33:21 +0200 Subject: [PATCH 09/12] Rename write/search cluster to remote/local --- qa/ccs-common-rest/build.gradle | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index 3e9662e65e707..b67e63c4be9d4 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -18,8 +18,8 @@ apply plugin: 'elasticsearch.rest-resources' // This subproject copies a subset of the rest-api-spec rest tests and runs them in a slightly // modified way on two clusters connected via CCS. All operations except searches and a few other -// APIs that support CCS are run against the "write" cluster, so all indexed data will live there. -// All search requests however are run against the other cluster connected via CCS to the "write" +// APIs that support CCS are run against the remote "write" cluster where all indexed data will live. +// All search requests however are run against the local cluster connected via CCS to the remote // cluster. The test runner modifies index names on these API calls to route to the remote cluster // and also modifies certain "match" sections to expect index names with the remote cluster prefix // on the fly while running these tests. @@ -34,16 +34,16 @@ restResources { } } -def writeCluster = testClusters.register("ccs-write") { +def remoteCluster = testClusters.register("ccs-remote") { numberOfNodes = 2 setting 'node.roles', '[data,ingest,master]' } -def searchCluster = testClusters.register("ccs-search") { +def localCluster = testClusters.register("ccs-local") { setting 'node.roles', '[data,ingest,master,remote_cluster_client]' setting 'cluster.remote.connections_per_cluster', '1' setting 'cluster.remote.remote_cluster.seeds', - { "\"${writeCluster.get().getAllTransportPortURI().get(0)}\"" } + { "\"${remoteCluster.get().getAllTransportPortURI().get(0)}\"" } } testClusters.configureEach { @@ -51,24 +51,24 @@ testClusters.configureEach { requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") } -// the following task is needed to make sure the write cluster is running before the search cluster +// the following task is needed to make sure the remote cluster is running before the local cluster // gets configured with the remotes cluster seed -tasks.register('startWriteCluster', DefaultTestClustersTask) { - useCluster writeCluster +tasks.register('startRemoteCluster', DefaultTestClustersTask) { + useCluster remoteCluster doLast { clusters.each { c -> - print "Writer cluster transport uri for ccs configuration is: " + print "Remote cluster transport uri for ccs configuration is: " println c.getAllTransportPortURI().get(0) } } } -tasks.register("ccs-write", RestIntegTestTask) { +tasks.register("ccs-remote", RestIntegTestTask) { mustRunAfter("precommit") - dependsOn startWriteCluster + dependsOn startRemoteCluster - useCluster writeCluster - useCluster searchCluster + useCluster remoteCluster + useCluster localCluster // subset of test suite that are run in this module systemProperty 'tests.rest.suite', @@ -103,10 +103,10 @@ tasks.register("ccs-write", RestIntegTestTask) { doFirst { - println "Writer cluster endpoints are: ${-> writeCluster.get().allHttpSocketURI.join(",")}" - println "Search cluster endpoints are: ${-> searchCluster.get().allHttpSocketURI.join(",")}" - nonInputProperties.systemProperty('tests.rest.cluster', writeCluster.map(c -> c.allHttpSocketURI.join(","))) - nonInputProperties.systemProperty('tests.rest.search_cluster', searchCluster.map(c -> c.allHttpSocketURI.join(","))) + println "Remote cluster endpoints are: ${-> remoteCluster.get().allHttpSocketURI.join(",")}" + println "Local cluster endpoints are: ${-> localCluster.get().allHttpSocketURI.join(",")}" + nonInputProperties.systemProperty('tests.rest.cluster', remoteCluster.map(c -> c.allHttpSocketURI.join(","))) + nonInputProperties.systemProperty('tests.rest.search_cluster', localCluster.map(c -> c.allHttpSocketURI.join(","))) } } From c55437a62e4ca34d60e2fcbcae4f0d282531ef05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 19 May 2022 15:49:44 +0200 Subject: [PATCH 10/12] Remove suite filter in test runner --- qa/ccs-common-rest/build.gradle | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index b67e63c4be9d4..7e8e2f7825c9f 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -70,20 +70,6 @@ tasks.register("ccs-remote", RestIntegTestTask) { useCluster remoteCluster useCluster localCluster - // subset of test suite that are run in this module - systemProperty 'tests.rest.suite', - [ - 'field_caps', - 'indices.resolve_index', - "msearch", - "scroll", - 'search', - 'search.aggregation', - 'search.highlight', - 'search.inner_hits', - 'suggest', - ].join(',') - systemProperty 'tests.rest.blacklist', [ // TODO look into fixing these From e792260136ae8227f1a0286f385fbfe7aa246ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 24 May 2022 12:54:30 +0200 Subject: [PATCH 11/12] Adding 'async_search' x-pack tests to set of tests run by this qa module --- qa/ccs-common-rest/build.gradle | 7 ++++--- .../rest/yaml/CcsCommonYamlTestSuiteIT.java | 19 +++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index 7e8e2f7825c9f..e30268b5072d6 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -27,10 +27,11 @@ apply plugin: 'elasticsearch.rest-resources' restResources { restApi { include '_common', 'bulk', 'count', 'cluster', 'index', 'indices', 'field_caps', 'msearch', - 'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', "clear_scroll" + 'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', 'clear_scroll' } restTests { includeCore 'field_caps', 'msearch', 'search', 'suggest', 'scroll', "indices.resolve_index" + includeXpack 'async_search' } } @@ -72,7 +73,6 @@ tasks.register("ccs-remote", RestIntegTestTask) { systemProperty 'tests.rest.blacklist', [ - // TODO look into fixing these 'search/70_response_filtering/Search with response filtering', // makes assertions on "_clusters" section 'search/150_rewrite_on_coordinator/Ensure that we fetch the document only once', // terms lookup query with index 'search/170_terms_query/Terms Query with No.of terms exceeding index.max_terms_count should FAIL', // terms lookup query with index @@ -84,7 +84,8 @@ tasks.register("ccs-remote", RestIntegTestTask) { 'search.aggregation/220_filters_bucket/cache hits', // node_selector? 'search.aggregation/50_filter/Standard queries get cached', 'search.aggregation/50_filter/Terms lookup gets cached', // terms lookup by "index" doesn't seem to work correctly - 'search.aggregation/70_adjacency_matrix/Terms lookup' // terms lookup by "index" doesn't seem to work correctly + 'search.aggregation/70_adjacency_matrix/Terms lookup', // terms lookup by "index" doesn't seem to work correctly + 'async_search/20-with-poin-in-time/Async search with point in time' // [indices] cannot be used with point in time ].join(',') diff --git a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index faea2e2a87b2e..1df4380901f7a 100644 --- a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -27,6 +27,8 @@ import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; import org.elasticsearch.test.rest.yaml.section.DoSection; import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion; +import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion; import org.elasticsearch.test.rest.yaml.section.MatchAssertion; import org.junit.AfterClass; import org.junit.Before; @@ -66,7 +68,11 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { "msearch", "scroll", "clear_scroll", - "indices.resolve_index" + "indices.resolve_index", + "async_search.submit", + "async_search.get", + "async_search.status", + "async_search.delete" ); /** @@ -120,7 +126,11 @@ public ClientYamlTestResponse callApi( NodeSelector nodeSelector ) throws IOException { // on request, we need to replace index specifications by prefixing the remote cluster - if (apiName.equals("scroll") == false && apiName.equals("clear_scroll") == false) { + if (apiName.equals("scroll") == false + && apiName.equals("clear_scroll") == false + && apiName.equals("async_search.get") == false + && apiName.equals("async_search.delete") == false + && apiName.equals("async_search.status") == false) { String parameterName = "index"; if (apiName.equals("indices.resolve_index")) { // in this specific api, the index parameter is called "name" @@ -184,6 +194,11 @@ private static ClientYamlTestCandidate rewrite(ClientYamlTestCandidate clientYam modifiedExpectedValue = rewriteExpectedIndexValue(matchSection.getExpectedValue()); } rewrittenSection = new MatchAssertion(matchSection.getLocation(), matchSection.getField(), modifiedExpectedValue); + } else if (section instanceof IsFalseAssertion falseAssertion) { + if (lastAPIDoSection.startsWith("async_") && ((IsFalseAssertion) section).getField().endsWith("_clusters")) { + // in ccs scenarios, the response "_cluster" section will be there + rewrittenSection = new IsTrueAssertion(falseAssertion.getLocation(), falseAssertion.getField()); + } } else if (section instanceof DoSection) { lastAPIDoSection = ((DoSection) section).getApiCallSection().getApi(); if (lastAPIDoSection.equals("msearch")) { From 8aa56c4c7104de5e39533324450e52c74886a3c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 24 May 2022 13:58:52 +0200 Subject: [PATCH 12/12] add 70_response_filtering/Search with response filtering --- qa/ccs-common-rest/build.gradle | 1 - .../elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index e30268b5072d6..f326181298615 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -73,7 +73,6 @@ tasks.register("ccs-remote", RestIntegTestTask) { systemProperty 'tests.rest.blacklist', [ - 'search/70_response_filtering/Search with response filtering', // makes assertions on "_clusters" section 'search/150_rewrite_on_coordinator/Ensure that we fetch the document only once', // terms lookup query with index 'search/170_terms_query/Terms Query with No.of terms exceeding index.max_terms_count should FAIL', // terms lookup query with index 'search/350_point_in_time/basic', // [indices] cannot be used with point in time diff --git a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index 1df4380901f7a..5a4df29b6488d 100644 --- a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -195,7 +195,8 @@ private static ClientYamlTestCandidate rewrite(ClientYamlTestCandidate clientYam } rewrittenSection = new MatchAssertion(matchSection.getLocation(), matchSection.getField(), modifiedExpectedValue); } else if (section instanceof IsFalseAssertion falseAssertion) { - if (lastAPIDoSection.startsWith("async_") && ((IsFalseAssertion) section).getField().endsWith("_clusters")) { + if ((lastAPIDoSection.startsWith("async_") || lastAPIDoSection.equals("search")) + && ((IsFalseAssertion) section).getField().endsWith("_clusters")) { // in ccs scenarios, the response "_cluster" section will be there rewrittenSection = new IsTrueAssertion(falseAssertion.getLocation(), falseAssertion.getField()); }