Merge branch 'long_sort_optimization' into skip_optimization_on_homog…

…eneous_dataset

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -35,7 +35,7 @@
 import org.gradle.api.credentials.HttpHeaderCredentials;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
-import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.authentication.http.HttpHeaderAuthentication;
 
@@ -54,15 +54,19 @@
  */
 public class DistributionDownloadPlugin implements Plugin<Project> {
 
-    private static final String FAKE_GROUP = "elasticsearch-distribution";
+    private static final String FAKE_IVY_GROUP = "elasticsearch-distribution";
     private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads";
 
     private BwcVersions bwcVersions;
     private NamedDomainObjectContainer<ElasticsearchDistribution> distributionsContainer;
 
     @Override
     public void apply(Project project) {
-        distributionsContainer = project.container(ElasticsearchDistribution.class, name -> new ElasticsearchDistribution(name, project));
+        distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
+            Configuration fileConfiguration = project.getConfigurations().create("es_distro_file_" + name);
+            Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name);
+            return new ElasticsearchDistribution(name, project.getObjects(), fileConfiguration, extractedConfiguration);
+        });
         project.getExtensions().add("elasticsearch_distributions", distributionsContainer);
 
         setupDownloadServiceRepo(project);
@@ -112,18 +116,16 @@ private void setupRootDownload(Project rootProject, ElasticsearchDistribution di
         String extractedConfigName = "extracted_" + downloadConfigName;
         final Configuration downloadConfig = configurations.create(downloadConfigName);
         configurations.create(extractedConfigName);
-        Object distroDep = dependencyNotation(rootProject, distribution);
-        rootProject.getDependencies().add(downloadConfigName, distroDep);
+        rootProject.getDependencies().add(downloadConfigName, dependencyNotation(rootProject, distribution));
 
         // add task for extraction, delaying resolving config until runtime
         if (distribution.getType() == Type.ARCHIVE || distribution.getType() == Type.INTEG_TEST_ZIP) {
             Supplier<File> archiveGetter = downloadConfig::getSingleFile;
             String extractDir = rootProject.getBuildDir().toPath().resolve("elasticsearch-distros").resolve(extractedConfigName).toString();
-            TaskProvider<Copy> extractTask = rootProject.getTasks().register(extractTaskName, Copy.class, copyTask -> {
-                copyTask.dependsOn(downloadConfig);
-                copyTask.doFirst(t -> rootProject.delete(extractDir));
-                copyTask.into(extractDir);
-                copyTask.from((Callable<FileTree>)() -> {
+            TaskProvider<Sync> extractTask = rootProject.getTasks().register(extractTaskName, Sync.class, syncTask -> {
+                syncTask.dependsOn(downloadConfig);
+                syncTask.into(extractDir);
+                syncTask.from((Callable<FileTree>)() -> {
                     File archiveFile = archiveGetter.get();
                     String archivePath = archiveFile.toString();
                     if (archivePath.endsWith(".zip")) {
@@ -155,12 +157,12 @@ private static void setupDownloadServiceRepo(Project project) {
             });
             ivyRepo.getAuthentication().create("header", HttpHeaderAuthentication.class);
             ivyRepo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/[module]-[revision](-[classifier]).[ext]"));
-            ivyRepo.content(content -> content.includeGroup(FAKE_GROUP));
+            ivyRepo.content(content -> content.includeGroup(FAKE_IVY_GROUP));
         });
         project.getRepositories().all(repo -> {
             if (repo.getName().equals(DOWNLOAD_REPO_NAME) == false) {
                 // all other repos should ignore the special group name
-                repo.content(content -> content.excludeGroup(FAKE_GROUP));
+                repo.content(content -> content.excludeGroup(FAKE_IVY_GROUP));
             }
         });
         // TODO: need maven repo just for integ-test-zip, but only in external cases
@@ -199,7 +201,7 @@ private Object dependencyNotation(Project project, ElasticsearchDistribution dis
             extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
             classifier = distribution.getPlatform() + "-" + classifier;
         }
-        return FAKE_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":")
+        return FAKE_IVY_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":")
             + distribution.getVersion() + ":" + classifier + "@" + extension;
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -20,8 +20,8 @@
 package org.elasticsearch.gradle;
 
 import org.gradle.api.Buildable;
-import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
 import org.gradle.api.tasks.TaskDependency;
 
@@ -107,17 +107,18 @@ public String toString() {
     private final Property<Flavor> flavor;
     private final Property<Boolean> bundledJdk;
 
-    ElasticsearchDistribution(String name, Project project) {
+    ElasticsearchDistribution(String name, ObjectFactory objectFactory, Configuration fileConfiguration,
+                              Configuration extractedConfiguration) {
         this.name = name;
-        this.configuration = project.getConfigurations().create("es_distro_file_" + name);
-        this.version = project.getObjects().property(Version.class);
+        this.configuration = fileConfiguration;
+        this.version = objectFactory.property(Version.class);
         this.version.convention(Version.fromString(VersionProperties.getElasticsearch()));
-        this.type = project.getObjects().property(Type.class);
+        this.type = objectFactory.property(Type.class);
         this.type.convention(Type.ARCHIVE);
-        this.platform = project.getObjects().property(Platform.class);
-        this.flavor = project.getObjects().property(Flavor.class);
-        this.bundledJdk = project.getObjects().property(Boolean.class);
-        this.extracted = new Extracted(project.getConfigurations().create("es_distro_extracted_" + name));
+        this.platform = objectFactory.property(Platform.class);
+        this.flavor = objectFactory.property(Flavor.class);
+        this.bundledJdk = objectFactory.property(Boolean.class);
+        this.extracted = new Extracted(extractedConfiguration);
     }
 
     public String getName() {

buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java

@@ -129,18 +129,14 @@ public boolean wait(int durationInMs) throws GeneralSecurityException, Interrupt
     }
 
     protected void checkResource(SSLContext ssl) throws IOException {
-        try {
-            final HttpURLConnection connection = buildConnection(ssl);
-            connection.connect();
-            final Integer response = connection.getResponseCode();
-            if (validResponseCodes.contains(response)) {
-                logger.info("Got successful response [{}] from URL [{}]", response, url);
-                return;
-            } else {
-                throw new IOException(response + " " + connection.getResponseMessage());
-            }
-        } catch (IOException e) {
-            throw e;
+        final HttpURLConnection connection = buildConnection(ssl);
+        connection.connect();
+        final Integer response = connection.getResponseCode();
+        if (validResponseCodes.contains(response)) {
+            logger.info("Got successful response [{}] from URL [{}]", response, url);
+            return;
+        } else {
+            throw new IOException(response + " " + connection.getResponseMessage());
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -75,6 +75,8 @@ public ElasticsearchCluster(String path, String clusterName, Project project, Fi
                 services, artifactsExtractDir, workingDirBase
             )
         );
+        // configure the cluster name eagerly so nodes know about it
+        this.nodes.all((node) -> node.defaultConfig.put("cluster.name", safeName(clusterName)));
 
         addWaitForClusterHealth();
     }
@@ -215,12 +217,19 @@ public void setJavaHome(File javaHome) {
 
     @Override
     public void start() {
-        String nodeNames = nodes.stream().map(ElasticsearchNode::getName).collect(Collectors.joining(","));
+        final String nodeNames;
+        if (nodes.stream().map(ElasticsearchNode::getName).anyMatch( name -> name == null)) {
+            nodeNames = null;
+        } else {
+            nodeNames = nodes.stream().map(ElasticsearchNode::getName).collect(Collectors.joining(","));
+        };
         for (ElasticsearchNode node : nodes) {
-            node.defaultConfig.put("cluster.name", safeName(clusterName));
-            if (Version.fromString(node.getVersion()).getMajor() >= 7) {
-                node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]");
-                node.defaultConfig.put("discovery.seed_providers", "file");
+            if (nodeNames != null) {
+                // Can only configure master nodes if we have node names defined
+                if (Version.fromString(node.getVersion()).getMajor() >= 7) {
+                    node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]");
+                    node.defaultConfig.put("discovery.seed_providers", "file");
+                }
             }
             node.start();
         }
@@ -319,7 +328,6 @@ public ElasticsearchNode singleNode() {
 
     private void addWaitForClusterHealth() {
         waitConditions.put("cluster health yellow", (node) -> {
-
             try {
                 boolean httpSslEnabled = getFirstNode().isHttpSslEnabled();
                 WaitForHttpResource wait = new WaitForHttpResource(
@@ -328,7 +336,8 @@ private void addWaitForClusterHealth() {
                     nodes.size()
                 );
                 if (httpSslEnabled) {
-                    wait.setCertificateAuthorities(getFirstNode().getHttpCertificateAuthoritiesFile());
+
+                    getFirstNode().configureHttpWait(wait);
                 }
                 List<Map<String, String>> credentials = getFirstNode().getCredentials();
                 if (getFirstNode().getCredentials().isEmpty() == false) {
@@ -337,7 +346,7 @@ private void addWaitForClusterHealth() {
                 }
                 return wait.wait(500);
             } catch (IOException e) {
-                throw new IllegalStateException("Connection attempt to " + this + " failed", e);
+                throw new UncheckedIOException("IO error while waiting cluster", e);
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
                 throw new TestClustersException("Interrupted while waiting for " + this, e);

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.gradle.FileSupplier;
 import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.http.WaitForHttpResource;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 
@@ -581,7 +582,11 @@ public List<String> getAllTransportPortURI() {
     }
 
     public File getServerLog() {
-        return confPathLogs.resolve(safeName(getName()).replaceAll("-[0-9]+$", "") + "_server.json").toFile();
+        return confPathLogs.resolve(defaultConfig.get("cluster.name") + "_server.json").toFile();
+    }
+
+    public File getAuditLog() {
+        return confPathLogs.resolve(defaultConfig.get("cluster.name") + "_audit.json").toFile();
     }
 
     @Override
@@ -727,7 +732,10 @@ private void syncWithLinks(Path sourceRoot, Path destinationRoot) {
     }
 
     private void createConfiguration()  {
-        defaultConfig.put("node.name", nameCustomization.apply(safeName(name)));
+        String nodeName = nameCustomization.apply(safeName(name));
+        if (nodeName != null) {
+            defaultConfig.put("node.name", nodeName);
+        }
         defaultConfig.put("path.repo", confPathRepo.toAbsolutePath().toString());
         defaultConfig.put("path.data", confPathData.toAbsolutePath().toString());
         defaultConfig.put("path.logs", confPathLogs.toAbsolutePath().toString());
@@ -880,12 +888,32 @@ public boolean isHttpSslEnabled() {
         );
     }
 
-    public File getHttpCertificateAuthoritiesFile() {
-        if (settings.containsKey("xpack.security.http.ssl.certificate_authorities") == false) {
-            throw new TestClustersException("Can't get certificates authority file, not configured for " + this);
+    void configureHttpWait(WaitForHttpResource wait) {
+        if (settings.containsKey("xpack.security.http.ssl.certificate_authorities")) {
+            wait.setCertificateAuthorities(
+                getConfigDir()
+                    .resolve(settings.get("xpack.security.http.ssl.certificate_authorities").get().toString())
+                    .toFile()
+            );
+        }
+        if (settings.containsKey("xpack.security.http.ssl.certificate")) {
+            wait.setCertificateAuthorities(
+                getConfigDir()
+                    .resolve(settings.get("xpack.security.http.ssl.certificate").get().toString())
+                    .toFile()
+            );
+        }
+        if (settings.containsKey("xpack.security.http.ssl.keystore.path")) {
+            wait.setTrustStoreFile(
+                getConfigDir()
+                    .resolve(settings.get("xpack.security.http.ssl.keystore.path").get().toString())
+                    .toFile()
+            );
+        }
+        if (keystoreSettings.containsKey("xpack.security.http.ssl.keystore.secure_password")) {
+            wait.setTrustStorePassword(
+                keystoreSettings.get("xpack.security.http.ssl.keystore.secure_password").get().toString()
+            );
         }
-        return getConfigDir()
-            .resolve(settings.get("xpack.security.http.ssl.certificate_authorities").get().toString())
-            .toFile();
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -122,7 +122,7 @@ default void waitForConditions(
                 } catch (TestClustersException e) {
                     throw e;
                 } catch (Exception e) {
-                    throw  e;
+                    lastException = e;
                 }
             }
             if (conditionMet == false) {
@@ -131,7 +131,17 @@ default void waitForConditions(
                 if (lastException == null) {
                     throw new TestClustersException(message);
                 } else {
-                    throw new TestClustersException(message + message, lastException);
+                    String extraCause = "";
+                    Throwable cause = lastException;
+                    int ident = 2;
+                    while (cause != null) {
+                        if (cause.getMessage() != null && cause.getMessage().isEmpty() == false) {
+                            extraCause += "\n" + " ".repeat(ident) + cause.getMessage();
+                            ident += 2;
+                        }
+                        cause = cause.getCause();
+                    }
+                    throw new TestClustersException(message + extraCause, lastException);
                 }
             }
             logger.info(

client/rest-high-level/build.gradle

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'nebula.maven-base-publish'
@@ -115,14 +116,15 @@ if (isEclipse) {
 File nodeCert = file("./testnode.crt")
 File nodeTrustStore = file("./testnode.jks")
 
-integTestRunner {
+integTest.runner {
   systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
   systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'test-password')
 }
 
-integTestCluster {
+testClusters.integTest {
+  distribution = "DEFAULT"
   systemProperty 'es.scripting.update.ctx_in_params', 'false'
-  setting 'reindex.remote.whitelist', ['"[::1]:*"', '"127.0.0.1:*"']
+  setting 'reindex.remote.whitelist', '[ "[::1]:*", "127.0.0.1:*" ]'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.authc.token.enabled', 'true'
@@ -131,22 +133,10 @@ integTestCluster {
   setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
   setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks'
   setting 'indices.lifecycle.poll_interval', '1000ms'
-  keystoreSetting 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
-  setupCommand 'setupDummyUser',
-          'bin/elasticsearch-users',
-          'useradd', System.getProperty('tests.rest.cluster.username', 'test_user'),
-          '-p', System.getProperty('tests.rest.cluster.password', 'test-password'),
-          '-r', 'superuser'
+  keystore 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
+  user username:  System.getProperty('tests.rest.cluster.username', 'test_user'),
+          password:  System.getProperty('tests.rest.cluster.password', 'test-password')
+
   extraConfigFile nodeCert.name, nodeCert
   extraConfigFile nodeTrustStore.name, nodeTrustStore
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: System.getProperty('tests.rest.cluster.username', 'test_user'),
-            password: System.getProperty('tests.rest.cluster.password', 'test-password'),
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
 }