Merge branch 'master' into remove_thread_barrier

TESTING.asciidoc

@@ -42,7 +42,7 @@ In order to start with a different distribution use the `-Drun.distribution` arg
 To for example start the open source distribution:
 
 -------------------------------------
-./gradlew run -Drun.distribution=oss-zip
+./gradlew run -Drun.distribution=oss
 -------------------------------------
 
 ==== License type
@@ -631,3 +631,13 @@ inside `/etc/hosts`, e.g.:
 255.255.255.255 broadcasthost
 ::1             localhost ElasticMBP.local`
 ....
+
+== Benchmarking
+
+For changes that might affect the performance characteristics of Elasticsearch
+you should also run macrobenchmarks. We maintain a macrobenchmarking tool
+called https://github.com/elastic/rally[Rally]
+which you can use to measure the performance impact. It comes with a set of
+default benchmarks that we also
+https://elasticsearch-benchmarks.elastic.co/[run every night]. To get started,
+please see https://esrally.readthedocs.io/en/stable/[Rally's documentation].

benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterBenchmark.java

@@ -55,4 +55,3 @@ public TemporalAccessor parseJodaDate() {
         return jodaFormatter.parse("1234567890");
     }
 }
-

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -392,7 +392,7 @@ class BuildPlugin implements Plugin<Project> {
     static void requireJavaHome(Task task, int version) {
         Project rootProject = task.project.rootProject // use root project for global accounting
         if (rootProject.hasProperty('requiredJavaVersions') == false) {
-            rootProject.rootProject.ext.requiredJavaVersions = [:].withDefault{key -> return []}
+            rootProject.rootProject.ext.requiredJavaVersions = [:]
             rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
                 List<String> messages = []
                 for (entry in rootProject.requiredJavaVersions) {
@@ -415,7 +415,7 @@ class BuildPlugin implements Plugin<Project> {
                 throw new GradleException("JAVA${version}_HOME required to run task:\n${task}")
             }
         } else {
-            rootProject.requiredJavaVersions.get(version).add(task)
+            rootProject.requiredJavaVersions.getOrDefault(version, []).add(task)
         }
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -34,7 +34,7 @@ public class DocsTestPlugin extends RestTestPlugin {
         project.pluginManager.apply('elasticsearch.standalone-rest-test')
         super.apply(project)
         // The distribution can be configured with -Dtests.distribution on the command line
-        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'zip')
+        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'default')
         // Docs are published separately so no need to assemble
         project.tasks.assemble.enabled = false
         Map<String, String> defaultSubstitutions = [

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -29,7 +29,7 @@ class ClusterConfiguration {
     private final Project project
 
     @Input
-    String distribution = 'zip'
+    String distribution = 'default'
 
     @Input
     int numNodes = 1

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -29,7 +29,6 @@ import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
 import org.gradle.api.AntBuilder
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
-import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.artifacts.Configuration
@@ -87,7 +86,7 @@ class ClusterFormationTasks {
         Configuration currentDistro = project.configurations.create("${prefix}_elasticsearchDistro")
         Configuration bwcDistro = project.configurations.create("${prefix}_elasticsearchBwcDistro")
         Configuration bwcPlugins = project.configurations.create("${prefix}_elasticsearchBwcPlugins")
-        if (System.getProperty('tests.distribution', 'oss-zip') == 'integ-test-zip') {
+        if (System.getProperty('tests.distribution', 'oss') == 'integ-test-zip') {
             throw new Exception("tests.distribution=integ-test-zip is not supported")
         }
         configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
@@ -172,31 +171,31 @@ class ClusterFormationTasks {
 
     /** Adds a dependency on the given distribution */
     static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
+        // TEMP HACK
+        // The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated.
+        if (distro.equals('oss-zip')) {
+            distro = 'oss'
+        }
+        if (distro.equals('zip')) {
+            distro = 'default'
+        }
+        // END TEMP HACK
+        if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) {
+            throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
+        }
         Version version = Version.fromString(elasticsearchVersion)
         if (version.before('6.3.0') && distro.startsWith('oss-')) {
             distro = distro.substring('oss-'.length())
         }
-        String os = "linux"
-        if (Os.FAMILY_WINDOWS) {
-            os = "windows"
-        } else if (Os.FAMILY_MAC) {
-            os = "darwin"
-        }
-        String packaging = distro
-        if (distro.contains('tar')) {
-            packaging = 'tar.gz'\
-        } else if (distro.contains('zip')) {
-            packaging = 'zip'
-        }
-        String group = "downloads.${packaging}"
+        String group = "downloads.zip"
         if (distro.equals("integ-test-zip")) {
             group = "org.elasticsearch.distribution.integ-test-zip"
         }
         String artifactName = 'elasticsearch'
-        if (distro.contains('oss')) {
+        if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
             artifactName += '-oss'
         }
-        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@${packaging}")
+        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip")
     }
 
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
@@ -319,31 +318,13 @@ class ClusterFormationTasks {
           elasticsearch source tree. If this is a plugin built in the elasticsearch source tree or this is a distro in
           the elasticsearch source tree then this should be the version of elasticsearch built by the source tree.
           If it isn't then Bad Things(TM) will happen. */
-        Task extract
-
-        switch (node.config.distribution) {
-            case 'integ-test-zip':
-            case 'zip':
-            case 'oss-zip':
-                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
-                    from {
-                        project.zipTree(configuration.singleFile)
-                    }
-                    into node.baseDir
-                }
-                break;
-            case 'tar':
-            case 'oss-tar':
-                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
-                    from {
-                        project.tarTree(project.resources.gzip(configuration.singleFile))
-                    }
-                    into node.baseDir
-                }
-                break;
-            default:
-                throw new InvalidUserDataException("Unknown distribution: ${node.config.distribution}")
+        Task extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
+            from {
+                project.zipTree(configuration.singleFile)
+            }
+            into node.baseDir
         }
+
         return extract
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -29,9 +29,6 @@ import org.gradle.api.Project
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
-
-import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
-
 /**
  * A container for the files and configuration associated with a single node in a test cluster.
  */
@@ -125,8 +122,8 @@ class NodeInfo {
         baseDir = new File(project.buildDir, "cluster/${prefix} node${nodeNum}")
         pidFile = new File(baseDir, 'es.pid')
         this.nodeVersion = Version.fromString(nodeVersion)
-        homeDir = homeDir(baseDir, config.distribution, nodeVersion)
-        pathConf = pathConf(baseDir, config.distribution, nodeVersion)
+        homeDir = new File(baseDir, "elasticsearch-${nodeVersion}")
+        pathConf = new File(homeDir, 'config')
         if (config.dataDir != null) {
             dataDir = "${config.dataDir(nodeNum)}"
         } else {
@@ -299,41 +296,4 @@ class NodeInfo {
         }
         return dataDir
     }
-
-    /** Returns the directory elasticsearch home is contained in for the given distribution */
-    static File homeDir(File baseDir, String distro, String nodeVersion) {
-        String path
-        switch (distro) {
-            case 'integ-test-zip':
-            case 'zip':
-            case 'tar':
-            case 'oss-zip':
-            case 'oss-tar':
-                path = "elasticsearch-${nodeVersion}"
-                break
-            case 'rpm':
-            case 'deb':
-                path = "${distro}-extracted/usr/share/elasticsearch"
-                break
-            default:
-                throw new InvalidUserDataException("Unknown distribution: ${distro}")
-        }
-        return new File(baseDir, path)
-    }
-
-    static File pathConf(File baseDir, String distro, String nodeVersion) {
-        switch (distro) {
-            case 'integ-test-zip':
-            case 'zip':
-            case 'oss-zip':
-            case 'tar':
-            case 'oss-tar':
-                return new File(homeDir(baseDir, distro, nodeVersion), 'config')
-            case 'rpm':
-            case 'deb':
-                return new File(baseDir, "${distro}-extracted/etc/elasticsearch")
-            default:
-                throw new InvalidUserDataException("Unknown distribution: ${distro}")
-        }
-    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy

@@ -16,7 +16,7 @@ public class RunTask extends DefaultTask {
         clusterConfig.httpPort = 9200
         clusterConfig.transportPort = 9300
         clusterConfig.daemonize = false
-        clusterConfig.distribution = 'zip'
+        clusterConfig.distribution = 'default'
         project.afterEvaluate {
             ClusterFormationTasks.setup(project, name, this, clusterConfig)
         }

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -96,7 +96,7 @@ public void apply(Project project) {
             if (dockerComposeSupported(project) == false) {
                 project.getLogger().warn(
                     "Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose " +
-                        "but none could not be found so these will be skipped", project.getPath()
+                        "but none could be found so these will be skipped", project.getPath()
                 );
                 tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task ->
                     task.setEnabled(false)

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0
-lucene            = 8.0.0-snapshot-a1c6e642aa
+lucene            = 8.0.0-snapshot-83f9835
 
 # optional dependencies
 spatial4j         = 0.7

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -1177,7 +1177,7 @@ public void testIndexPutSettings() throws IOException {
         createIndex(index, Settings.EMPTY);
 
         assertThat(dynamicSetting.getDefault(Settings.EMPTY), not(dynamicSettingValue));
-        UpdateSettingsRequest dynamicSettingRequest = new UpdateSettingsRequest();
+        UpdateSettingsRequest dynamicSettingRequest = new UpdateSettingsRequest(index);
         dynamicSettingRequest.settings(Settings.builder().put(dynamicSettingKey, dynamicSettingValue).build());
         AcknowledgedResponse response = execute(dynamicSettingRequest, highLevelClient().indices()::putSettings,
                 highLevelClient().indices()::putSettingsAsync);
@@ -1187,7 +1187,7 @@ public void testIndexPutSettings() throws IOException {
         assertThat(indexSettingsAsMap.get(dynamicSettingKey), equalTo(String.valueOf(dynamicSettingValue)));
 
         assertThat(staticSetting.getDefault(Settings.EMPTY), not(staticSettingValue));
-        UpdateSettingsRequest staticSettingRequest = new UpdateSettingsRequest();
+        UpdateSettingsRequest staticSettingRequest = new UpdateSettingsRequest(index);
         staticSettingRequest.settings(Settings.builder().put(staticSettingKey, staticSettingValue).build());
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(staticSettingRequest,
                 highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync));
@@ -1207,7 +1207,7 @@ public void testIndexPutSettings() throws IOException {
         assertThat(indexSettingsAsMap.get(staticSettingKey), equalTo(staticSettingValue));
 
         assertThat(unmodifiableSetting.getDefault(Settings.EMPTY), not(unmodifiableSettingValue));
-        UpdateSettingsRequest unmodifiableSettingRequest = new UpdateSettingsRequest();
+        UpdateSettingsRequest unmodifiableSettingRequest = new UpdateSettingsRequest(index);
         unmodifiableSettingRequest.settings(Settings.builder().put(unmodifiableSettingKey, unmodifiableSettingValue).build());
         exception = expectThrows(ElasticsearchException.class, () -> execute(unmodifiableSettingRequest,
                 highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync));

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java

@@ -630,6 +630,22 @@ public void onFailure(Exception e) {
     public void testGetFollowStats() throws Exception {
         RestHighLevelClient client = highLevelClient();
 
+        {
+            // Create leader index:
+            CreateIndexRequest createIndexRequest = new CreateIndexRequest("leader");
+            createIndexRequest.settings(Collections.singletonMap("index.soft_deletes.enabled", true));
+            CreateIndexResponse response = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
+            assertThat(response.isAcknowledged(), is(true));
+        }
+        {
+            // Follow index, so that we can query for follow stats:
+            PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", "follower");
+            PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT);
+            assertThat(putFollowResponse.isFollowIndexCreated(), is(true));
+            assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true));
+            assertThat(putFollowResponse.isIndexFollowingStarted(), is(true));
+        }
+
         // tag::ccr-get-follow-stats-request
         FollowStatsRequest request =
             new FollowStatsRequest("follower"); // <1>
@@ -671,6 +687,12 @@ public void onFailure(Exception e) {
         // end::ccr-get-follow-stats-execute-async
 
         assertTrue(latch.await(30L, TimeUnit.SECONDS));
+
+        {
+            PauseFollowRequest pauseFollowRequest = new PauseFollowRequest("follower");
+            AcknowledgedResponse pauseFollowResponse =  client.ccr().pauseFollow(pauseFollowRequest, RequestOptions.DEFAULT);
+            assertThat(pauseFollowResponse.isAcknowledged(), is(true));
+        }
     }
 
     static Map<String, Object> toMap(Response response) throws IOException {

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutMappingRequestTests.java

@@ -19,15 +19,14 @@
 
 package org.elasticsearch.client.indices;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.RandomCreateIndexGenerator;
 import org.elasticsearch.test.AbstractXContentTestCase;
 
 import java.io.IOException;
+import java.util.Map;
 
-@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37654")
 public class PutMappingRequestTests extends AbstractXContentTestCase<PutMappingRequest> {
 
     @Override
@@ -47,7 +46,10 @@ protected PutMappingRequest createTestInstance() {
     @Override
     protected PutMappingRequest doParseInstance(XContentParser parser) throws IOException {
         PutMappingRequest request = new PutMappingRequest();
-        request.source(parser.map());
+        Map<String, Object> map = parser.map();
+        if (map.isEmpty() == false) {
+            request.source(map);
+        }
         return request;
     }
 
@@ -58,11 +60,16 @@ protected boolean supportsUnknownFields() {
 
     @Override
     protected void assertEqualInstances(PutMappingRequest expected, PutMappingRequest actual) {
-        try (XContentParser expectedJson = createParser(expected.xContentType().xContent(), expected.source());
-             XContentParser actualJson = createParser(actual.xContentType().xContent(), actual.source())) {
-            assertEquals(expectedJson.mapOrdered(), actualJson.mapOrdered());
-        } catch (IOException e) {
-            throw new RuntimeException(e);
+        if (actual.source() != null) {
+            try (XContentParser expectedJson = createParser(expected.xContentType().xContent(), expected.source());
+                    XContentParser actualJson = createParser(actual.xContentType().xContent(), actual.source())) {
+                assertEquals(expectedJson.mapOrdered(), actualJson.mapOrdered());
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        } else {
+            // if the original `source` is null, the parsed source should be so too
+            assertNull(expected.source());
         }
     }
 }

client/rest/src/main/java/org/elasticsearch/client/Response.java

@@ -103,7 +103,7 @@ public HttpEntity getEntity() {
     private static final Pattern WARNING_HEADER_PATTERN = Pattern.compile(
             "299 " + // warn code
             "Elasticsearch-\\d+\\.\\d+\\.\\d+(?:-(?:alpha|beta|rc)\\d+)?(?:-SNAPSHOT)?-(?:[a-f0-9]{7}|Unknown) " + // warn agent
-            "\"((?:\t| |!|[\\x23-\\x5B]|[\\x5D-\\x7E]|[\\x80-\\xFF]|\\\\|\\\\\")*)\" " + // quoted warning value, captured
+            "\"((?:\t| |!|[\\x23-\\x5B]|[\\x5D-\\x7E]|[\\x80-\\xFF]|\\\\|\\\\\")*)\"( " + // quoted warning value, captured
             // quoted RFC 1123 date format
             "\"" + // opening quote
             "(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), " + // weekday
@@ -112,7 +112,7 @@ public HttpEntity getEntity() {
             "\\d{4} " + // 4-digit year
             "\\d{2}:\\d{2}:\\d{2} " + // (two-digit hour):(two-digit minute):(two-digit second)
             "GMT" + // GMT
-            "\""); // closing quote
+            "\")?"); // closing quote (optional, since an older version can still send a warn-date)
 
     /**
      * Returns a list of all warning headers returned in the response.

distribution/build.gradle

@@ -337,8 +337,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
 }
 
 task run(type: RunTask) {
-  distribution = System.getProperty('run.distribution', 'zip')
-  if (distribution == 'zip') {
+  distribution = System.getProperty('run.distribution', 'default')
+  if (distribution == 'default') {
     String licenseType = System.getProperty("run.license_type", "basic")
     if (licenseType == 'trial') {
       setting 'xpack.ml.enabled', 'true'