Merge branch 'master' into replicated-closed-indices

.ci/java-versions.properties

@@ -8,3 +8,4 @@ ES_BUILD_JAVA=java11
 ES_RUNTIME_JAVA=java8
 GRADLE_TASK=build
 
+GRADLE_EXTRA_ARGS=-Dtests.bwc.refspec=elastic/index-lifecycle-6.x

CONTRIBUTING.md

@@ -159,7 +159,9 @@ Please follow these formatting guidelines:
 
 * Java indent is 4 spaces
 * Line width is 140 characters
-* Line width for code snippets that are included in the documentation (the ones surrounded by `// tag` and `// end` comments) is 76 characters
+* Lines of code surrounded by `// tag` and `// end` comments are included in the
+documentation and should only be 76 characters wide not counting
+leading indentation
 * The rest is left to Java coding standards
 * Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do.
 * Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. This can be done automatically by your IDE:
@@ -194,11 +196,26 @@ the settings window and/or restart IntelliJ to see your changes take effect.
 
 ### Creating A Distribution
 
-To create a distribution from the source, simply run:
+Run all build commands from within the root directory:
 
 ```sh
 cd elasticsearch/
-./gradlew assemble
+```
+
+To build a tar distribution, run this command:
+
+```sh
+./gradlew -p distribution/archives/tar assemble --parallel
+```
+
+You will find the distribution under:
+`./distribution/archives/tar/build/distributions/`
+
+To create all build artifacts (e.g., plugins and Javadocs) as well as
+distributions in all formats, run this command:
+
+```sh
+./gradlew assemble --parallel
 ```
 
 The package distributions (Debian and RPM) can be found under:
@@ -207,7 +224,6 @@ The package distributions (Debian and RPM) can be found under:
 The archive distributions (tar and zip) can be found under:
 `./distribution/archives/(tar|zip)/build/distributions/`
 
-
 ### Running The Full Test Suite
 
 Before submitting your changes, run the test suite to make sure that nothing is broken, with:

TESTING.asciidoc

@@ -229,34 +229,6 @@ Pass arbitrary jvm arguments.
 ./gradlew test -Dtests.jvm.argline="-Djava.security.debug=access,failure"
 ------------------------------
 
-== Backwards Compatibility Tests
-
-Running backwards compatibility tests is disabled by default since it
-requires a release version of elasticsearch to be present on the test system.
-To run backwards compatibility tests untar or unzip a release and run the tests
-with the following command:
-
----------------------------------------------------------------------------
-./gradlew test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.bwc.path=/path/to/elasticsearch -Dtests.security.manager=false
----------------------------------------------------------------------------
-
-Note that backwards tests must be run with security manager disabled.
-If the elasticsearch release is placed under `./backwards/elasticsearch-x.y.z` the path
-can be omitted:
-
----------------------------------------------------------------------------
-./gradlew test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.security.manager=false
----------------------------------------------------------------------------
-
-To setup the bwc test environment execute the following steps (provided you are
-already in your elasticsearch clone):
-
----------------------------------------------------------------------------
-$ mkdir backwards && cd backwards
-$ curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.2.1.tar.gz
-$ tar -xzf elasticsearch-1.2.1.tar.gz
----------------------------------------------------------------------------
-
 == Running verification tasks
 
 To run all verification tasks, including static checks, unit tests, and integration tests:
@@ -554,25 +526,28 @@ environment variable.
 == Testing backwards compatibility
 
 Backwards compatibility tests exist to test upgrading from each supported version
-to the current version. To run all backcompat tests use:
+to the current version. To run them all use:
 
 -------------------------------------------------
 ./gradlew bwcTest
 -------------------------------------------------
 
-A specific version can be tested as well. For example, to test backcompat with
+A specific version can be tested as well. For example, to test bwc with
 version 5.3.2 run:
 
 -------------------------------------------------
 ./gradlew v5.3.2#bwcTest
 -------------------------------------------------
 
-When running `./gradlew check`, some minimal backcompat checks are run. Which version
-is tested depends on the branch. On master, this will test against the current
-stable branch. On the stable branch, it will test against the latest release
-branch. Finally, on a release branch, it will test against the most recent release.
+Tests are ran for versions that are not yet released but with which the current version will be compatible with.
+These are automatically checked out and built from source.
+See link:./buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java[VersionCollection]
+and link:./distribution/bwc/build.gradle[distribution/bwc/build.gradle] 
+for more information.
+
+When running `./gradlew check`, minimal bwc checks are also run against compatible versions that are not yet released.
 
-=== BWC Testing against a specific remote/branch
+==== BWC Testing against a specific remote/branch
 
 Sometimes a backward compatibility change spans two versions. A common case is a new functionality
 that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
@@ -597,7 +572,7 @@ will contain your change.
 . Push both branches to your remote repository.
 . Run the tests with `./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec.5.x=index_req_bwc_5.x`.
 
-== Skip fetching latest
+==== Skip fetching latest
 
 For some BWC testing scenarios, you want to use the local clone of the
 repository without fetching latest. For these use cases, you can set the system

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java

@@ -46,10 +46,6 @@ public final class Allocators {
     private static class NoopGatewayAllocator extends GatewayAllocator {
         public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator();
 
-        protected NoopGatewayAllocator() {
-            super(Settings.EMPTY);
-        }
-
         @Override
         public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
             // noop
@@ -79,7 +75,7 @@ public static AllocationService createAllocationService(Settings settings) throw
 
     public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) throws
         InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
-        return new AllocationService(settings,
+        return new AllocationService(
             defaultAllocationDeciders(settings, clusterSettings),
             NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
     }
@@ -88,7 +84,7 @@ public static AllocationDeciders defaultAllocationDeciders(Settings settings, Cl
         IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException {
         Collection<AllocationDecider> deciders =
             ClusterModule.createAllocationDeciders(settings, clusterSettings, Collections.emptyList());
-        return new AllocationDeciders(settings, deciders);
+        return new AllocationDeciders(deciders);
 
     }
 

build.gradle

@@ -39,7 +39,7 @@ if (properties.get("org.elasticsearch.acceptScanTOS", "false") == "true") {
 // common maven publishing configuration
 subprojects {
   group = 'org.elasticsearch'
-  version = VersionProperties.elasticsearch.toString()
+  version = VersionProperties.elasticsearch
   description = "Elasticsearch subproject ${project.path}"
 }
 
@@ -103,10 +103,6 @@ subprojects {
  * in a branch if there are only betas and rcs in the branch so we have
  * *something* to test against. */
 VersionCollection versions = new VersionCollection(file('server/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
-if (versions.currentVersion != VersionProperties.elasticsearch) {
-  throw new GradleException("The last version in Versions.java [${versions.currentVersion}] does not match " +
-          "VersionProperties.elasticsearch [${VersionProperties.elasticsearch}]")
-}
 
 // build metadata from previous build, contains eg hashes for bwc builds
 String buildMetadataValue = System.getenv('BUILD_METADATA')
@@ -140,26 +136,16 @@ task verifyVersions {
     if (gradle.startParameter.isOffline()) {
       throw new GradleException("Must run in online mode to verify versions")
     }
-    // Read the list from maven central
-    Node xml
+    // Read the list from maven central. 
+    // Fetch the metadata an parse the xml into Version instances because it's more straight forward here 
+    // rather than bwcVersion ( VersionCollection ).
     new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
-        xml = new XmlParser().parse(s)
-    }
-    Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }.collect { Version.fromString(it) })
-
-    // Limit the known versions to those that should be index compatible, and are not future versions
-    knownVersions = knownVersions.findAll { it.major >= bwcVersions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }
-
-    /* Limit the listed versions to those that have been marked as released.
-     * Versions not marked as released don't get the same testing and we want
-     * to make sure that we flip all unreleased versions to released as soon
-     * as possible after release. */
-    Set<Version> actualVersions = new TreeSet<>(bwcVersions.indexCompatible.findAll { false == it.snapshot })
-
-    // Finally, compare!
-    if (knownVersions.equals(actualVersions) == false) {
-      throw new GradleException("out-of-date released versions\nActual  :" + actualVersions + "\nExpected:" + knownVersions +
-        "\nUpdate Version.java. Note that Version.CURRENT doesn't count because it is not released.")
+      bwcVersions.compareToAuthoritative(
+              new XmlParser().parse(s)
+                      .versioning.versions.version
+                      .collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }
+                      .collect { Version.fromString(it) }
+      )
     }
   }
 }
@@ -251,20 +237,17 @@ subprojects {
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
-
-  bwcVersions.snapshotProjectNames.each { snapshotName ->
-    Version snapshot = bwcVersions.getSnapshotForProject(snapshotName)
-    if (snapshot != null ) {
-      String snapshotProject = ":distribution:bwc:${snapshotName}"
-      project(snapshotProject).ext.bwcVersion = snapshot
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${snapshot}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${snapshot}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${snapshot}"] = snapshotProject
-      if (snapshot.onOrAfter('6.3.0')) {
-        ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch-oss:${snapshot}"] = snapshotProject
-        ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch-oss:${snapshot}"] = snapshotProject
-        ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch-oss:${snapshot}"] = snapshotProject
-      }
+  // substitute unreleased versions with projects that check out and build locally
+  bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
+    Version unreleased = unreleasedVersion.version
+    String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
+    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${unreleased}"] = snapshotProject
+    if (unreleased.onOrAfter('6.3.0')) {
+      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
     }
   }
 
@@ -299,7 +282,7 @@ subprojects {
     // other packages (e.g org.elasticsearch.client) will point to server rather than
     // their own artifacts.
     if (project.plugins.hasPlugin(BuildPlugin) || project.plugins.hasPlugin(PluginBuildPlugin)) {
-      String artifactsHost = VersionProperties.elasticsearch.isSnapshot() ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
+      String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
       Closure sortClosure = { a, b -> b.group <=> a.group }
       Closure depJavadocClosure = { shadowed, dep ->
         if (dep.group == null || false == dep.group.startsWith('org.elasticsearch')) {
@@ -572,11 +555,13 @@ wrapper {
     }
 }
 
-/* Remove assemble/dependenciesInfo on all qa projects because we don't need to publish
- * artifacts for them. */
 gradle.projectsEvaluated {
   subprojects {
-    if (project.path.startsWith(':qa')) {
+    /*
+     * Remove assemble/dependenciesInfo on all qa projects because we don't
+     * need to publish artifacts for them.
+     */
+    if (project.name.equals('qa') || project.path.contains(':qa:')) {
       Task assemble = project.tasks.findByName('assemble')
       if (assemble) {
         assemble.enabled = false

buildSrc/build.gradle

@@ -41,46 +41,33 @@ if (project == rootProject) {
  *         Propagating version.properties to the rest of the build           *
  *****************************************************************************/
 
-Properties props = new Properties()
-props.load(project.file('version.properties').newDataInputStream())
-version = props.getProperty('elasticsearch')
-boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true"));
-if (snapshot) {
-  // we update the version property to reflect if we are building a snapshot or a release build
-  // we write this back out below to load it in the Build.java which will be shown in rest main action
-  // to indicate this being a snapshot build or a release build.
-  version += "-SNAPSHOT"
-  props.put("elasticsearch", version);
-}
-
-File tempPropertiesFile = new File(project.buildDir, "version.properties")
-task writeVersionProperties {
-  inputs.properties(props)
-  outputs.file(tempPropertiesFile)
+// we update the version property to reflect if we are building a snapshot or a release build
+// we write this back out below to load it in the Build.java which will be shown in rest main action
+// to indicate this being a snapshot build or a release build.
+File propsFile = project.file('version.properties')
+Properties props = VersionPropertiesLoader.loadBuildSrcVersion(propsFile)
+version = props.getProperty("elasticsearch")
+processResources {
+  inputs.file(propsFile)
+  // We need to be explicit with the version because we add snapshot and qualifier to it based on properties
+  inputs.property("dynamic_elasticsearch_version", props.getProperty("elasticsearch"))
   doLast {
-    OutputStream stream = Files.newOutputStream(tempPropertiesFile.toPath());
+    Writer writer = file("$destinationDir/version.properties").newWriter()
     try {
-      props.store(stream, "UTF-8");
+      props.store(writer, "Generated version properties")
     } finally {
-      stream.close();
+      writer.close()
     }
   }
 }
 
-processResources {
-  dependsOn writeVersionProperties
-  from tempPropertiesFile
-}
-
-
-if (JavaVersion.current() < JavaVersion.VERSION_1_10) {
-  throw new GradleException('At least Java 10 is required to build elasticsearch gradle tools')
-}
-
 /*****************************************************************************
  *         Java version                                                      *
  *****************************************************************************/
 
+if (JavaVersion.current() < JavaVersion.VERSION_11) {
+  throw new GradleException('At least Java 11 is required to build elasticsearch gradle tools')
+}
 // Gradle 4.10 does not support setting this to 11 yet
 targetCompatibility = "10"
 sourceCompatibility = "10"
@@ -232,3 +219,42 @@ if (project != rootProject) {
     generatePomFileForPluginMavenPublication.enabled = false
   }
 }
+
+// Define this here because we need it early.
+class VersionPropertiesLoader {
+  static Properties loadBuildSrcVersion(File input) throws IOException {
+    Properties props = new Properties();
+    InputStream is = new FileInputStream(input)
+    try {
+      props.load(is)
+    } finally {
+      is.close()
+    }
+    loadBuildSrcVersion(props, System.getProperties())
+    return props
+  }
+
+  protected static void loadBuildSrcVersion(Properties loadedProps, Properties systemProperties) {
+    String elasticsearch = loadedProps.getProperty("elasticsearch")
+    if (elasticsearch == null) {
+      throw new IllegalStateException("Elasticsearch version is missing from properties.")
+    }
+    if (elasticsearch.matches("[0-9]+\\.[0-9]+\\.[0-9]+") == false) {
+      throw new IllegalStateException(
+              "Expected elasticsearch version to be numbers only of the form  X.Y.Z but it was: " +
+                      elasticsearch
+      )
+    }
+    String qualifier = systemProperties.getProperty("build.version_qualifier", "alpha1");
+    if (qualifier.isEmpty() == false) {
+      if (qualifier.matches("(alpha|beta|rc)\\d+") == false) {
+        throw new IllegalStateException("Invalid qualifier: " + qualifier)
+      }
+      elasticsearch += "-" + qualifier
+    }
+    if ("true".equals(systemProperties.getProperty("build.snapshot", "true"))) {
+      elasticsearch += "-SNAPSHOT"
+    }
+    loadedProps.put("elasticsearch", elasticsearch)
+  }
+}