Merge remote-tracking branch 'elastic/master' into bundled-usage

* elastic/master: (77 commits)
  Update ingest jdocs that a null return value will drop the current document. (elastic#40359)
  Remove -Xlint exclusions in the ingest-common module. (elastic#40505)
  Update docs for the DFR similarity (elastic#40579)
  Fix merging of search_as_you_type field mapper (elastic#40593)
  Support roles with application privileges against wildcard applications (elastic#40398)
  Remove obsolete security settings (elastic#40496)
  Remove Gradle deprecation warnings (elastic#40449)
  Run the build integ test in parallel (elastic#39788)
  Fix 3rd pary S3 tests (elastic#40588)
  lower bwc skip for search as you type (elastic#40599)
  Muting XContentParserTests#testSubParserArray
  Fixing typo in test error message (elastic#40611)
  Update max dims for vectors to 1024. (elastic#40597)
  Add start and stop time to cat recovery API (elastic#40378)
  [DOCS] Correct keystore commands for Email and Jira actions in Watcher (elastic#40417)
  [DOCS] Document common settings for snapshot repository plugins (elastic#40475)
  Remove with(out)-system-key tests (elastic#40547)
  Geo Point parse error fix (elastic#40447)
  Handle null retention leases in WaitForNoFollowersStep (elastic#40477)
  [DOCS] Adds anchors for ruby client (elastic#39867)
  ...

.ci/packer_cache.sh

@@ -20,5 +20,5 @@ export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA}
 # We are caching BWC versions too, need these so we can build those
 export JAVA8_HOME="${HOME}"/.java/java8
 export JAVA11_HOME="${HOME}"/.java/java11
-export JAVA12_HOME="${HOME}"/.java/java12
+export JAVA12_HOME="${HOME}"/.java/openjdk12
 ./gradlew --parallel clean --scan -Porg.elasticsearch.acceptScanTOS=true -s resolveAllDependencies

build.gradle

@@ -32,7 +32,7 @@ plugins {
     id 'com.gradle.build-scan' version '2.0.2'
     id 'base'
 }
-if (properties.get("org.elasticsearch.acceptScanTOS", "false") == "true") {
+if (Boolean.valueOf(project.findProperty('org.elasticsearch.acceptScanTOS') ?: "false")) {
     buildScan {
         termsOfServiceUrl = 'https://gradle.com/terms-of-service'
         termsOfServiceAgree = 'yes'
@@ -163,7 +163,7 @@ task verifyVersions {
  */
 
 boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/40616" /* place a PR link here when committing bwc changes */
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/40319" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/build.gradle

@@ -232,6 +232,11 @@ if (project != rootProject) {
     if (isLuceneSnapshot) {
       systemProperty 'test.lucene-snapshot-revision', isLuceneSnapshot[0][1]
     }
+    String defaultParallel = System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel)
+    if (defaultParallel == "auto") {
+        defaultParallel = Math.max(Runtime.getRuntime().availableProcessors(), 4)
+    }
+    maxParallelForks defaultParallel as Integer
   }
   check.dependsOn(integTest)
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -44,6 +44,7 @@ public class PluginBuildPlugin extends BuildPlugin {
     public void apply(Project project) {
         super.apply(project)
         configureDependencies(project)
+
         // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
         project.afterEvaluate {
@@ -69,7 +70,7 @@ public class PluginBuildPlugin extends BuildPlugin {
                 if (isModule) {
                     throw new RuntimeException("Testclusters does not support modules yet");
                 } else {
-                    project.testClusters.integTestCluster.plugin(
+                    project.testClusters.integTest.plugin(
                             project.file(project.tasks.bundlePlugin.archiveFile)
                     )
                 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -62,13 +62,13 @@ public class RestIntegTestTask extends DefaultTask {
             clusterConfig = project.extensions.create("${name}Cluster", ClusterConfiguration.class, project)
         } else {
             project.testClusters {
-                integTestCluster {
+                "$name" {
                     distribution = 'INTEG_TEST'
                     version = project.version
                     javaHome = project.file(project.ext.runtimeJavaHome)
                 }
             }
-            runner.useCluster project.testClusters.integTestCluster
+            runner.useCluster project.testClusters."$name"
         }
 
         // override/add more for rest tests
@@ -81,7 +81,7 @@ public class RestIntegTestTask extends DefaultTask {
                 throw new IllegalArgumentException("tests.rest.cluster and tests.cluster must both be null or non-null")
             }
             if (usesTestclusters == true) {
-                ElasticsearchNode node = project.testClusters.integTestCluster
+                ElasticsearchNode node = project.testClusters."${name}"
                 runner.systemProperty('tests.rest.cluster', {node.allHttpSocketURI.join(",") })
                 runner.systemProperty('tests.config.dir', {node.getConfigDir()})
                 runner.systemProperty('tests.cluster', {node.transportPortURI})
@@ -187,6 +187,10 @@ public class RestIntegTestTask extends DefaultTask {
         clusterInit.mustRunAfter(tasks)
     }
 
+    public void runner(Closure configure) {
+        project.tasks.getByName("${name}Runner").configure(configure)
+    }
+
     /** Print out an excerpt of the log from the given node. */
     protected static void printLogExcerpt(NodeInfo nodeInfo) {
         File logFile = new File(nodeInfo.homeDir, "logs/${nodeInfo.clusterName}.log")

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -78,7 +78,6 @@ class VagrantTestPlugin implements Plugin<Project> {
     private static final PACKAGING_TEST_CONFIGURATION = 'packagingTest'
     private static final BATS = 'bats'
     private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
-    private static final String PLATFORM_TEST_COMMAND ="rm -rf ~/elasticsearch && rsync -r /elasticsearch/ ~/elasticsearch && cd ~/elasticsearch && ./gradlew test integTest"
 
     /** Boxes that have been supplied and are available for testing **/
     List<String> availableBoxes = []
@@ -388,15 +387,6 @@ class VagrantTestPlugin implements Plugin<Project> {
         }
     }
 
-    private static void createPlatformTestTask(Project project) {
-        project.tasks.create('platformTest') {
-            group 'Verification'
-            description "Test unit and integ tests on different platforms using vagrant. See TESTING.asciidoc for details. This test " +
-                    "is unmaintained."
-            dependsOn 'vagrantCheckVersion'
-        }
-    }
-
     private void createBoxListTasks(Project project) {
         project.tasks.create('listAllBoxes') {
             group 'Verification'
@@ -429,7 +419,6 @@ class VagrantTestPlugin implements Plugin<Project> {
         createSmokeTestTask(project)
         createPrepareVagrantTestEnvTask(project)
         createPackagingTestTask(project)
-        createPlatformTestTask(project)
         createBoxListTasks(project)
     }
 
@@ -454,9 +443,6 @@ class VagrantTestPlugin implements Plugin<Project> {
         assert project.tasks.packagingTest != null
         Task packagingTest = project.tasks.packagingTest
 
-        assert project.tasks.platformTest != null
-        Task platformTest = project.tasks.platformTest
-
         /*
          * We always use the main project.rootDir as Vagrant's current working directory (VAGRANT_CWD)
          * so that boxes are not duplicated for every Gradle project that use this VagrantTestPlugin.
@@ -610,31 +596,6 @@ class VagrantTestPlugin implements Plugin<Project> {
                     packagingTest.dependsOn(javaPackagingTest)
                 }
             }
-
-            /*
-             * This test is unmaintained and was created to run on Linux. We won't allow it to run on Windows
-             * until it's been brought back into maintenance
-             */
-            if (LINUX_BOXES.contains(box)) {
-                Task platform = project.tasks.create("vagrant${boxTask}#platformTest", VagrantCommandTask) {
-                    command 'ssh'
-                    boxName box
-                    environmentVars vagrantEnvVars
-                    dependsOn up
-                    finalizedBy halt
-                    args '--command', PLATFORM_TEST_COMMAND + " -Dtests.seed=${-> project.testSeed}"
-                }
-                TaskExecutionAdapter platformReproListener = createReproListener(project, platform.path)
-                platform.doFirst {
-                    project.gradle.addListener(platformReproListener)
-                }
-                platform.doLast {
-                    project.gradle.removeListener(platformReproListener)
-                }
-                if (project.extensions.esvagrant.boxes.contains(box)) {
-                    platformTest.dependsOn(platform)
-                }
-            }
         }
     }