Merge remote-tracking branch 'upstream/master' into long_sort_optimiz…

…ation

.ci/build.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
 
+JAVA_HOME=${JAVA_HOME:-$HOME/.java/openjdk12}
+RUNTIME_JAVA_HOME=${RUNTIME_JAVA_HOME:-$HOME/.java/openjdk11}
+
+JAVA7_HOME=$HOME/.java/java7
+JAVA8_HOME=$HOME/.java/java8
+JAVA9_HOME=$HOME/.java/java9
+JAVA10_HOME=$HOME/.java/java10
+JAVA11_HOME=$HOME/.java/java11
+JAVA12_HOME=$HOME/.java/openjdk12
+JAVA13_HOME=$HOME/.java/openjdk13
+
 # drop page cache and kernel slab objects on linux
 [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
 
@@ -10,9 +21,10 @@ if [ -f /proc/cpuinfo ] ; then
    MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
 else
    if [[ "$OSTYPE" == "darwin"* ]]; then
-      MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
-      # Looks like it's too much for our workers so reduce it further
-      MAX_WORKERS=$(($MAX_WORKERS/2))
+      # Parallel is disabled at  this time (eventually set to 1) due to errors on the Mac workers
+      # We'll have to do more testing to see if this can be re-enabled or what the proper value is.
+      # MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
+      MAX_WORKERS=2
    else
       echo "Unsupported OS Type: $OSTYPE"
       exit 1
@@ -29,4 +41,4 @@ set -e
 ./gradlew --parallel --scan \
   -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ \
   --parallel --max-workers=$MAX_WORKERS \
-   "$@"
+   "$@"

.ci/bwcVersions

@@ -11,6 +11,7 @@ BWC_VERSION:
   - "7.4.0"
   - "7.4.1"
   - "7.4.2"
+  - "7.4.3"
   - "7.5.0"
   - "7.6.0"
   - "8.0.0"

.ci/jobs.t/defaults.yml

@@ -35,30 +35,17 @@
           fail: true
       - ansicolor
       - timestamps
-      # TODO: No support un JJBB ?
-      # - gradle-build-scan
+      - gradle-build-scan
     properties:
         - github:
             url: https://github.com/elastic/elasticsearch/
         - inject:
             properties-content: |
               HOME=$JENKINS_HOME
     builders:
-       - inject:
-          properties-file: '.ci/java-versions.properties'
-          properties-content: |
-            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
-            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
-            JAVA7_HOME=$HOME/.java/java7
-            JAVA8_HOME=$HOME/.java/java8
-            JAVA9_HOME=$HOME/.java/java9
-            JAVA10_HOME=$HOME/.java/java10
-            JAVA11_HOME=$HOME/.java/java11
-            JAVA12_HOME=$HOME/.java/openjdk12
-            JAVA13_HOME=$HOME/.java/openjdk13
        - shell: |
           #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push
-          .ci/build.sh check
+          .ci/build.sh -Dbwc.checkout.align=true check
     publishers:
       - email:
           recipients: infra-root+build@elastic.co

.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml

@@ -5,7 +5,3 @@
     description: "Periodic testing of the Elasticsearch %BRANCH% branch.\n"
     triggers:
       - timed: "H H/1 * * *"
-    builders:
-      - shell: |
-          #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push
-          .ci/build.sh -Dbwc.checkout.align=true check

CONTRIBUTING.md

@@ -107,7 +107,9 @@ and `JAVA11_HOME` available so that the tests can pass.
 `jrunscript` for jdk distributions.
 
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
-using the wrapper via the `gradlew` script in the root of the repository.
+using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat` 
+script on Windows in the root of the repository. The examples below show the
+usage on Unix.
 
 We support development in the Eclipse and IntelliJ IDEs.
 For Eclipse, the minimum version that we support is [4.13][eclipse].

build.gradle

@@ -41,6 +41,7 @@ plugins {
 apply plugin: 'nebula.info-scm'
 apply from: 'gradle/build-scan.gradle'
 apply from: 'gradle/build-complete.gradle'
+apply from: 'gradle/runtime-jdk-provision.gradle'
 
 // common maven publishing configuration
 allprojects {

buildSrc/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 
+import org.gradle.internal.jvm.Jvm
 import org.gradle.util.GradleVersion
 
 plugins {
@@ -168,6 +169,9 @@ if (project != rootProject) {
   forbiddenApisTest.enabled = false
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
+  if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))){
+    test.enabled = false
+  }
 
   configurations {
     distribution
@@ -223,8 +227,13 @@ if (project != rootProject) {
   }
   check.dependsOn(integTest)
 
+  // for now we hardcode the tests for our build to use the gradle jvm.
+  tasks.withType(Test).configureEach {
+    it.executable = Jvm.current().getJavaExecutable()
+  }
+
   /*
-   * We alread configure publication and we don't need or want this one that
+   * We already configure publication and we don't need or want this one that
    * comes from the java-gradle-plugin.
    */
   afterEvaluate {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -26,6 +26,7 @@ import groovy.transform.CompileStatic
 import org.apache.commons.io.IOUtils
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
+import org.elasticsearch.gradle.info.JavaHome
 import org.elasticsearch.gradle.precommit.DependencyLicensesTask
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.elasticsearch.gradle.test.ErrorReportingTestListener
@@ -233,41 +234,55 @@ class BuildPlugin implements Plugin<Project> {
     static void requireJavaHome(Task task, int version) {
         // use root project for global accounting
         Project rootProject = task.project.rootProject
-        ExtraPropertiesExtension ext = rootProject.extensions.getByType(ExtraPropertiesExtension)
+        ExtraPropertiesExtension extraProperties = rootProject.extensions.extraProperties
 
-        if (rootProject.hasProperty('requiredJavaVersions') == false) {
-            ext.set('requiredJavaVersions', [:])
-            rootProject.gradle.taskGraph.whenReady({ TaskExecutionGraph taskGraph ->
-                List<String> messages = []
-                Map<Integer, List<Task>> requiredJavaVersions = (Map<Integer, List<Task>>) ext.get('requiredJavaVersions')
-                for (Map.Entry<Integer, List<Task>> entry : requiredJavaVersions) {
-                    if (BuildParams.javaVersions.find { it.version == entry.key } != null) {
-                        continue
-                    }
-                    List<String> tasks = entry.value.findAll { taskGraph.hasTask(it) }.collect { "  ${it.path}".toString() }
-                    if (tasks.isEmpty() == false) {
-                        messages.add("JAVA${entry.key}_HOME required to run tasks:\n${tasks.join('\n')}".toString())
-                    }
-                }
-                if (messages.isEmpty() == false) {
-                    throw new GradleException(messages.join('\n'))
-                }
-                ext.set('requiredJavaVersions', null) // reset to null to indicate the pre-execution checks have executed
-            })
-        } else if (ext.has('requiredJavaVersions') == false || ext.get('requiredJavaVersions') == null) {
+        // hacky way (but the only way) to find if the task graph has already been populated
+        boolean taskGraphReady
+        try {
+            rootProject.gradle.taskGraph.getAllTasks()
+            taskGraphReady = true
+        } catch (IllegalStateException) {
+            taskGraphReady = false
+        }
+
+        if (taskGraphReady) {
             // check directly if the version is present since we are already executing
             if (BuildParams.javaVersions.find { it.version == version } == null) {
                 throw new GradleException("JAVA${version}_HOME required to run task:\n${task}")
             }
         } else {
-            (ext.get('requiredJavaVersions') as Map<Integer, List<Task>>).getOrDefault(version, []).add(task)
+            // setup list of java versions we will check at the end of configuration time
+            if (extraProperties.has('requiredJavaVersions') == false) {
+                extraProperties.set('requiredJavaVersions', [:])
+                rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
+                    List<String> messages = []
+                    Map<Integer, List<Task>> requiredJavaVersions = (Map<Integer, List<Task>>) extraProperties.get('requiredJavaVersions')
+                    task.logger.warn(requiredJavaVersions.toString())
+                    for (Map.Entry<Integer, List<Task>> entry : requiredJavaVersions) {
+                        if (BuildParams.javaVersions.any { it.version == entry.key }) {
+                            continue
+                        }
+                        List<String> tasks = entry.value.findAll { taskGraph.hasTask(it) }.collect { "  ${it.path}".toString() }
+                        if (tasks.isEmpty() == false) {
+                            messages.add("JAVA${entry.key}_HOME required to run tasks:\n${tasks.join('\n')}".toString())
+                        }
+                    }
+                    if (messages.isEmpty() == false) {
+                        throw new GradleException(messages.join('\n'))
+                    }
+                }
+            }
+            Map<Integer, List<Task>> requiredJavaVersions = (Map<Integer, List<Task>>) extraProperties.get('requiredJavaVersions')
+            requiredJavaVersions.putIfAbsent(version, [])
+            requiredJavaVersions.get(version).add(task)
         }
     }
 
     /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
     static String getJavaHome(final Task task, final int version) {
         requireJavaHome(task, version)
-        return BuildParams.javaVersions.find { it.version == version }.javaHome.absolutePath
+        JavaHome java = BuildParams.javaVersions.find { it.version == version }
+        return java == null ? null : java.javaHome.absolutePath
     }
 
     /**
@@ -651,7 +666,6 @@ class BuildPlugin implements Plugin<Project> {
                 test.jvmArgumentProviders.add(nonInputProperties)
                 test.extensions.add('nonInputProperties', nonInputProperties)
 
-                test.executable = "${BuildParams.runtimeJavaHome}/bin/java"
                 test.workingDir = project.file("${project.buildDir}/testrun/${test.name}")
                 test.maxParallelForks = System.getProperty('tests.jvms', BuildParams.defaultParallel.toString()) as Integer
 

buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java

@@ -37,7 +37,7 @@ public class Jdk implements Buildable, Iterable<File> {
     private static final List<String> ALLOWED_VENDORS = List.of("adoptopenjdk", "openjdk");
     static final Pattern VERSION_PATTERN =
         Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
-    private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("darwin", "linux", "windows"));
+    private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("darwin", "linux", "windows", "mac"));
 
     private final String name;
     private final Configuration configuration;
@@ -111,6 +111,17 @@ public TaskDependency getBuildDependencies() {
         return configuration.getBuildDependencies();
     }
 
+    public Object getBinJavaPath() {
+        return new Object() {
+            @Override
+            public String toString() {
+                final String platform = getPlatform();
+                final boolean isOSX = "mac".equals(platform) || "darwin".equals(platform);
+                return getPath() + (isOSX ? "/Contents/Home" : "") + "/bin/java";
+            }
+        };
+    }
+
     // internal, make this jdks configuration unmodifiable
     void finalizeValues() {
         if (version.isPresent() == false) {

buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

@@ -173,7 +173,8 @@ private static void setupRootJdkDownload(Project rootProject, String platform, S
             jdkConfig = configurations.create(remoteConfigName);
             configurations.create(localConfigName);
         }
-        String platformDep = platform.equals("darwin") ? (vendor.equals("adoptopenjdk") ? "mac" : "osx") : platform;
+        String platformDep = platform.equals("darwin") || platform.equals("osx") ?
+            (vendor.equals("adoptopenjdk") ? "mac" : "osx") : platform;
         String extension = platform.equals("windows") ? "zip" : "tar.gz";
         String jdkDep = vendor + ":" + platformDep + ":" + jdkVersion + "@" + extension;
         rootProject.getDependencies().add(configName(vendor, version, platform), jdkDep);

buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java

@@ -41,14 +41,12 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask {
     private final RegularFileProperty outputFile;
     private final RegularFileProperty compilerVersionFile;
     private final RegularFileProperty runtimeVersionFile;
-    private final RegularFileProperty fipsJvmFile;
 
     @Inject
     public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) {
         this.outputFile = objectFactory.fileProperty();
         this.compilerVersionFile = objectFactory.fileProperty();
         this.runtimeVersionFile = objectFactory.fileProperty();
-        this.fipsJvmFile = objectFactory.fileProperty();
     }
 
     @Input
@@ -113,11 +111,6 @@ public RegularFileProperty getRuntimeVersionFile() {
         return runtimeVersionFile;
     }
 
-    @OutputFile
-    public RegularFileProperty getFipsJvmFile() {
-        return fipsJvmFile;
-    }
-
     @TaskAction
     public void generate() {
         String javaVendorVersion = System.getProperty("java.vendor.version", System.getProperty("java.vendor"));
@@ -130,7 +123,6 @@ public void generate() {
         String runtimeJavaVersionDetails = gradleJavaVersionDetails;
         JavaVersion runtimeJavaVersionEnum = JavaVersion.current();
         File gradleJavaHome = Jvm.current().getJavaHome();
-        boolean inFipsJvm = false;
 
         try {
             if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) {
@@ -146,8 +138,6 @@ public void generate() {
                 if (runtimeJavaHome.exists()) {
                     runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome);
                     runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome));
-
-                    inFipsJvm = Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
                 } else {
                     throw new RuntimeException("Runtime Java home path of '" + compilerJavaHome + "' does not exist");
                 }
@@ -213,7 +203,6 @@ public void generate() {
 
         writeToFile(compilerVersionFile.getAsFile().get(), compilerJavaVersionEnum.name());
         writeToFile(runtimeVersionFile.getAsFile().get(), runtimeJavaVersionEnum.name());
-        writeToFile(fipsJvmFile.getAsFile().get(), Boolean.toString(inFipsJvm));
     }
 
     private void writeToFile(File file, String content) {

buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -76,14 +76,12 @@ public void apply(Project project) {
                 task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info"));
                 task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version"));
                 task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version"));
-                task.getFipsJvmFile().set(new File(project.getBuildDir(), "in-fips-jvm"));
             });
 
         PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> {
             task.getBuildInfoFile().set(generateTask.getOutputFile());
             task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile());
             task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile());
-            task.getFipsJvmFile().set(generateTask.getFipsJvmFile());
             task.setGlobalInfoListeners(extension.listeners);
         });
 
@@ -103,6 +101,7 @@ public void apply(Project project) {
             params.setIsCi(System.getenv("JENKINS_URL") != null);
             params.setIsInternal(GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null);
             params.setDefaultParallel(findDefaultParallel(project));
+            params.setInFipsJvm(isInFipsJvm());
         });
 
         project.allprojects(p -> {
@@ -153,6 +152,10 @@ private static String getJavaHomeEnvVarName(String version) {
         return "JAVA" + version + "_HOME";
     }
 
+    private static boolean isInFipsJvm() {
+        return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
+    }
+
     private static String getResourceContents(String resourcePath) {
         try (BufferedReader reader = new BufferedReader(
             new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath))