Bundle java in distributions

Setting up a jdk is currently a required external step when installing
elasticsearch. This is particularly problematic for the rpm/deb packages
as installing a jdk in the same package installation command does not
guarantee any order, so must be done in separate steps. Additionally,
JAVA_HOME must be set and often causes problems in selecting a correct
jdk when, for example, the system java is an older unsupported version.

This commit bundles platform specific openjdks into each distribution.
In addition to eliminating the issues above, it also presents future
possible improvements like using jlink to build jdk images only
containing modules that elasticsearch uses.

closes elastic#31845

Vagrantfile

@@ -361,6 +361,9 @@ def sh_install_deps(config,
       rm -rf /tmp/bats
     }
 
+    echo "export SYSTEM_JAVA_HOME=$JAVA_HOME" >> /etc/profile.d/java_home.sh
+    echo 'unset JAVA_HOME' >> /etc/profile.d/java_home.sh
+
     cat \<\<VARS > /etc/profile.d/elasticsearch_vars.sh
 export ZIP=/elasticsearch/distribution/zip/build/distributions
 export TAR=/elasticsearch/distribution/tar/build/distributions

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -668,7 +668,6 @@ class ClusterFormationTasks {
     static Task configureExecTask(String name, Project project, Task setup, NodeInfo node, Object[] execArgs) {
         return project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { Exec exec ->
             exec.workingDir node.cwd
-            exec.environment 'JAVA_HOME', node.getJavaHome()
             if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                 exec.executable 'cmd'
                 exec.args '/C', 'call'

buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/VersionProperties.java

@@ -2,14 +2,21 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
  */
 public class VersionProperties {
+    private static final Pattern JDK_VERSION = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)\\+(\\d+)");
+
     public static String getElasticsearch() {
         return elasticsearch;
     }
@@ -18,17 +25,41 @@ public static String getLucene() {
         return lucene;
     }
 
+    /**
+     * Return the bundled jdk version, broken into elements as:
+     * [feature, interim, update, build]
+     *
+     * Note the "patch" version is not yet handled here, as it has not yet
+     * been used by java.
+     */
+    public static List<String> getBundledJdk() {
+        return bundledJdk;
+    }
+
     public static Map<String, String> getVersions() {
         return versions;
     }
 
     private static final String elasticsearch;
     private static final String lucene;
+    private static final List<String> bundledJdk;
     private static final Map<String, String> versions = new HashMap<String, String>();
     static {
         Properties props = getVersionProperties();
         elasticsearch = props.getProperty("elasticsearch");
         lucene = props.getProperty("lucene");
+
+        String bundledJdkProperty = props.getProperty("bundled_jdk");
+        Matcher jdkVersionMatcher = JDK_VERSION.matcher(bundledJdkProperty);
+        if (jdkVersionMatcher.matches() == false) {
+            throw new IllegalArgumentException("Malformed jdk version [" + bundledJdkProperty + "]");
+        }
+        List<String> bundledJdkList = new ArrayList<>();
+        for (int i = 1; i <= jdkVersionMatcher.groupCount(); ++i) {
+            bundledJdkList.add(jdkVersionMatcher.group(i));
+        }
+        bundledJdk = Collections.unmodifiableList(bundledJdkList);
+
         for (String property : props.stringPropertyNames()) {
             versions.put(property, props.getProperty(property));
         }

buildSrc/version.properties

@@ -1,6 +1,8 @@
 elasticsearch     = 7.0.0
 lucene            = 8.0.0-snapshot-83f9835
 
+bundled_jdk       = 11.0.1+13
+
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0

distribution/archives/build.gradle

@@ -45,7 +45,7 @@ task createPluginsDir(type: EmptyDirTask) {
   dirMode 0755
 }
 
-CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean oss) {
+CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss) {
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
@@ -59,6 +59,11 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean os
       into('bin') {
         with binFiles(distributionType, oss)
       }
+      if (platform != null) {
+        into('jdk') {
+          with jdkFiles(platform)
+        }
+      }
       into('') {
         from {
           dirMode 0755
@@ -102,19 +107,19 @@ Closure commonZipConfig = {
 
 task buildIntegTestZip(type: Zip) {
   configure(commonZipConfig)
-  with archiveFiles(transportModulesFiles, 'zip', true)
+  with archiveFiles(transportModulesFiles, 'zip', null, true)
 }
 
 task buildWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(false), 'zip', false)
+  with archiveFiles(modulesFiles(false), 'zip', 'windows', false)
 }
 
 task buildOssWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(true), 'zip', true)
+  with archiveFiles(modulesFiles(true), 'zip', 'windows', true)
 }
 
 Closure commonTarConfig = {
@@ -127,25 +132,25 @@ Closure commonTarConfig = {
 task buildDarwinTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(false), 'tar', false)
+  with archiveFiles(modulesFiles(false), 'tar', 'darwin', false)
 }
 
 task buildOssDarwinTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(true), 'tar', true)
+  with archiveFiles(modulesFiles(true), 'tar', 'darwin', true)
 }
 
 task buildLinuxTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(false), 'tar', false)
+  with archiveFiles(modulesFiles(false), 'tar', 'linux', false)
 }
 
 task buildOssLinuxTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(true), 'tar', true)
+  with archiveFiles(modulesFiles(true), 'tar', 'linux', true)
 }
 
 Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }

distribution/build.gradle

@@ -20,6 +20,7 @@
 import org.elasticsearch.gradle.ConcatFilesTask
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.NoticeTask
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.test.RunTask
 import org.apache.tools.ant.filters.FixCrLfFilter
 
@@ -210,6 +211,44 @@ xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
   copyLog4jProperties(buildDefaultLog4jConfig, xpackModule)
 }
 
+/*****************************************************************************
+ *                                    JDKs                                   *
+ *****************************************************************************/
+List<String> bundledJdkVersion = VersionProperties.bundledJdk
+String jdkVersion = "${bundledJdkVersion[0..2].join('.')}"
+String jdkMajor = bundledJdkVersion[0]
+String jdkBuild = bundledJdkVersion[3]
+repositories {
+  ivy {
+    url "https://download.java.net"
+    patternLayout {
+      artifact "java/GA/jdk${jdkMajor}/${jdkBuild}/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"
+    }
+  }
+}
+for (String platform : ['linux', 'darwin', 'windows']) {
+  String jdkConfigName = "jdk_${platform}"
+  Configuration jdkConfig = configurations.create(jdkConfigName)
+  String extension = platform.equals('windows') ? 'zip' : 'tar.gz'
+  dependencies.add(jdkConfigName, "jdk:${platform.equals('darwin') ? 'osx' : platform}:${jdkVersion}@${extension}")
+
+  int rootNdx = platform.equals('darwin') ? 2 : 1
+  Closure removeRootDir = {
+    it.eachFile { FileCopyDetails details ->
+      details.relativePath = new RelativePath(true, details.relativePath.segments[rootNdx..-1] as String[])
+    }
+    it.includeEmptyDirs false
+  }
+  project.task("extract${platform.capitalize()}Jdk", type: Copy) {
+    into "${buildDir}/jdks/openjdk-${jdkVersion}_${platform}"
+    if (extension.equals('zip')) {
+      from({ zipTree(jdkConfig.singleFile) }, removeRootDir)
+    } else {
+      from({ tarTree(resources.gzip(jdkConfig.singleFile)) }, removeRootDir)
+    }
+  }
+}
+
 // make sure we have a clean task since we aren't a java project, but we have tasks that
 // put stuff in the build dir
 task clean(type: Delete) {
@@ -332,6 +371,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         from buildDefaultNotice
       }
     }
+
+    jdkFiles = { platform ->
+      copySpec {
+        from project(':distribution').tasks.getByName("extract${platform.capitalize()}Jdk")
+      }
+    }
   }
 
 }

distribution/packages/build.gradle

@@ -134,6 +134,9 @@ Closure commonPackageConfig(String type, boolean oss) {
       into('modules') {
         with modulesFiles(oss)
       }
+      into('jdk') {
+        with jdkFiles('linux')
+      }
       // we need to specify every intermediate directory in these paths so the package managers know they are explicitly
       // intended to manage them; otherwise they may be left behind on uninstallation. duplicate calls of the same
       // directory are fine

distribution/src/bin/elasticsearch-env

@@ -39,13 +39,11 @@ ES_CLASSPATH="$ES_HOME/lib/*"
 if [ -x "$JAVA_HOME/bin/java" ]; then
   JAVA="$JAVA_HOME/bin/java"
 else
-  set +e
-  JAVA=`which java`
-  set -e
+  JAVA="$ES_HOME/jdk/bin/java"
 fi
 
 if [ ! -x "$JAVA" ]; then
-  echo "could not find java; set JAVA_HOME or ensure java is in PATH" >&2
+  echo "could not find java in JAVA_HOME or bundled in $ES_HOME/jdk" >&2
   exit 1
 fi
 

distribution/src/bin/elasticsearch-env.bat

@@ -20,11 +20,11 @@ rem now set the path to java
 if defined JAVA_HOME (
   set JAVA="%JAVA_HOME%\bin\java.exe"
 ) else (
-  for %%I in (java.exe) do set JAVA="%%~$PATH:I"
+  set JAVA="%ES_HOME%\jdk\bin\java.exe"
 )
 
 if not exist %JAVA% (
-  echo could not find java; set JAVA_HOME or ensure java is in PATH 1>&2
+  echo could not find java in JAVA_HOME or bundled in %ES_HOME%\jdk
   exit /b 1
 )
 

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java

@@ -76,7 +76,7 @@ public void test20PluginsListWithNoPlugins() {
         assertThat(r.stdout, isEmptyString());
     }
 
-    public void test30AbortWhenJavaMissing() {
+    public void test30OverrideJava() {
         assumeThat(installation, is(notNullValue()));
 
         final Installation.Executables bin = installation.executables();

qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats

@@ -61,7 +61,9 @@ setup() {
 }
 
 @test "[UPGRADE] start old version" {
+    export JAVA_HOME=$SYSTEM_JAVA_HOME
     start_elasticsearch_service
+    unset JAVA_HOME
 }
 
 @test "[UPGRADE] check elasticsearch version is old version" {

qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash

@@ -162,22 +162,6 @@ fi
     remove_plugin_example
 }
 
-@test "[$GROUP] fail if java executable is not found" {
-  [ "$GROUP" == "TAR PLUGINS" ] || skip "Test case only supported by TAR PLUGINS"
-  local JAVA=$(which java)
-
-  sudo chmod -x $JAVA
-  run "$ESHOME/bin/elasticsearch-plugin"
-  sudo chmod +x $JAVA
-
-  [ "$status" -eq 1 ]
-  local expected="could not find java; set JAVA_HOME or ensure java is in PATH"
-  [[ "$output" == *"$expected"* ]] || {
-    echo "Expected error message [$expected] but found: $output"
-    false
-  }
-}
-
 # Note that all of the tests from here to the end of the file expect to be run
 # in sequence and don't take well to being run one at a time.
 @test "[$GROUP] install a sample plugin" {