[8.19] [Gradle/BWC] Patch bundled OpenJdk17 with Adoptium Jdk17 for older ES Distros (#135300)

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -44,10 +44,14 @@
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.internal.artifacts.dependencies.ProjectDependencyInternal;
+import org.gradle.api.plugins.JvmToolchainsPlugin;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.ClasspathNormalizer;
 import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.util.PatternFilterable;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaToolchainService;
+import org.gradle.jvm.toolchain.JvmVendorSpec;
 
 import java.util.Collection;
 import java.util.Iterator;
@@ -59,6 +63,7 @@
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
+import static org.elasticsearch.gradle.util.OsUtils.jdkIsIncompatibleWithOS;
 
 /**
  * Base plugin used for wiring up build tasks to REST testing tasks using new JUnit rule-based test clusters framework.
@@ -93,6 +98,7 @@ public RestTestBasePlugin(ProviderFactory providerFactory) {
     public void apply(Project project) {
         project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
         project.getPluginManager().apply(InternalDistributionDownloadPlugin.class);
+        project.getPluginManager().apply(JvmToolchainsPlugin.class);
         var bwcVersions = loadBuildParams(project).get().getBwcVersions();
 
         // Register integ-test and default distributions
@@ -226,6 +232,17 @@ public Void call(Object... args) {
                     String versionString = version.toString();
                     ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
 
+                    if (jdkIsIncompatibleWithOS(Version.fromString(versionString))) {
+                        var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
+                        var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
+                            spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
+                            spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
+                        });
+                        task.environment(
+                            "ES_FALLBACK_JAVA_HOME",
+                            fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
+                        );
+                    }
                     task.dependsOn(bwcDistro);
                     registerDistributionInputs(task, bwcDistro);
 

build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy

@@ -9,14 +9,14 @@
 
 package org.elasticsearch.gradle
 
-import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
-import org.gradle.testkit.runner.GradleRunner
 import spock.lang.IgnoreIf
 import spock.lang.Unroll
+import spock.util.environment.RestoreSystemProperties
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.GradleRunner
 
-import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withChangedClasspathMockedDistributionDownload
-import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withChangedConfigMockedDistributionDownload
-import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withMockedDistributionDownload
+import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.*
 
 /**
  * We do not have coverage for the test cluster startup on windows yet.
@@ -108,8 +108,8 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
         def runningClosure = { GradleRunner r -> r.build() }
         withMockedDistributionDownload(runner, runningClosure)
         def result = inputProperty == "distributionClasspath" ?
-                withChangedClasspathMockedDistributionDownload(runner, runningClosure) :
-                withChangedConfigMockedDistributionDownload(runner, runningClosure)
+            withChangedClasspathMockedDistributionDownload(runner, runningClosure) :
+            withChangedConfigMockedDistributionDownload(runner, runningClosure)
 
         then:
         result.output.contains("Task ':myTask' is not up-to-date because:\n  Input property 'clusters.myCluster\$0.nodes.\$0.$inputProperty'")
@@ -166,18 +166,24 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
         }
 
         then:
-        result.output.contains("Task ':myTask' is not up-to-date because:\n" +
-                "  Input property 'clusters.myCluster\$0.$propertyName'")
+        result.output.contains(
+            "Task ':myTask' is not up-to-date because:\n" +
+                "  Input property 'clusters.myCluster\$0.$propertyName'"
+        )
         result.output.contains("elasticsearch-keystore script executed!")
         assertEsOutputContains("myCluster", "Starting Elasticsearch process")
         assertEsOutputContains("myCluster", "Stopping node")
 
         where:
         pluginType | propertyName         | fileChange
-        'module'   | "installedFiles"     | { def testClazz -> testClazz.file("test-module/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
-        'plugin'   | "installedFiles"     | { def testClazz -> testClazz.file("test-plugin/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
-        'module'   | "installedClasspath" | { def testClazz -> testClazz.file("test-module/src/main/java/SomeClass.java") << "class SomeClass {}" }
-        'plugin'   | "installedClasspath" | { def testClazz -> testClazz.file("test-plugin/src/main/java/SomeClass.java") << "class SomeClass {}" }
+        'module'   | "installedFiles"     |
+            { def testClazz -> testClazz.file("test-module/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
+        'plugin'   | "installedFiles"     |
+            { def testClazz -> testClazz.file("test-plugin/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
+        'module'   | "installedClasspath" |
+            { def testClazz -> testClazz.file("test-module/src/main/java/SomeClass.java") << "class SomeClass {}" }
+        'plugin'   | "installedClasspath" |
+            { def testClazz -> testClazz.file("test-plugin/src/main/java/SomeClass.java") << "class SomeClass {}" }
     }
 
     def "can declare test cluster in lazy evaluated task configuration block"() {
@@ -232,9 +238,51 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
         assertCustomDistro('myCluster')
     }
 
+    @RestoreSystemProperties
+    def "override jdk usage via ES_JAVA_HOME for known jdk os incompatibilities"() {
+        given:
+
+        settingsFile.text = """
+            plugins {
+                id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0'
+            }
+            """ + settingsFile.text
+
+        buildFile << """
+            testClusters {
+              myCluster {
+                testDistribution = 'default'
+                version = '8.10.4'
+              }
+            }
+
+            // Force linux platform to trigger jdk override
+            elasticsearch_distributions.forEach { d ->
+                d.platform = org.elasticsearch.gradle.ElasticsearchDistribution.Platform.LINUX
+            }
+
+            tasks.register('myTask', SomeClusterAwareTask) {
+                useCluster testClusters.myCluster
+            }
+        """
+        when:
+        def result = withMockedDistributionDownload(
+            "8.10.4",
+            ElasticsearchDistribution.Platform.LINUX,
+            gradleRunner("myTask", '-Dos.name=Linux', '-Dos.version=6.14.0-1015-gcp', '-i')
+        ) {
+            build()
+        }
+
+        then:
+        result.output.lines().anyMatch { line -> line.startsWith("Running") && line.split().find { it.startsWith("ES_JAVA_HOME=") }.contains("eclipse_adoptium-17") }
+    }
+
     boolean assertEsOutputContains(String testCluster, String expectedOutput) {
-        assert new File(testProjectDir.root,
-                "build/testclusters/${testCluster}-0/logs/es.out").text.contains(expectedOutput)
+        assert new File(
+            testProjectDir.root,
+            "build/testclusters/${testCluster}-0/logs/es.out"
+        ).text.contains(expectedOutput)
         true
     }
 

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -37,6 +37,7 @@
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.bundling.AbstractArchiveTask;
 import org.gradle.api.tasks.bundling.Zip;
+import org.gradle.jvm.toolchain.JavaLauncher;
 import org.gradle.process.ExecOperations;
 
 import java.io.File;
@@ -84,6 +85,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
     private int nodeIndex = 0;
 
     private final ConfigurableFileCollection pluginAndModuleConfiguration;
+    private final Provider<JavaLauncher> jdk17FallbackLauncher;
 
     private boolean shared = false;
 
@@ -101,7 +103,8 @@ public ElasticsearchCluster(
         FileOperations fileOperations,
         File workingDirBase,
         Provider<File> runtimeJava,
-        Function<Version, Boolean> isReleasedVersion
+        Function<Version, Boolean> isReleasedVersion,
+        Provider<JavaLauncher> jdk17FallbackLauncher
     ) {
         this.path = path;
         this.clusterName = clusterName;
@@ -117,6 +120,7 @@ public ElasticsearchCluster(
         this.isReleasedVersion = isReleasedVersion;
         this.nodes = project.container(ElasticsearchNode.class);
         this.pluginAndModuleConfiguration = project.getObjects().fileCollection();
+        this.jdk17FallbackLauncher = jdk17FallbackLauncher;
         this.nodes.add(
             new ElasticsearchNode(
                 safeName(clusterName),
@@ -131,7 +135,8 @@ public ElasticsearchCluster(
                 fileOperations,
                 workingDirBase,
                 runtimeJava,
-                isReleasedVersion
+                isReleasedVersion,
+                jdk17FallbackLauncher
             )
         );
 
@@ -189,7 +194,8 @@ public void setNumberOfNodes(int numberOfNodes) {
                     fileOperations,
                     workingDirBase,
                     runtimeJava,
-                    isReleasedVersion
+                    isReleasedVersion,
+                    jdk17FallbackLauncher
                 )
             );
         }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -49,6 +49,7 @@
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.bundling.Zip;
 import org.gradle.api.tasks.util.PatternFilterable;
+import org.gradle.jvm.toolchain.JavaLauncher;
 import org.gradle.process.ExecOperations;
 
 import java.io.ByteArrayInputStream;
@@ -94,6 +95,7 @@
 
 import static java.util.Objects.requireNonNull;
 import static java.util.Optional.ofNullable;
+import static org.elasticsearch.gradle.util.OsUtils.jdkIsIncompatibleWithOS;
 
 public class ElasticsearchNode implements TestClusterConfiguration {
 
@@ -166,6 +168,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final Path tmpDir;
     private final Provider<File> runtimeJava;
     private final Function<Version, Boolean> isReleasedVersion;
+    private final Provider<JavaLauncher> jdk17FallbackLauncher;
     private final List<ElasticsearchDistribution> distributions = new ArrayList<>();
     private int currentDistro = 0;
     private TestDistribution testDistribution;
@@ -190,7 +193,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         FileOperations fileOperations,
         File workingDirBase,
         Provider<File> runtimeJava,
-        Function<Version, Boolean> isReleasedVersion
+        Function<Version, Boolean> isReleasedVersion,
+        Provider<JavaLauncher> jdk17FallbackLauncher
     ) {
         this.path = path;
         this.name = name;
@@ -203,6 +207,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         this.fileOperations = fileOperations;
         this.runtimeJava = runtimeJava;
         this.isReleasedVersion = isReleasedVersion;
+        this.jdk17FallbackLauncher = jdk17FallbackLauncher;
         workingDir = workingDirBase.toPath().resolve(safeName(name)).toAbsolutePath();
         confPathRepo = workingDir.resolve("repo");
         configFile = workingDir.resolve("config/elasticsearch.yml");
@@ -793,7 +798,19 @@ private Map<String, String> getESEnvironment() {
         if (getTestDistribution() == TestDistribution.INTEG_TEST || getVersion().equals(VersionProperties.getElasticsearchVersion())) {
             defaultEnv.put("ES_JAVA_HOME", runtimeJava.get().getAbsolutePath());
         }
+        // Older distributions ship with openjdk versions that are not compatible with newer kernels of ubuntu 24.04 and later
+        // Therefore we pass explicitly the runtime java to use the adoptium jdk that is maintained longer and compatible
+        // with newer kernels.
+        // 8.10.4 is the last version shipped with jdk < 21. We configure these cluster to run with jdk 17 adoptium as 17 was
+        // the last LTS release before 21
+        else if (jdkIsIncompatibleWithOS(getVersion())) {
+            defaultEnv.put(
+                "ES_JAVA_HOME",
+                jdk17FallbackLauncher.map(j -> j.getMetadata().getInstallationPath().getAsFile().getAbsolutePath()).get()
+            );
+        }
         defaultEnv.put("ES_PATH_CONF", configFile.getParent().toString());
+
         String systemPropertiesString = "";
         if (systemProperties.isEmpty() == false) {
             systemPropertiesString = " " + systemProperties.entrySet().stream().peek(entry -> {

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -26,13 +26,18 @@
 import org.gradle.api.invocation.Gradle;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
+import org.gradle.api.plugins.JvmToolchainsPlugin;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.services.BuildService;
 import org.gradle.api.services.BuildServiceParameters;
 import org.gradle.build.event.BuildEventsListenerRegistry;
 import org.gradle.internal.jvm.Jvm;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaLauncher;
+import org.gradle.jvm.toolchain.JavaToolchainService;
+import org.gradle.jvm.toolchain.JvmVendorSpec;
 import org.gradle.process.ExecOperations;
 import org.gradle.tooling.events.FinishEvent;
 import org.gradle.tooling.events.OperationCompletionListener;
@@ -99,11 +104,19 @@ public void setIsReleasedVersion(Function<Version, Boolean> isReleasedVersion) {
     @Override
     public void apply(Project project) {
         project.getPlugins().apply(DistributionDownloadPlugin.class);
+        project.getPlugins().apply(JvmToolchainsPlugin.class);
         project.getRootProject().getPluginManager().apply(ReaperPlugin.class);
         Provider<ReaperService> reaperServiceProvider = GradleUtils.getBuildService(
             project.getGradle().getSharedServices(),
             ReaperPlugin.REAPER_SERVICE_NAME
         );
+
+        JavaToolchainService toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
+        Provider<JavaLauncher> fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
+            spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
+            spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
+        });
+
         runtimeJavaProvider = providerFactory.provider(
             () -> System.getenv("RUNTIME_JAVA_HOME") == null ? Jvm.current().getJavaHome() : new File(System.getenv("RUNTIME_JAVA_HOME"))
         );
@@ -117,7 +130,8 @@ public void apply(Project project) {
         NamedDomainObjectContainer<ElasticsearchCluster> container = createTestClustersContainerExtension(
             project,
             testClustersRegistryProvider,
-            reaperServiceProvider
+            reaperServiceProvider,
+            fallbackJdk17Launcher
         );
 
         // provide a task to be able to list defined clusters.
@@ -154,7 +168,8 @@ private void configureArtifactTransforms(Project project) {
     private NamedDomainObjectContainer<ElasticsearchCluster> createTestClustersContainerExtension(
         Project project,
         Provider<TestClustersRegistry> testClustersRegistryProvider,
-        Provider<ReaperService> reaper
+        Provider<ReaperService> reaper,
+        Provider<JavaLauncher> fallbackJdk17Launcher
     ) {
         // Create an extensions that allows describing clusters
         NamedDomainObjectContainer<ElasticsearchCluster> container = project.container(
@@ -171,7 +186,8 @@ private NamedDomainObjectContainer<ElasticsearchCluster> createTestClustersConta
                 getFileOperations(),
                 new File(project.getBuildDir(), "testclusters"),
                 runtimeJavaProvider,
-                isReleasedVersion
+                isReleasedVersion,
+                fallbackJdk17Launcher
             )
         );
         project.getExtensions().add(EXTENSION_NAME, container);