Update Gradle to 8.5 (#377)

* Update Gradle to 8.5

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Update OpenSearch version to 2.12.0

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Fix Scala source set convention deprecations

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Fix dist/build.gradle archivesName usage

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Use build-tools 2.11.1

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Remove org.gradle.api.plugins.Convention usage

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

---------

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

CHANGELOG.md

@@ -25,6 +25,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Bumps `org.json4s:json4s-jackson_2.12` from 4.0.6 to 4.0.7
 
 ### Changed
+- Update Gradle to 8.5 ([#377](https://github.com/opensearch-project/opensearch-hadoop/pull/377))
 
 ### Deprecated
 

buildSrc/opensearch-hadoop-version.properties

@@ -1,4 +1,4 @@
 opensearch_hadoop        = 1.0.1
-opensearch      = 2.7.0
+opensearch      = 2.12.0
 lucene          = 9.5.0-snapshot-a4ef70f
-build-tools     = 2.7.0
+build-tools     = 2.11.1

buildSrc/src/main/groovy/org/opensearch/hadoop/gradle/BuildPlugin.groovy

@@ -335,8 +335,8 @@ class BuildPlugin implements Plugin<Project>  {
      */
     private static void configureBuildTasks(Project project) {
         // Target Java 1.8 compilation
-        project.sourceCompatibility = '1.8'
-        project.targetCompatibility = '1.8'
+        project.java.sourceCompatibility = '1.8'
+        project.java.targetCompatibility = '1.8'
 
         // TODO: Remove all root project distribution logic. It should exist in a separate dist project.
         if (project != project.rootProject) {
@@ -679,7 +679,7 @@ class BuildPlugin implements Plugin<Project>  {
         // Set the pom's destination to the distribution directory
         project.tasks.withType(GenerateMavenPom).all { GenerateMavenPom pom ->
             if (pom.name == "generatePomFileFor${publication.name.capitalize()}Publication") {
-                pom.destination = project.provider({"${project.buildDir}/poms/${project.archivesBaseName}-${project.getVersion()}.pom"})
+                pom.destination = project.provider({"${project.buildDir}/poms/${project.base.archivesName}-${project.getVersion()}.pom"})
             }
         }
 
@@ -738,7 +738,7 @@ class BuildPlugin implements Plugin<Project>  {
     private static void updateVariantPomLocationAndArtifactId(Project project, MavenPublication publication, SparkVariant variant) {
         // Add variant classifier to the pom file name if required
         String classifier = variant.shouldClassifySparkVersion() && variant.isDefaultVariant() == false ? "-${variant.getName()}" : ''
-        String filename = "${project.archivesBaseName}_${variant.scalaMajorVersion}-${project.getVersion()}${classifier}"
+        String filename = "${project.base.archivesName}_${variant.scalaMajorVersion}-${project.getVersion()}${classifier}"
         // Fix the pom name
         project.tasks.withType(GenerateMavenPom).all { GenerateMavenPom pom ->
             if (pom.name == "generatePomFileFor${publication.name.capitalize()}Publication") {
@@ -749,7 +749,7 @@ class BuildPlugin implements Plugin<Project>  {
         publication.getPom().withXml { XmlProvider xml ->
             Node root = xml.asNode()
             Node artifactId = (root.get('artifactId') as NodeList).get(0) as Node
-            artifactId.setValue("${project.archivesBaseName}_${variant.scalaMajorVersion}")
+            artifactId.setValue("${project.base.archivesName}_${variant.scalaMajorVersion}")
         }
     }
 

buildSrc/src/main/groovy/org/opensearch/hadoop/gradle/fixture/hadoop/conf/HadoopClusterConfiguration.groovy

@@ -36,7 +36,7 @@ import org.opensearch.hadoop.gradle.fixture.hadoop.services.SparkYarnServiceDesc
 import org.gradle.api.GradleException
 import org.gradle.api.Project
 import org.gradle.api.Task
-import org.gradle.util.ConfigureUtil
+import org.gradle.util.internal.ConfigureUtil
 
 /**
  * Configuration for a Hadoop cluster, used for integration tests.

buildSrc/src/main/groovy/org/opensearch/hadoop/gradle/fixture/hadoop/conf/ProcessConfiguration.groovy

@@ -33,7 +33,7 @@ import org.opensearch.gradle.testclusters.OpenSearchCluster
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 import org.gradle.api.Task
-import org.gradle.util.ConfigureUtil
+import org.gradle.util.internal.ConfigureUtil
 
 /**
  * All the configurations that can be set hierarchically for a cluster.

buildSrc/src/main/groovy/org/opensearch/hadoop/gradle/fixture/hadoop/conf/RoleConfiguration.groovy

@@ -32,7 +32,7 @@ package org.opensearch.hadoop.gradle.fixture.hadoop.conf
 import org.opensearch.hadoop.gradle.fixture.hadoop.RoleDescriptor
 import org.gradle.api.GradleException
 import org.gradle.api.Project
-import org.gradle.util.ConfigureUtil
+import org.gradle.util.internal.ConfigureUtil
 
 /**
  * Shared configurations for all instances of a role within a Hadoop service.

buildSrc/src/main/groovy/org/opensearch/hadoop/gradle/fixture/hadoop/conf/ServiceConfiguration.groovy

@@ -34,7 +34,7 @@ import org.opensearch.hadoop.gradle.fixture.hadoop.RoleDescriptor
 import org.opensearch.hadoop.gradle.fixture.hadoop.ServiceDescriptor
 import org.gradle.api.GradleException
 import org.gradle.api.Project
-import org.gradle.util.ConfigureUtil
+import org.gradle.util.internal.ConfigureUtil
 
 /**
  * Handles configurations for a sub-cluster of services within the larger Hadoop cluster.

buildSrc/src/main/java/org/opensearch/hadoop/gradle/buildtools/Util.java

@@ -35,7 +35,7 @@
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
 import org.gradle.api.file.FileTree;
-import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.util.PatternFilterable;
 
@@ -144,7 +144,7 @@ public static FileTree getJavaTestAndMainSourceResources(Project project, Action
      * @return An Optional that contains the Java test SourceSet if it exists.
      */
     public static Optional<SourceSet> getJavaTestSourceSet(Project project) {
-        return project.getConvention().findPlugin(JavaPluginConvention.class) == null
+        return project.getExtensions().findByType(JavaPluginExtension.class) == null
             ? Optional.empty()
             : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(SourceSet.TEST_SOURCE_SET_NAME));
     }
@@ -154,7 +154,7 @@ public static Optional<SourceSet> getJavaTestSourceSet(Project project) {
      * @return An Optional that contains the Java main SourceSet if it exists.
      */
     public static Optional<SourceSet> getJavaMainSourceSet(Project project) {
-        return project.getConvention().findPlugin(JavaPluginConvention.class) == null
+        return project.getExtensions().findByType(JavaPluginExtension.class) == null
             ? Optional.empty()
             : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME));
     }

buildSrc/src/main/java/org/opensearch/hadoop/gradle/buildtools/info/GlobalBuildInfoPlugin.java

@@ -39,11 +39,12 @@
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.internal.jvm.Jvm;
+import org.gradle.internal.jvm.inspection.JavaInstallationRegistry;
 import org.gradle.internal.jvm.inspection.JvmInstallationMetadata;
 import org.gradle.internal.jvm.inspection.JvmMetadataDetector;
+import org.gradle.internal.jvm.inspection.JvmToolchainMetadata;
 import org.gradle.internal.jvm.inspection.JvmVendor;
 import org.gradle.jvm.toolchain.internal.InstallationLocation;
-import org.gradle.jvm.toolchain.internal.JavaInstallationRegistry;
 import org.gradle.util.GradleVersion;
 
 import javax.inject.Inject;
@@ -203,7 +204,7 @@ private List<JavaHome> getAvailableJavaVersions() {
 
     private Stream<InstallationLocation> getAvailableJavaInstallationLocationSteam() {
         return Stream.concat(
-            javaInstallationRegistry.listInstallations().stream(),
+            javaInstallationRegistry.toolchains().stream().map(t -> t.location),
             Stream.of(new InstallationLocation(Jvm.current().getJavaHome(), "Current JVM"))
         );
     }

buildSrc/src/main/java/org/opensearch/hadoop/gradle/scala/SparkVariantPlugin.java

@@ -34,7 +34,6 @@
 import java.util.List;
 
 import groovy.lang.Closure;
-import org.codehaus.groovy.runtime.InvokerHelper;
 import org.codehaus.groovy.runtime.StringGroovyMethods;
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
@@ -49,12 +48,10 @@
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.SourceDirectorySet;
 import org.gradle.api.internal.artifacts.repositories.resolver.ComponentMetadataDetailsAdapter;
-import org.gradle.api.internal.tasks.DefaultScalaSourceSet;
+import org.gradle.api.internal.tasks.DefaultScalaSourceDirectorySet;
 import org.gradle.api.plugins.BasePlugin;
-import org.gradle.api.plugins.Convention;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.plugins.JavaPlugin;
-import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
@@ -284,18 +281,16 @@ public void execute(ComponentMetadataContext componentMetadataContext) {
     @Override
     public void apply(final Project project) {
         SparkVariantPluginExtension extension = project.getExtensions().create("sparkVariants", SparkVariantPluginExtension.class, project);
-        final JavaPluginConvention javaPluginConvention = project.getConvention().getPlugin(JavaPluginConvention.class);
         final JavaPluginExtension javaPluginExtension = project.getExtensions().getByType(JavaPluginExtension.class);
 
         // Add a rule that annotates scala-library dependencies with the scala-library capability
         project.getDependencies().getComponents().all(ScalaRuntimeCapability.class);
 
-        extension.defaultVariant(sparkVariant -> configureDefaultVariant(project, sparkVariant, javaPluginExtension, javaPluginConvention));
-        extension.featureVariants(sparkVariant -> configureVariant(project, sparkVariant, javaPluginExtension, javaPluginConvention));
+        extension.defaultVariant(sparkVariant -> configureDefaultVariant(project, sparkVariant, javaPluginExtension));
+        extension.featureVariants(sparkVariant -> configureVariant(project, sparkVariant, javaPluginExtension));
     }
 
-    private static void configureDefaultVariant(Project project, SparkVariant sparkVariant, JavaPluginExtension javaPluginExtension,
-                                                JavaPluginConvention javaPluginConvention) {
+    private static void configureDefaultVariant(Project project, SparkVariant sparkVariant, JavaPluginExtension javaPluginExtension) {
         ConfigurationContainer configurations = project.getConfigurations();
         String capability = sparkVariant.getCapabilityName(project.getVersion());
 
@@ -308,9 +303,8 @@ private static void configureDefaultVariant(Project project, SparkVariant sparkV
         configureScalaJarClassifiers(project, sparkVariant);
     }
 
-    private static void configureVariant(Project project, SparkVariant sparkVariant, JavaPluginExtension javaPluginExtension,
-                                         JavaPluginConvention javaPluginConvention) {
-        SourceSetContainer sourceSets = javaPluginConvention.getSourceSets();
+    private static void configureVariant(Project project, SparkVariant sparkVariant, JavaPluginExtension javaPluginExtension) {
+        SourceSetContainer sourceSets = javaPluginExtension.getSourceSets();
         ConfigurationContainer configurations = project.getConfigurations();
         TaskContainer tasks = project.getTasks();
         Object version = project.getVersion();
@@ -333,9 +327,8 @@ private static void configureVariant(Project project, SparkVariant sparkVariant,
     }
 
     public static SourceSet configureAdditionalVariantSourceSet(Project project, SparkVariant sparkVariant, String sourceSetName) {
-        final JavaPluginConvention javaPluginConvention = project.getConvention().getPlugin(JavaPluginConvention.class);
         final JavaPluginExtension javaPluginExtension = project.getExtensions().getByType(JavaPluginExtension.class);
-        SourceSetContainer sourceSets = javaPluginConvention.getSourceSets();
+        SourceSetContainer sourceSets = javaPluginExtension.getSourceSets();
         ConfigurationContainer configurations = project.getConfigurations();
         String version = project.getVersion().toString();
 
@@ -373,7 +366,7 @@ private static SourceSet createVariantSourceSet(SparkVariant sparkVariant, Sourc
         SourceDirectorySet resourcesSourceSet = sourceSet.getResources();
         resourcesSourceSet.setSrcDirs(Collections.singletonList("src/" + sourceSetName + "/resources"));
 
-        SourceDirectorySet scalaSourceSet = getScalaSourceSet(sourceSet).getScala();
+        SourceDirectorySet scalaSourceSet = getScalaSourceSet(sourceSet);
         scalaSourceSet.setSrcDirs(Arrays.asList(
                 "src/" + sourceSetName + "/scala",
                 "src/" + sourceSetName + "/" + sparkVariant.getName()
@@ -393,9 +386,8 @@ private static void configureAdditionalSourceSetClasspaths(Project project, Conf
         additionalSourceSet.setRuntimeClasspath(project.files(additionalSourceSet.getOutput(), mainSourceSet.getOutput(), additionalRuntimeClasspath));
     }
 
-    private static DefaultScalaSourceSet getScalaSourceSet(SourceSet sourceSet) {
-        Convention sourceSetConvention = (Convention) InvokerHelper.getProperty(sourceSet, "convention");
-        return (DefaultScalaSourceSet) sourceSetConvention.getPlugins().get("scala");
+    private static DefaultScalaSourceDirectorySet getScalaSourceSet(SourceSet sourceSet) {
+        return (DefaultScalaSourceDirectorySet) sourceSet.getExtensions().getByName("scala");
     }
 
     private static void registerMainVariant(JavaPluginExtension java, SparkVariant sparkVariant, SourceSet main, Object version) {
@@ -448,7 +440,7 @@ private static void registerVariantScaladoc(Project project, TaskContainer tasks
             scaladocClasspath.from(main.getCompileClasspath());
 
             scalaDoc.setClasspath(scaladocClasspath);
-            scalaDoc.setSource(getScalaSourceSet(main).getScala());
+            scalaDoc.setSource(getScalaSourceSet(main));
         });
     }
 

dist/build.gradle

@@ -6,7 +6,10 @@ import org.opensearch.hadoop.gradle.BuildPlugin
 apply plugin: 'opensearch.hadoop.build'
 
 description = "OpenSearch for Apache Hadoop"
-project.archivesBaseName = 'opensearch-hadoop'
+
+base {
+  archivesName = 'opensearch-hadoop'
+}
 
 def sparkVariantIncluded = 'spark30scala212'
 
@@ -142,7 +145,7 @@ publishing {
                 repository.appendNode('url', 'https://clojars.org/repo')
 
                 // Correct the artifact Id, otherwise it is listed as 'dist'
-                root.get('artifactId').get(0).setValue(project.archivesBaseName)
+                root.get('artifactId').get(0).setValue(project.base.archivesName)
             }
         }
     }

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
 networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -83,10 +83,8 @@ done
 # This is normally unused
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -133,26 +131,29 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045 
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045 
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -197,11 +198,15 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
-# Collect all arguments for the java command;
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-#     shell script including quotes and variable substitutions, so put them in
-#     double quotes to make sure that they get re-expanded; and
-#   * put everything else in single quotes, so that it's not re-expanded.
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \

mr/build.gradle

@@ -25,24 +25,18 @@ configurations {
     // This will force the MR project to require jars from upstream project dependencies, which shouldn't be a problem since
     // there should only be one upstream project.
     compileClasspath {
-        beforeLocking {
-            attributes {
-                attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, LibraryElements.JAR))
-            }
+        attributes {
+            attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, LibraryElements.JAR))
         }
     }
     testCompileClasspath {
-        beforeLocking {
-            attributes {
-                attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, LibraryElements.JAR))
-            }
+        attributes {
+            attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, LibraryElements.JAR))
         }
     }
     itestCompileClasspath {
-        beforeLocking {
-            attributes {
-                attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, LibraryElements.JAR))
-            }
+        attributes {
+            attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, LibraryElements.JAR))
         }
     }
 }

qa/kerberos/build.gradle

@@ -62,10 +62,8 @@ boolean localRepo = project.getProperties().containsKey("localRepo")
 // TODO: Clean this up when we get to variants, they should persist their classes dirs in a sane way
 configurations {
     compileClasspath {
-        beforeLocking {
-            attributes {
-                attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, LibraryElements.JAR))
-            }
+        attributes {
+            attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, LibraryElements.JAR))
         }
     }
 }

test/fixtures/minikdc/build.gradle

@@ -43,8 +43,10 @@ dependencies {
 }
 
 // Target Java 1.8 compilation
-sourceCompatibility = '1.8'
-targetCompatibility = '1.8'
+java {
+    sourceCompatibility = '1.8'
+    targetCompatibility = '1.8'
+}
 
 // for testing, until fixture are actually debuggable.
 // gradle hides EVERYTHING so you have no clue what went wrong.