Modularize Elasticsearch (#81066)

This PR represents the initial phase of Modularizing Elasticsearch (with
Java Modules).

This initial phase modularizes the core of the Elasticsearch server
with Java Modules, which is then used to load and configure extension
components atop the server. Only a subset of extension components are
modularized at this stage (other components come in a later phase).
Components are loaded dynamically at runtime with custom class loaders
(same as is currently done). Components with a module-info.class are
defined to a module layer.

This architecture is somewhat akin to the Modular JDK, where
applications run on the classpath. In the analogy, the Elasticsearch
server modules are the platform (thus are always resolved and present),
while components without a module-info.class are non-modular code
running atop the Elasticsearch server modules. The extension components
cannot access types from non-exported packages of the server modules, in
the same way that classpath applications cannot access types from
non-exported packages of modules from the JDK. Broadly, the core
Elasticseach java modules simply "wrap" the existing packages and export
them. There are opportunites to export less, which is best done in more
narrowly focused follow-up PRs.

The Elasticsearch distribution startup scripts are updated to put jars
on the module path (the class path is empty), so the distribution will
run the core of the server as java modules. A number of key components
have been retrofitted with module-info.java's too, and the remaining
components can follow later. Unit and functional tests run as
non-modular (since they commonly require package-private access), while
higher-level integration tests, that run the distribution, run as
modular.

Co-authored-by: Chris Hegarty <christopher.hegarty@elastic.co>
Co-authored-by: Ryan Ernst <ryan@iernst.net>
Co-authored-by: Rene Groeschke <rene@elastic.co>

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPluginFuncTest.groovy

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.gradle.internal
 
+import spock.lang.IgnoreIf
+
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.fixtures.AbstractJavaGradleFuncTest
 import org.gradle.internal.os.OperatingSystem
@@ -17,6 +19,7 @@ import org.objectweb.asm.tree.ClassNode
 
 import java.nio.file.Files
 
+@IgnoreIf({ os.isWindows() })
 class ElasticsearchJavaModulePathPluginFuncTest extends AbstractJavaGradleFuncTest {
 
     public static final GString JAVA_BASE_MODULE = "java.base:${System.getProperty("java.version")}"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java

@@ -159,6 +159,7 @@ public Iterable<String> asArguments() {
             }
             if (isModuleProject) {
                 extraArgs.add("--module-version=" + VersionProperties.getElasticsearch());
+                extraArgs.add("-Xlint:-module,-exports,-requires-automatic,-requires-transitive-automatic,-missing-explicit-ctor");
             }
             return List.copyOf(extraArgs);
         }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java

@@ -43,6 +43,7 @@ public class ElasticsearchJavaPlugin implements Plugin<Project> {
     public void apply(Project project) {
         project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
         project.getPluginManager().apply(JavaLibraryPlugin.class);
+        project.getPluginManager().apply(ElasticsearchJavaModulePathPlugin.class);
 
         // configureConfigurations(project);
         configureJars(project);
@@ -137,5 +138,4 @@ private static void configureJavadoc(Project project) {
         // ensure javadoc task is run with 'check'
         project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(javadoc));
     }
-
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java

@@ -55,13 +55,17 @@ public void apply(Project project) {
         // sanity checks if archives can be extracted
         TaskProvider<Copy> checkExtraction = registerCheckExtractionTask(project, buildDistTask, archiveExtractionDir);
         TaskProvider<Task> checkLicense = registerCheckLicenseTask(project, checkExtraction);
-
         TaskProvider<Task> checkNotice = registerCheckNoticeTask(project, checkExtraction);
+        TaskProvider<Task> checkModulesTask = InternalDistributionModuleCheckTaskProvider.registerCheckModulesTask(
+            project,
+            checkExtraction
+        );
         TaskProvider<Task> checkTask = project.getTasks().named("check");
         checkTask.configure(task -> {
             task.dependsOn(checkExtraction);
             task.dependsOn(checkLicense);
             task.dependsOn(checkNotice);
+            task.dependsOn(checkModulesTask);
         });
 
         String projectName = project.getName();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.elasticsearch.gradle.VersionProperties;
+import org.gradle.api.Action;
+import org.gradle.api.GradleException;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.TaskProvider;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.lang.module.ModuleFinder;
+import java.lang.module.ModuleReference;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * Distribution level checks for Elasticsearch Java modules, i.e. modular jar files.
+ * Currently, ES modular jar files are in the lib directory.
+ */
+public class InternalDistributionModuleCheckTaskProvider {
+
+    private static final Logger LOGGER = Logging.getLogger(InternalDistributionModuleCheckTaskProvider.class);
+
+    private static final String MODULE_INFO = "module-info.class";
+
+    private static final String ES_JAR_PREFIX = "elasticsearch-";
+
+    /** ES jars in the lib directory that are not modularized. For now, es-log4j is the only one. */
+    private static final List<String> ES_JAR_EXCLUDES = List.of("elasticsearch-log4j");
+
+    /** List of the current Elasticsearch Java Modules, by name. */
+    private static final List<String> EXPECTED_ES_SERVER_MODULES = List.of(
+        "org.elasticsearch.base",
+        "org.elasticsearch.cli",
+        "org.elasticsearch.geo",
+        "org.elasticsearch.lz4",
+        "org.elasticsearch.pluginclassloader",
+        "org.elasticsearch.securesm",
+        "org.elasticsearch.server",
+        "org.elasticsearch.xcontent"
+    );
+
+    private static final Predicate<ModuleReference> isESModule = mref -> mref.descriptor().name().startsWith("org.elasticsearch");
+
+    private static Predicate<Path> isESJar = path -> path.getFileName().toString().startsWith(ES_JAR_PREFIX);
+
+    private static Predicate<Path> isNotExcluded = path -> ES_JAR_EXCLUDES.stream()
+        .filter(path.getFileName().toString()::startsWith)
+        .findAny()
+        .isEmpty();
+
+    private static final Function<ModuleReference, String> toName = mref -> mref.descriptor().name();
+
+    private InternalDistributionModuleCheckTaskProvider() {};
+
+    /** Registers the checkModules tasks, which contains all checks relevant to ES Java Modules. */
+    static TaskProvider<Task> registerCheckModulesTask(Project project, TaskProvider<Copy> checkExtraction) {
+        return project.getTasks().register("checkModules", task -> {
+            task.dependsOn(checkExtraction);
+            task.doLast(new Action<Task>() {
+                @Override
+                public void execute(Task task) {
+                    final Path libPath = checkExtraction.get()
+                        .getDestinationDir()
+                        .toPath()
+                        .resolve("elasticsearch-" + VersionProperties.getElasticsearch())
+                        .resolve("lib");
+                    try {
+                        assertAllESJarsAreModular(libPath);
+                        assertAllModulesPresent(libPath);
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
+                    }
+                }
+            });
+        });
+    }
+
+    /** Checks that all expected ES jar files are modular, i.e. contain a module-info.class in their root. */
+    private static void assertAllESJarsAreModular(Path libPath) throws IOException {
+        try (var s = Files.walk(libPath, 1)) {
+            s.filter(Files::isRegularFile).filter(isESJar).filter(isNotExcluded).sorted().forEach(path -> {
+                try (JarFile jf = new JarFile(path.toFile())) {
+                    JarEntry entry = jf.getJarEntry(MODULE_INFO);
+                    if (entry == null) {
+                        throw new GradleException(MODULE_INFO + " no found in " + path);
+                    }
+                } catch (IOException e) {
+                    throw new GradleException("Failed when reading jar file " + path, e);
+                }
+            });
+        }
+    }
+
+    /** Checks that all expected Elasticsearch modules are present. */
+    private static void assertAllModulesPresent(Path libPath) {
+        List<String> actualESModules = ModuleFinder.of(libPath).findAll().stream().filter(isESModule).map(toName).sorted().toList();
+        if (actualESModules.equals(EXPECTED_ES_SERVER_MODULES) == false) {
+            throw new GradleException(
+                "expected modules " + listToString(EXPECTED_ES_SERVER_MODULES) + ", \nactual modules " + listToString(actualESModules)
+            );
+        }
+    }
+
+    // ####: eventually assert hashes, etc
+
+    static String listToString(List<String> list) {
+        return list.stream().sorted().collect(joining("\n  ", "[\n  ", "]"));
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java

@@ -44,6 +44,7 @@ public static void create(Project project, boolean withProductiveCode) {
             project.getPluginManager().apply(ThirdPartyAuditPrecommitPlugin.class);
             project.getPluginManager().apply(DependencyLicensesPrecommitPlugin.class);
             project.getPluginManager().apply(SplitPackagesAuditPrecommitPlugin.class);
+            project.getPluginManager().apply(JavaModulePrecommitPlugin.class);
         }
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitPlugin.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.precommit;
+
+import org.elasticsearch.gradle.internal.InternalPlugin;
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.TaskProvider;
+
+public class JavaModulePrecommitPlugin extends PrecommitPlugin implements InternalPlugin {
+
+    public static final String TASK_NAME = "validateModule";
+
+    @Override
+    public TaskProvider<? extends Task> createTask(Project project) {
+        TaskProvider<JavaModulePrecommitTask> task = project.getTasks().register(TASK_NAME, JavaModulePrecommitTask.class);
+        task.configure(t -> {
+            SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME);
+            t.getSrcDirs().set(project.provider(() -> mainSourceSet.getAllSource().getSrcDirs()));
+            t.setClasspath(project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME));
+            t.setClassesDirs(mainSourceSet.getOutput().getClassesDirs());
+            t.setResourcesDirs(mainSourceSet.getOutput().getResourcesDir());
+        });
+        return task;
+    }
+}