[8.6] Refactor plugin scanning into lib (#92437) (#92899)

new stable plugins require generated named_components.json file which contains all analysis components implemented by this plugin. The generation is currently done in build-tools by elasticsearch.stable-esplugin However this makes the generation only available for plugins using gradle. Plugin developers using maven or other building tooling will not be able to use it.

This commits refactors the scanning logic into libs:plugin-scanner which will allow for plugin install command to perform the scanning too.

relates #88980
backports #92437

build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy

@@ -19,15 +19,11 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.util.stream.Collectors
 
+import static org.elasticsearch.gradle.fixtures.TestClasspathUtils.setupNamedComponentScanner
+
 class StablePluginBuildPluginFuncTest extends AbstractGradleFuncTest {
 
     def setup() {
-        // underlaying TestClusterPlugin and StandaloneRestIntegTestTask are not cc compatible
-        configurationCacheCompatible = false
-    }
-
-    def "can build stable plugin properties"() {
-        given:
         buildFile << """plugins {
                 id 'elasticsearch.stable-esplugin'
             }
@@ -38,8 +34,27 @@ class StablePluginBuildPluginFuncTest extends AbstractGradleFuncTest {
                 name = 'myplugin'
                 description = 'test plugin'
             }
+            repositories {
+              maven {
+                name = "local-test"
+                url = file("local-repo")
+                metadataSources {
+                  artifact()
+                }
+              }
+            }
             """
 
+        // underlaying TestClusterPlugin and StandaloneRestIntegTestTask are not cc compatible
+        configurationCacheCompatible = false
+
+        def version = VersionProperties.elasticsearch
+        setupNamedComponentScanner(dir("local-repo/org/elasticsearch/elasticsearch-plugin-scanner/${version}/"), version)
+
+    }
+
+    def "can build stable plugin properties"() {
+        given:
         when:
         def result = gradleRunner(":pluginProperties").build()
         def props = getPluginProperties()
@@ -62,26 +77,16 @@ class StablePluginBuildPluginFuncTest extends AbstractGradleFuncTest {
     }
 
     def "can scan and create named components file"() {
+        //THIS IS RUNNING A MOCK CONFIGURED IN setup()
         given:
         File jarFolder = new File(testProjectDir.root, "jars")
         jarFolder.mkdirs()
 
-        buildFile << """plugins {
-                id 'elasticsearch.stable-esplugin'
-            }
-
-            version = '1.2.3'
-
-            esplugin {
-                name = 'myplugin'
-                description = 'test plugin'
-            }
-
+        buildFile << """
             dependencies {
                 implementation files('${normalized(StableApiJarMocks.createPluginApiJar(jarFolder.toPath()).toAbsolutePath().toString())}')
                 implementation files('${normalized(StableApiJarMocks.createExtensibleApiJar(jarFolder.toPath()).toAbsolutePath().toString())}')
             }
-
             """
 
         file("src/main/java/org/acme/A.java") << """
@@ -95,18 +100,16 @@ class StablePluginBuildPluginFuncTest extends AbstractGradleFuncTest {
             }
         """
 
-
         when:
-        def result = gradleRunner(":assemble").build()
-        Path namedComponents = file("build/generated-named-components/named_components.json").toPath();
-        def map = new JsonSlurper().parse(namedComponents.toFile())
+        def result = gradleRunner(":assemble", "-i").build()
+
         then:
         result.task(":assemble").outcome == TaskOutcome.SUCCESS
-
-        map  == ["org.elasticsearch.plugin.scanner.test_classes.ExtensibleClass" : (["componentA" : "org.acme.A"]) ]
+        //we expect that a Fake namedcomponent scanner used in this test will be passed a filename to be created
+        File namedComponents = file("build/generated-named-components/named_components.json")
+        namedComponents.exists() == true
     }
 
-
     Map<String, String> getPluginProperties() {
         Path propsFile = file("build/generated-descriptor/stable-plugin-descriptor.properties").toPath();
         Properties rawProps = new Properties()

build-tools/src/main/java/org/elasticsearch/gradle/NamedComponentScannerMock.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class NamedComponentScannerMock {
+    public static void main(String[] args) throws IOException {
+        // expect a file name to passed in as a parameter
+        // creating a file so that we can assert about this in a test
+        Path path = Path.of(args[0]);
+        Files.createDirectories(path.getParent());
+        Files.createFile(path);
+    }
+}

build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateNamedComponentsTask.java

@@ -8,55 +8,60 @@
 
 package org.elasticsearch.gradle.plugin;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import org.elasticsearch.gradle.plugin.scanner.ClassReaders;
-import org.elasticsearch.gradle.plugin.scanner.NamedComponentScanner;
+import org.elasticsearch.gradle.LoggedExec;
 import org.gradle.api.DefaultTask;
-import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.file.RegularFileProperty;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
-import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.tasks.CompileClasspath;
+import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskAction;
-import org.gradle.workers.WorkAction;
-import org.gradle.workers.WorkParameters;
+import org.gradle.process.ExecOperations;
+import org.gradle.process.ExecResult;
 import org.gradle.workers.WorkerExecutor;
-import org.objectweb.asm.ClassReader;
 
 import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
 public abstract class GenerateNamedComponentsTask extends DefaultTask {
     private static final Logger LOGGER = Logging.getLogger(GenerateNamedComponentsTask.class);
+    private static final String NAMED_COMPONENTS_DIR = "generated-named-components/";
     private static final String NAMED_COMPONENTS_FILE = "named_components.json";
+    private static final String NAMED_COMPONENTS_PATH = NAMED_COMPONENTS_DIR + NAMED_COMPONENTS_FILE;
 
     private final WorkerExecutor workerExecutor;
+    private FileCollection pluginScannerClasspath;
     private FileCollection classpath;
+    private ExecOperations execOperations;
+    private ProjectLayout projectLayout;
 
     @Inject
-    public GenerateNamedComponentsTask(WorkerExecutor workerExecutor, ObjectFactory objectFactory, ProjectLayout projectLayout) {
+    public GenerateNamedComponentsTask(WorkerExecutor workerExecutor, ExecOperations execOperations, ProjectLayout projectLayout) {
         this.workerExecutor = workerExecutor;
-        getOutputFile().convention(projectLayout.getBuildDirectory().file("generated-named-components/" + NAMED_COMPONENTS_FILE));
+        this.execOperations = execOperations;
+        this.projectLayout = projectLayout;
+
+        getOutputFile().convention(projectLayout.getBuildDirectory().file(NAMED_COMPONENTS_PATH));
     }
 
     @TaskAction
     public void scanPluginClasses() {
-        workerExecutor.noIsolation().submit(GenerateNamedComponentsAction.class, params -> {
-            params.getClasspath().from(classpath);
-            params.getOutputFile().set(getOutputFile());
+        File outputFile = projectLayout.getBuildDirectory().file(NAMED_COMPONENTS_PATH).get().getAsFile();
+
+        ExecResult execResult = LoggedExec.javaexec(execOperations, spec -> {
+            spec.classpath(pluginScannerClasspath.plus(getClasspath()).getAsPath());
+            spec.getMainClass().set("org.elasticsearch.plugin.scanner.NamedComponentScanner");
+            spec.args(outputFile);
+            spec.setErrorOutput(System.err);
+            spec.setStandardOutput(System.out);
         });
+        execResult.assertNormalExitValue();
     }
 
     @OutputFile
@@ -71,37 +76,13 @@ public void setClasspath(FileCollection classpath) {
         this.classpath = classpath;
     }
 
-    public abstract static class GenerateNamedComponentsAction implements WorkAction<Parameters> {
-        private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
-        @Override
-        public void execute() {
-            Set<File> classpathFiles = getParameters().getClasspath().getFiles();
-
-            List<ClassReader> classReaders = ClassReaders.ofPaths(classpathFiles.stream().map(File::toPath)).collect(Collectors.toList());
-
-            NamedComponentScanner namedComponentScanner = new NamedComponentScanner();
-            Map<String, Map<String, String>> namedComponentsMap = namedComponentScanner.scanForNamedClasses(classReaders);
-            writeToFile(namedComponentsMap);
-        }
-
-        private void writeToFile(Map<String, Map<String, String>> namedComponentsMap) {
-            try {
-                String json = OBJECT_MAPPER.writeValueAsString(namedComponentsMap);
-                File file = getParameters().getOutputFile().getAsFile().get();
-                Path of = Path.of(file.getAbsolutePath());
-                Files.writeString(of, json);
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
-
-        }
+    public void setPluginScannerClasspath(FileCollection pluginScannerClasspath) {
+        this.pluginScannerClasspath = pluginScannerClasspath;
     }
 
-    interface Parameters extends WorkParameters {
-
-        ConfigurableFileCollection getClasspath();
-
-        RegularFileProperty getOutputFile();
+    @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
+    public FileCollection getPluginScannerClasspath() {
+        return pluginScannerClasspath;
     }
 }

build-tools/src/main/java/org/elasticsearch/gradle/plugin/StablePluginBuildPlugin.java

@@ -8,9 +8,12 @@
 
 package org.elasticsearch.gradle.plugin;
 
+import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.RegularFile;
 import org.gradle.api.plugins.JavaPlugin;
@@ -33,12 +36,21 @@ public void apply(Project project) {
         });
 
         final var pluginNamedComponents = project.getTasks().register("pluginNamedComponents", GenerateNamedComponentsTask.class, t -> {
+
             SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME);
             FileCollection dependencyJars = project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME);
             FileCollection compiledPluginClasses = mainSourceSet.getOutput().getClassesDirs();
             FileCollection classPath = dependencyJars.plus(compiledPluginClasses);
             t.setClasspath(classPath);
         });
+        Configuration pluginScannerConfig = project.getConfigurations().create("pluginScannerConfig");
+        DependencyHandler dependencyHandler = project.getDependencies();
+        pluginScannerConfig.defaultDependencies(
+            deps -> deps.add(
+                dependencyHandler.create("org.elasticsearch:elasticsearch-plugin-scanner:" + VersionProperties.getElasticsearch())
+            )
+        );
+        pluginNamedComponents.configure(t -> { t.setPluginScannerClasspath(pluginScannerConfig); });
 
         final var pluginExtension = project.getExtensions().getByType(PluginPropertiesExtension.class);
         pluginExtension.getBundleSpec().from(pluginNamedComponents);