[Stable plugin API] Load plugin named components (#89969)

Stable plugins are using @ extensible and @ NamedComponents annotations
to mark components to be loaded.
This commit is loading extensible classNames from extensibles.json and
named components from named_components.json

The scanning mechanism that can generate these files will be done later in a gradle plugin/plugin installer

relates #88980

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -54,6 +54,8 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.geo",
         "org.elasticsearch.logging",
         "org.elasticsearch.lz4",
+        "org.elasticsearch.plugin.analysis.api",
+        "org.elasticsearch.plugin.api",
         "org.elasticsearch.pluginclassloader",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",

docs/changelog/89969.yaml

@@ -0,0 +1,5 @@
+pr: 89969
+summary: "[Stable plugin API] Load plugin named components"
+area: Infra/Plugins
+type: enhancement
+issues: []

libs/build.gradle

@@ -39,5 +39,5 @@ configure(subprojects - project('elasticsearch-log4j')) {
 }
 
 boolean isPluginApi(Project project, Project depProject) {
-  return project.path.matches(".*elasticsearch-plugin-.*-api") &&  depProject.path.equals(':libs:elasticsearch-plugin-api')
+  return project.path.matches(".*elasticsearch-plugin-.*api")
 }

server/build.gradle

@@ -30,6 +30,8 @@ dependencies {
   api project(':libs:elasticsearch-x-content')
   api project(":libs:elasticsearch-geo")
   api project(":libs:elasticsearch-lz4")
+  api project(":libs:elasticsearch-plugin-api")
+  api project(":libs:elasticsearch-plugin-analysis-api")
 
   implementation project(':libs:elasticsearch-plugin-classloader')
 

server/src/main/java/module-info.java

@@ -22,6 +22,8 @@
     requires org.elasticsearch.securesm;
     requires org.elasticsearch.xcontent;
     requires org.elasticsearch.logging;
+    requires org.elasticsearch.plugin.api;
+    requires org.elasticsearch.plugin.analysis.api;
 
     requires com.sun.jna;
     requires hppc;

server/src/main/java/org/elasticsearch/plugins/PluginBundle.java

@@ -20,14 +20,16 @@
 /**
  * A "bundle" is a group of jars that will be loaded in their own classloader
  */
-class PluginBundle {
+public class PluginBundle {
     public final PluginDescriptor plugin;
+    private final Path dir;
     public final Set<URL> urls;
     public final Set<URL> spiUrls;
     public final Set<URL> allUrls;
 
     PluginBundle(PluginDescriptor plugin, Path dir) throws IOException {
         this.plugin = Objects.requireNonNull(plugin);
+        this.dir = dir;
 
         Path spiDir = dir.resolve("spi");
         // plugin has defined an explicit api for extension
@@ -40,6 +42,10 @@ class PluginBundle {
         this.allUrls = allUrls;
     }
 
+    public Path getDir() {
+        return dir;
+    }
+
     public PluginDescriptor pluginDescriptor() {
         return this.plugin;
     }
@@ -82,4 +88,5 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(plugin);
     }
+
 }

server/src/main/java/org/elasticsearch/plugins/scanners/ExtensibleFileReader.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.scanners;
+
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.xcontent.XContentType.JSON;
+
+public class ExtensibleFileReader {
+    private static final Logger logger = LogManager.getLogger(ExtensibleFileReader.class);
+
+    private String extensibleFile;
+
+    public ExtensibleFileReader(String extensibleFile) {
+        this.extensibleFile = extensibleFile;
+    }
+
+    public Map<String, String> readFromFile() {
+        Map<String, String> res = new HashMap<>();
+        // todo should it be BufferedInputStream ?
+        try (InputStream in = getClass().getResourceAsStream(extensibleFile)) {
+            if (in != null) {
+                try (XContentParser parser = JSON.xContent().createParser(XContentParserConfiguration.EMPTY, in)) {
+                    // TODO should we validate the classes actually exist?
+                    return parser.mapStrings();
+                }
+            }
+        } catch (IOException e) {
+            logger.error("failed reading extensible file", e);
+        }
+        return res;
+    }
+
+}

server/src/main/java/org/elasticsearch/plugins/scanners/ExtensiblesRegistry.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.scanners;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Map;
+
+import static org.elasticsearch.core.Strings.format;
+
+/**
+ * A registry of Extensible interfaces/classes read from extensibles.json file.
+ * The file is generated during Elasticsearch built time (or commited)
+ * basing on the classes declared in stable plugins api (i.e. plugin-analysis-api)
+ *
+ * This file is present in server jar.
+ * a class/interface is directly extensible when is marked with @Extensible annotation
+ * a class/interface can be indirectly extensible when it extends/implements a directly extensible class
+ *
+ * Information about extensible interfaces/classes are stored in a map where:
+ * key and value are the same cannonical name of the class that is directly marked with @Extensible
+ * or
+ * key: a cannonical name of the class that is indirectly extensible but extends another extensible class (directly/indirectly)
+ * value: cannonical name of the class that is directly extensible
+ *
+ * The reason for indirectly extensible classes is to allow stable plugin apis to create hierarchies
+ *
+ * Example:
+ * <pre>
+ * &#64;Extensible
+ * interface E{
+ *     public void foo();
+ * }
+ * interface Eprim extends E{
+ * }
+ *
+ * class Aclass implements E{
+ *
+ * }
+ *
+ * &#64;Extensible
+ * class E2 {
+ *     public void bar(){}
+ * }
+ * </pre>
+ * the content of extensibles.json should be
+ * {
+ *     "E" : "E",
+ *     "Eprim" : "E",
+ *     "A" : "E",
+ *     "E2" : "E2"
+ * }
+ *
+ * @see org.elasticsearch.plugin.api.Extensible
+ */
+public class ExtensiblesRegistry {
+
+    private static final Logger logger = LogManager.getLogger(ExtensiblesRegistry.class);
+
+    private static final String EXTENSIBLES_FILE = "/org/elasticsearch/plugins/scanners/extensibles.json";
+    public static final ExtensiblesRegistry INSTANCE = new ExtensiblesRegistry(EXTENSIBLES_FILE);
+
+    // classname (potentially extending/implementing extensible) to interface/class annotated with extensible
+    private final Map<String, String> loadedExtensible;
+
+    ExtensiblesRegistry(String extensiblesFile) {
+        ExtensibleFileReader extensibleFileReader = new ExtensibleFileReader(extensiblesFile);
+
+        this.loadedExtensible = extensibleFileReader.readFromFile();
+        if (loadedExtensible.size() > 0) {
+            logger.debug(() -> format("Loaded extensible from cache file %s", loadedExtensible));
+        }
+    }
+
+    public boolean hasExtensible(String extensibleName) {
+        return loadedExtensible.containsKey(extensibleName);
+    }
+
+}

server/src/main/java/org/elasticsearch/plugins/scanners/NameToPluginInfo.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.scanners;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public record NameToPluginInfo(Map<String, PluginInfo> nameToPluginInfoMap) {
+
+    public NameToPluginInfo() {
+        this(new HashMap<>());
+    }
+
+    public NameToPluginInfo put(String name, PluginInfo pluginInfo) {
+        nameToPluginInfoMap.put(name, pluginInfo);
+        return this;
+    }
+
+    public void putAll(Map<String, PluginInfo> namedPluginInfoMap) {
+        this.nameToPluginInfoMap.putAll(namedPluginInfoMap);
+    }
+
+    public NameToPluginInfo put(NameToPluginInfo nameToPluginInfo) {
+        putAll(nameToPluginInfo.nameToPluginInfoMap);
+        return this;
+    }
+
+    public PluginInfo getForPluginName(String pluginName) {
+        return nameToPluginInfoMap.get(pluginName);
+    }
+
+}

server/src/main/java/org/elasticsearch/plugins/scanners/NamedComponentReader.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.scanners;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.plugins.PluginBundle;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xcontent.XContentType.JSON;
+
+/**
+ * Reads named components declared by a plugin in a cache file.
+ * Cache file is expected to be present in plugin's lib directory
+ * <p>
+ * The content of a cache file is a JSON representation of a map where:
+ * keys -> name of the extensible interface (a class/interface marked with @Extensible)
+ * values -> a map of name to implementation class name
+ */
+public class NamedComponentReader {
+
+    private Logger logger = LogManager.getLogger(NamedComponentReader.class);
+    private static final String NAMED_COMPONENTS_FILE_NAME = "named_components.json";
+    /**
+     * a registry of known classes marked or indirectly marked (extending marked class) with @Extensible
+     */
+    private final ExtensiblesRegistry extensiblesRegistry;
+
+    public NamedComponentReader() {
+        this(ExtensiblesRegistry.INSTANCE);
+    }
+
+    NamedComponentReader(ExtensiblesRegistry extensiblesRegistry) {
+        this.extensiblesRegistry = extensiblesRegistry;
+    }
+
+    public Map<String, NameToPluginInfo> findNamedComponents(PluginBundle bundle, ClassLoader pluginClassLoader) {
+        Path pluginDir = bundle.getDir();
+        return findNamedComponents(pluginDir, pluginClassLoader);
+    }
+
+    // scope for testing
+    Map<String, NameToPluginInfo> findNamedComponents(Path pluginDir, ClassLoader pluginClassLoader) {
+        try {
+            Path namedComponent = findNamedComponentCacheFile(pluginDir);
+            if (namedComponent != null) {
+                Map<String, NameToPluginInfo> namedComponents = readFromFile(namedComponent, pluginClassLoader);
+                logger.debug(() -> Strings.format("Plugin in dir %s declared named components %s.", pluginDir, namedComponents));
+
+                return namedComponents;
+            }
+            logger.debug(() -> Strings.format("No named component defined in plugin dir %s", pluginDir));
+        } catch (IOException e) {
+            logger.error("unable to read named components", e);
+        }
+        return emptyMap();
+    }
+
+    private Path findNamedComponentCacheFile(Path pluginDir) throws IOException {
+        try (Stream<Path> list = Files.list(pluginDir)) {
+            return list.filter(p -> p.getFileName().toString().equals(NAMED_COMPONENTS_FILE_NAME)).findFirst().orElse(null);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    Map<String, NameToPluginInfo> readFromFile(Path namedComponent, ClassLoader pluginClassLoader) throws IOException {
+        Map<String, NameToPluginInfo> res = new HashMap<>();
+
+        try (
+            var json = new BufferedInputStream(Files.newInputStream(namedComponent));
+            var parser = JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)
+        ) {
+            Map<String, Object> map = parser.map();
+            for (Map.Entry<String, Object> fileAsMap : map.entrySet()) {
+                String extensibleInterface = fileAsMap.getKey();
+                validateExtensible(extensibleInterface);
+                Map<String, Object> components = (Map<String, Object>) fileAsMap.getValue();
+                for (Map.Entry<String, Object> nameToComponent : components.entrySet()) {
+                    String name = nameToComponent.getKey();
+                    String value = (String) nameToComponent.getValue();
+
+                    res.computeIfAbsent(extensibleInterface, k -> new NameToPluginInfo())
+                        .put(name, new PluginInfo(name, value, pluginClassLoader));
+                }
+            }
+        }
+        return res;
+    }
+
+    private void validateExtensible(String extensibleInterface) {
+        if (extensiblesRegistry.hasExtensible(extensibleInterface) == false) {
+            throw new IllegalStateException("Unknown extensible name " + extensibleInterface);
+        }
+    }
+}

server/src/main/java/org/elasticsearch/plugins/scanners/PluginInfo.java

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.scanners;
+
+record PluginInfo(String name, String className, ClassLoader loader) {
+
+}