[8.x] [Entitlements] Differentiate between ES modules and plugins (external) (#117973)

libs/entitlement/qa/build.gradle

@@ -13,8 +13,8 @@ apply plugin: 'elasticsearch.internal-test-artifact'
 
 dependencies {
   javaRestTestImplementation project(':libs:entitlement:qa:common')
-  clusterPlugins project(':libs:entitlement:qa:entitlement-allowed')
-  clusterPlugins project(':libs:entitlement:qa:entitlement-allowed-nonmodular')
+  clusterModules project(':libs:entitlement:qa:entitlement-allowed')
+  clusterModules project(':libs:entitlement:qa:entitlement-allowed-nonmodular')
   clusterPlugins project(':libs:entitlement:qa:entitlement-denied')
   clusterPlugins project(':libs:entitlement:qa:entitlement-denied-nonmodular')
 }

libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java

@@ -28,8 +28,8 @@ public class EntitlementsAllowedIT extends ESRestTestCase {
 
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .plugin("entitlement-allowed")
-        .plugin("entitlement-allowed-nonmodular")
+        .module("entitlement-allowed")
+        .module("entitlement-allowed-nonmodular")
         .systemProperty("es.entitlements.enabled", "true")
         .setting("xpack.security.enabled", "false")
         .build();

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -15,7 +15,6 @@
 import com.sun.tools.attach.VirtualMachine;
 
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
@@ -29,7 +28,9 @@
 
 public class EntitlementBootstrap {
 
-    public record BootstrapArgs(Collection<Tuple<Path, Boolean>> pluginData, Function<Class<?>, String> pluginResolver) {}
+    public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) {}
+
+    public record BootstrapArgs(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {}
 
     private static BootstrapArgs bootstrapArgs;
 
@@ -40,11 +41,11 @@ public static BootstrapArgs bootstrapArgs() {
     /**
      * Activates entitlement checking. Once this method returns, calls to methods protected by Entitlements from classes without a valid
      * policy will throw {@link org.elasticsearch.entitlement.runtime.api.NotEntitledException}.
-     * @param pluginData a collection of (plugin path, boolean), that holds the paths of all the installed Elasticsearch modules and
-     *                   plugins, and whether they are Java modular or not.
+     * @param pluginData a collection of (plugin path, boolean, boolean), that holds the paths of all the installed Elasticsearch modules
+     *                   and plugins, whether they are Java modular or not, and whether they are Elasticsearch modules or external plugins.
      * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
      */
-    public static void bootstrap(Collection<Tuple<Path, Boolean>> pluginData, Function<Class<?>, String> pluginResolver) {
+    public static void bootstrap(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
             throw new IllegalStateException("plugin data is already set");

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -9,7 +9,6 @@
 
 package org.elasticsearch.entitlement.initialization;
 
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.core.internal.provider.ProviderLocator;
 import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
 import org.elasticsearch.entitlement.bridge.EntitlementChecker;
@@ -96,25 +95,25 @@ private static PolicyManager createPolicyManager() throws IOException {
         return new PolicyManager(serverPolicy, pluginPolicies, EntitlementBootstrap.bootstrapArgs().pluginResolver());
     }
 
-    private static Map<String, Policy> createPluginPolicies(Collection<Tuple<Path, Boolean>> pluginData) throws IOException {
+    private static Map<String, Policy> createPluginPolicies(Collection<EntitlementBootstrap.PluginData> pluginData) throws IOException {
         Map<String, Policy> pluginPolicies = new HashMap<>(pluginData.size());
-        for (Tuple<Path, Boolean> entry : pluginData) {
-            Path pluginRoot = entry.v1();
-            boolean isModular = entry.v2();
-
+        for (var entry : pluginData) {
+            Path pluginRoot = entry.pluginPath();
             String pluginName = pluginRoot.getFileName().toString();
-            final Policy policy = loadPluginPolicy(pluginRoot, isModular, pluginName);
+
+            final Policy policy = loadPluginPolicy(pluginRoot, entry.isModular(), pluginName, entry.isExternalPlugin());
 
             pluginPolicies.put(pluginName, policy);
         }
         return pluginPolicies;
     }
 
-    private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName) throws IOException {
+    private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName, boolean isExternalPlugin)
+        throws IOException {
         Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
 
         final Set<String> moduleNames = getModuleNames(pluginRoot, isModular);
-        final Policy policy = parsePolicyIfExists(pluginName, policyFile);
+        final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin);
 
         // TODO: should this check actually be part of the parser?
         for (Scope scope : policy.scopes) {
@@ -125,9 +124,9 @@ private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, Strin
         return policy;
     }
 
-    private static Policy parsePolicyIfExists(String pluginName, Path policyFile) throws IOException {
+    private static Policy parsePolicyIfExists(String pluginName, Path policyFile, boolean isExternalPlugin) throws IOException {
         if (Files.exists(policyFile)) {
-            return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName).parsePolicy();
+            return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName, isExternalPlugin).parsePolicy();
         }
         return new Policy(pluginName, List.of());
     }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java

@@ -33,4 +33,12 @@
      * have to match the parameter names of the constructor.
      */
     String[] parameterNames() default {};
+
+    /**
+     * This flag indicates if this Entitlement can be used in external plugins,
+     * or if it can be used only in Elasticsearch modules ("internal" plugins).
+     * Using an entitlement that is not {@code pluginsAccessible} in an external
+     * plugin policy will throw in exception while parsing.
+     */
+    boolean esModulesOnly() default true;
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java

@@ -26,7 +26,7 @@ public class FileEntitlement implements Entitlement {
     private final String path;
     private final int actions;
 
-    @ExternalEntitlement(parameterNames = { "path", "actions" })
+    @ExternalEntitlement(parameterNames = { "path", "actions" }, esModulesOnly = false)
     public FileEntitlement(String path, List<String> actionsList) {
         this.path = path;
         int actionsInt = 0;

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java

@@ -39,6 +39,7 @@ public class PolicyParser {
 
     protected final XContentParser policyParser;
     protected final String policyName;
+    private final boolean isExternalPlugin;
 
     static String getEntitlementTypeName(Class<? extends Entitlement> entitlementClass) {
         var entitlementClassName = entitlementClass.getSimpleName();
@@ -56,9 +57,10 @@ static String getEntitlementTypeName(Class<? extends Entitlement> entitlementCla
             .collect(Collectors.joining("_"));
     }
 
-    public PolicyParser(InputStream inputStream, String policyName) throws IOException {
+    public PolicyParser(InputStream inputStream, String policyName, boolean isExternalPlugin) throws IOException {
         this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream));
         this.policyName = policyName;
+        this.isExternalPlugin = isExternalPlugin;
     }
 
     public Policy parsePolicy() {
@@ -125,6 +127,10 @@ protected Entitlement parseEntitlement(String scopeName, String entitlementType)
             throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]");
         }
 
+        if (entitlementMetadata.esModulesOnly() && isExternalPlugin) {
+            throw newPolicyParserException("entitlement type [" + entitlementType + "] is allowed only on modules");
+        }
+
         Class<?>[] parameterTypes = entitlementConstructor.getParameterTypes();
         String[] parametersNames = entitlementMetadata.parameterNames();
 

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java

@@ -19,7 +19,7 @@ public class PolicyParserFailureTests extends ESTestCase {
     public void testParserSyntaxFailures() {
         PolicyParserException ppe = expectThrows(
             PolicyParserException.class,
-            () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml")
+            () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false)
                 .parsePolicy()
         );
         assertEquals("[1:1] policy parsing error for [test-failure-policy.yaml]: expected object <scope name>", ppe.getMessage());
@@ -29,7 +29,7 @@ public void testEntitlementDoesNotExist() {
         PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
             entitlement-module-name:
               - does_not_exist: {}
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name]: "
                 + "unknown entitlement type [does_not_exist]",
@@ -41,7 +41,7 @@ public void testEntitlementMissingParameter() {
         PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
             entitlement-module-name:
               - file: {}
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[2:12] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
                 + "for entitlement type [file]: missing entitlement parameter [path]",
@@ -52,7 +52,7 @@ public void testEntitlementMissingParameter() {
             entitlement-module-name:
               - file:
                   path: test-path
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[4:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
                 + "for entitlement type [file]: missing entitlement parameter [actions]",
@@ -68,11 +68,22 @@ public void testEntitlementExtraneousParameter() {
                   actions:
                     - read
                   extra: test
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[7:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
                 + "for entitlement type [file]: extraneous entitlement parameter(s) {extra=test}",
             ppe.getMessage()
         );
     }
+
+    public void testEntitlementIsNotForExternalPlugins() {
+        PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              - create_class_loader
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", true).parsePolicy());
+        assertEquals(
+            "[2:5] policy parsing error for [test-failure-policy.yaml]: entitlement type [create_class_loader] is allowed only on modules",
+            ppe.getMessage()
+        );
+    }
 }

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java

@@ -37,7 +37,17 @@ public void testGetEntitlementTypeName() {
     }
 
     public void testPolicyBuilder() throws IOException {
-        Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml")
+        Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", false)
+            .parsePolicy();
+        Policy builtPolicy = new Policy(
+            "test-policy.yaml",
+            List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write")))))
+        );
+        assertEquals(parsedPolicy, builtPolicy);
+    }
+
+    public void testPolicyBuilderOnExternalPlugin() throws IOException {
+        Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", true)
             .parsePolicy();
         Policy builtPolicy = new Policy(
             "test-policy.yaml",
@@ -50,7 +60,7 @@ public void testParseCreateClassloader() throws IOException {
         Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream("""
             entitlement-module-name:
               - create_class_loader
-            """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml").parsePolicy();
+            """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy();
         Policy builtPolicy = new Policy(
             "test-policy.yaml",
             List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement())))

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -30,7 +30,6 @@
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexVersion;
@@ -56,6 +55,7 @@
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.bootstrap.BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING;
 import static org.elasticsearch.nativeaccess.WindowsFunctions.ConsoleCtrlHandler.CTRL_CLOSE_EVENT;
@@ -209,10 +209,14 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
         if (Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"))) {
             LogManager.getLogger(Elasticsearch.class).info("Bootstrapping Entitlements");
 
-            List<Tuple<Path, Boolean>> pluginData = pluginsLoader.allBundles()
-                .stream()
-                .map(bundle -> Tuple.tuple(bundle.getDir(), bundle.pluginDescriptor().isModular()))
-                .toList();
+            List<EntitlementBootstrap.PluginData> pluginData = Stream.concat(
+                pluginsLoader.moduleBundles()
+                    .stream()
+                    .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), false)),
+                pluginsLoader.pluginBundles()
+                    .stream()
+                    .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), true))
+            ).toList();
 
             EntitlementBootstrap.bootstrap(pluginData, pluginsResolver::resolveClassToPluginName);
         } else {

server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java

@@ -122,7 +122,8 @@ public static LayerAndLoader ofUberModuleLoader(UberModuleClassLoader loader) {
     private final List<PluginDescriptor> moduleDescriptors;
     private final List<PluginDescriptor> pluginDescriptors;
     private final Map<String, LoadedPluginLayer> loadedPluginLayers;
-    private final Set<PluginBundle> allBundles;
+    private final Set<PluginBundle> moduleBundles;
+    private final Set<PluginBundle> pluginBundles;
 
     /**
      * Constructs a new PluginsLoader
@@ -153,37 +154,36 @@ public static PluginsLoader createPluginsLoader(Path modulesDirectory, Path plug
         Set<PluginBundle> seenBundles = new LinkedHashSet<>();
 
         // load (elasticsearch) module layers
-        List<PluginDescriptor> moduleDescriptors;
+        final Set<PluginBundle> modules;
         if (modulesDirectory != null) {
             try {
-                Set<PluginBundle> modules = PluginsUtils.getModuleBundles(modulesDirectory);
-                moduleDescriptors = modules.stream().map(PluginBundle::pluginDescriptor).toList();
+                modules = PluginsUtils.getModuleBundles(modulesDirectory);
                 seenBundles.addAll(modules);
             } catch (IOException ex) {
                 throw new IllegalStateException("Unable to initialize modules", ex);
             }
         } else {
-            moduleDescriptors = Collections.emptyList();
+            modules = Collections.emptySet();
         }
 
         // load plugin layers
-        List<PluginDescriptor> pluginDescriptors;
+        final Set<PluginBundle> plugins;
         if (pluginsDirectory != null) {
             try {
                 // TODO: remove this leniency, but tests bogusly rely on it
                 if (isAccessibleDirectory(pluginsDirectory, logger)) {
                     PluginsUtils.checkForFailedPluginRemovals(pluginsDirectory);
-                    Set<PluginBundle> plugins = PluginsUtils.getPluginBundles(pluginsDirectory);
-                    pluginDescriptors = plugins.stream().map(PluginBundle::pluginDescriptor).toList();
+                    plugins = PluginsUtils.getPluginBundles(pluginsDirectory);
+
                     seenBundles.addAll(plugins);
                 } else {
-                    pluginDescriptors = Collections.emptyList();
+                    plugins = Collections.emptySet();
                 }
             } catch (IOException ex) {
                 throw new IllegalStateException("Unable to initialize plugins", ex);
             }
         } else {
-            pluginDescriptors = Collections.emptyList();
+            plugins = Collections.emptySet();
         }
 
         Map<String, LoadedPluginLayer> loadedPluginLayers = new LinkedHashMap<>();
@@ -197,19 +197,15 @@ public static PluginsLoader createPluginsLoader(Path modulesDirectory, Path plug
             }
         }
 
-        return new PluginsLoader(moduleDescriptors, pluginDescriptors, loadedPluginLayers, Set.copyOf(seenBundles));
+        return new PluginsLoader(modules, plugins, loadedPluginLayers);
     }
 
-    PluginsLoader(
-        List<PluginDescriptor> moduleDescriptors,
-        List<PluginDescriptor> pluginDescriptors,
-        Map<String, LoadedPluginLayer> loadedPluginLayers,
-        Set<PluginBundle> allBundles
-    ) {
-        this.moduleDescriptors = moduleDescriptors;
-        this.pluginDescriptors = pluginDescriptors;
+    PluginsLoader(Set<PluginBundle> modules, Set<PluginBundle> plugins, Map<String, LoadedPluginLayer> loadedPluginLayers) {
+        this.moduleBundles = modules;
+        this.pluginBundles = plugins;
+        this.moduleDescriptors = modules.stream().map(PluginBundle::pluginDescriptor).toList();
+        this.pluginDescriptors = plugins.stream().map(PluginBundle::pluginDescriptor).toList();
         this.loadedPluginLayers = loadedPluginLayers;
-        this.allBundles = allBundles;
     }
 
     public List<PluginDescriptor> moduleDescriptors() {
@@ -224,8 +220,12 @@ public Stream<PluginLayer> pluginLayers() {
         return loadedPluginLayers.values().stream().map(Function.identity());
     }
 
-    public Set<PluginBundle> allBundles() {
-        return allBundles;
+    public Set<PluginBundle> moduleBundles() {
+        return moduleBundles;
+    }
+
+    public Set<PluginBundle> pluginBundles() {
+        return pluginBundles;
     }
 
     private static void loadPluginLayer(