[Extensions] Add ExtensionAwarePlugin extension point to add custom settings for extensions

CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
 - Add connectToNodeAsExtension in TransportService ([#6866](https://github.com/opensearch-project/OpenSearch/pull/6866))
 - Adds ExtensionsManager.lookupExtensionSettingsById ([#7466](https://github.com/opensearch-project/OpenSearch/pull/7466))
+- [Extensions] Allow IdentityPlugin to add settings for extensions/extensions.yml ([#7526](https://github.com/opensearch-project/OpenSearch/pull/7526))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -120,4 +121,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.7...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.7...2.x

plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java

@@ -88,4 +88,9 @@
             .orElseThrow(() -> new UnsupportedAuthenticationToken());
         shiroSubject.login(authToken);
     }
+
+    @Override
+    public boolean isAuthenticated() {
+        return shiroSubject.isAuthenticated();
+    }
 }

server/src/main/java/org/opensearch/common/settings/Setting.java

@@ -132,6 +132,11 @@ public enum Property {
          */
         Deprecated,
 
+        /**
+         * Extension scope
+         */
+        ExtensionScope,
+
         /**
          * Node scope
          */

server/src/main/java/org/opensearch/extensions/ExtensionAdditionalSettings.java

@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.extensions;
+
+import org.opensearch.common.settings.AbstractScopedSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Setting.Property;
+import org.opensearch.common.settings.SettingUpgrader;
+import org.opensearch.common.settings.Settings;
+
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * Encapsulates all valid extension level settings.
+ *
+ * @opensearch.internal
+ */
+public final class ExtensionAdditionalSettings extends AbstractScopedSettings {
+
+    public ExtensionAdditionalSettings(final Set<Setting<?>> settingsSet) {
+        this(settingsSet, Collections.emptySet());
+    }
+
+    public ExtensionAdditionalSettings(final Set<Setting<?>> settingsSet, final Set<SettingUpgrader<?>> settingUpgraders) {
+        super(Settings.EMPTY, settingsSet, settingUpgraders, Property.ExtensionScope);
+    }
+}

server/src/main/java/org/opensearch/extensions/ExtensionsManager.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.TimeUnit;
@@ -57,6 +58,7 @@
 import org.opensearch.extensions.rest.RestActionsRequestHandler;
 import org.opensearch.extensions.settings.CustomSettingsRequestHandler;
 import org.opensearch.extensions.settings.RegisterCustomSettingsRequest;
+import org.opensearch.identity.IdentityService;
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexService;
 import org.opensearch.index.IndicesModuleRequest;
@@ -116,6 +118,7 @@
     private CustomSettingsRequestHandler customSettingsRequestHandler;
     private TransportService transportService;
     private ClusterService clusterService;
+    private IdentityService identityService;
     private Settings environmentSettings;
     private AddSettingsUpdateConsumerRequestHandler addSettingsUpdateConsumerRequestHandler;
     private NodeClient client;
@@ -135,6 +138,34 @@
         // will be initialized in initializeServicesAndRestHandler which is called after the Node is initialized
         this.transportService = null;
         this.clusterService = null;
+        this.identityService = null;
+        this.client = null;
+        this.extensionTransportActionsHandler = null;
+
+        /*
+         * Now Discover extensions
+         */
+        discover();
+
+    }
+
+    /**
+     * Instantiate a new ExtensionsManager object to handle requests and responses from extensions. This is called during Node bootstrap.
+     *
+     * @param extensionsPath  Path to a directory containing extensions.
+     * @param identityService  Identity Service
+     * @throws IOException  If the extensions discovery file is not properly retrieved.
+     */
+    public ExtensionsManager(Path extensionsPath, IdentityService identityService) throws IOException {
+        logger.info("ExtensionsManager initialized");
+        this.extensionsPath = extensionsPath;
+        this.initializedExtensions = new HashMap<String, DiscoveryExtensionNode>();
+        this.extensionIdMap = new HashMap<String, DiscoveryExtensionNode>();
+        this.extensionSettingsMap = new HashMap<String, Extension>();
+        // will be initialized in initializeServicesAndRestHandler which is called after the Node is initialized
+        this.transportService = null;
+        this.clusterService = null;
+        this.identityService = identityService;
         this.client = null;
         this.extensionTransportActionsHandler = null;
 
@@ -627,6 +658,25 @@
                         }
                     }
 
+                    Settings.Builder output = Settings.builder();
+                    ExtensionAdditionalSettings additionalSettings = new ExtensionAdditionalSettings(Set.of());
+
+                    if (identityService != null) {
+                        additionalSettings = new ExtensionAdditionalSettings(
+                            identityService.getExtensionSettings().stream().collect(Collectors.toSet())
+                        );
+                        Set<String> securitySettingsKeys = identityService.getExtensionSettings()
+                            .stream()
+                            .map(s -> s.getKey())
+                            .collect(Collectors.toSet());
+                        Map<String, ?> securitySettings = extensionMap.entrySet()
+                            .stream()
+                            .filter(kv -> securitySettingsKeys.contains(kv.getKey()))
+                            .collect(Collectors.toMap(map -> map.getKey(), map -> map.getValue()));
+                        output.loadFromMap(securitySettings);
+                        additionalSettings.applySettings(output.build());
+                    }
+
                     // Create extension read from yml config
                     readExtensions.add(
                         new Extension(
@@ -637,7 +687,8 @@
                             extensionMap.get("version").toString(),
                             extensionMap.get("opensearchVersion").toString(),
                             extensionMap.get("minimumCompatibleVersion").toString(),
-                            extensionDependencyList
+                            extensionDependencyList,
+                            additionalSettings
                         )
                     );
                 } catch (IOException e) {
@@ -709,6 +760,10 @@
         this.clusterService = clusterService;
     }
 
+    void setIdentityService(IdentityService identityService) {
+        this.identityService = identityService;
+    }
+
     CustomSettingsRequestHandler getCustomSettingsRequestHandler() {
         return customSettingsRequestHandler;
     }

server/src/main/java/org/opensearch/extensions/ExtensionsSettings.java

@@ -44,6 +44,7 @@ public static class Extension {
         private String opensearchVersion;
         private String minimumCompatibleVersion;
         private List<ExtensionDependency> dependencies = Collections.emptyList();
+        private ExtensionAdditionalSettings additionalSettings;
 
         public Extension(
             String name,
@@ -53,7 +54,8 @@ public Extension(
             String version,
             String opensearchVersion,
             String minimumCompatibleVersion,
-            List<ExtensionDependency> dependencies
+            List<ExtensionDependency> dependencies,
+            ExtensionAdditionalSettings additionalSettings
         ) {
             this.name = name;
             this.uniqueId = uniqueId;
@@ -63,6 +65,7 @@ public Extension(
             this.opensearchVersion = opensearchVersion;
             this.minimumCompatibleVersion = minimumCompatibleVersion;
             this.dependencies = dependencies;
+            this.additionalSettings = additionalSettings;
         }
 
         public Extension() {
@@ -127,6 +130,10 @@ public List<ExtensionDependency> getDependencies() {
             return dependencies;
         }
 
+        public ExtensionAdditionalSettings getAdditionalSettings() {
+            return additionalSettings;
+        }
+
         public String getMinimumCompatibleVersion() {
             return minimumCompatibleVersion;
         }

server/src/main/java/org/opensearch/identity/IdentityService.java

@@ -8,6 +8,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.OpenSearchException;
+import org.opensearch.common.settings.Setting;
 import org.opensearch.identity.noop.NoopIdentityPlugin;
 import java.util.List;
 import org.opensearch.common.settings.Settings;
@@ -48,4 +49,11 @@ public IdentityService(final Settings settings, final List<IdentityPlugin> ident
     public Subject getSubject() {
         return identityPlugin.getSubject();
     }
+
+    /**
+     * Gets a list of settings to register for extensions
+     */
+    public List<Setting<?>> getExtensionSettings() {
+        return identityPlugin.getExtensionSettings();
+    }
 }

server/src/main/java/org/opensearch/identity/Subject.java

@@ -29,4 +29,9 @@ public interface Subject {
      * throws SubjectDisabled
      */
     void authenticate(final AuthToken token);
+
+    /**
+     * Method that returns whether the current subject of the running thread is authenticated
+     */
+    boolean isAuthenticated();
 }

server/src/main/java/org/opensearch/identity/noop/NoopSubject.java

@@ -54,4 +54,9 @@
     public void authenticate(AuthToken AuthToken) {
         // Do nothing as noop subject is always logged in
     }
+
+    @Override
+    public boolean isAuthenticated() {
+        return true;
+    }
 }

server/src/main/java/org/opensearch/node/Node.java

@@ -466,7 +466,7 @@
             final IdentityService identityService = new IdentityService(settings, identityPlugins);
 
             if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) {
-                this.extensionsManager = new ExtensionsManager(initialEnvironment.extensionDir());
+                this.extensionsManager = new ExtensionsManager(initialEnvironment.extensionDir(), identityService);
             } else {
                 this.extensionsManager = new NoopExtensionsManager();
             }

server/src/main/java/org/opensearch/plugins/IdentityPlugin.java

@@ -8,8 +8,12 @@
 
 package org.opensearch.plugins;
 
+import org.opensearch.common.settings.Setting;
 import org.opensearch.identity.Subject;
 
+import java.util.Collections;
+import java.util.List;
+
 /**
  * Plugin that provides identity and access control for OpenSearch
  *
@@ -22,5 +26,12 @@
      *
      * Should never return null
      * */
-    public Subject getSubject();
+    Subject getSubject();
+
+    /**
+     * Returns a list of additional {@link Setting} definitions for that this plugin adds for extensions
+     */
+    default List<Setting<?>> getExtensionSettings() {
+        return Collections.emptyList();
+    }
 }