[Extensions] REST API to initialize an extension and dynamically load…

… it (opensearch-project#8029) * Implemented REST API for initializing extension Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Cleanup extensions.yml design Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Added tests for RestInitializeExtensionAction Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Pulled extensions REST request in extensions directory Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Removed forbidden APIs from rest action and modified tests Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Added entry in changelog Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Added test for parse Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Addressed PR comments Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Addressed PR comments Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Spotless Fixed Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Handled exceptions Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> * Handled test failure Signed-off-by: Owais Kazi <owaiskazi19@gmail.com> --------- Signed-off-by: Owais Kazi <owaiskazi19@gmail.com>
owaiskazi19 · Jun 22, 2023 · e37d6ab · e37d6ab
1 parent fced6d4
commit e37d6ab
Show file tree

Hide file tree

Showing 17 changed files with 540 additions and 368 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add descending order search optimization through reverse segment read. ([#7967](https://github.com/opensearch-project/OpenSearch/pull/7967))
 - Update components of segrep backpressure to support remote store. ([#8020](https://github.com/opensearch-project/OpenSearch/pull/8020))
 - Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038))
+- Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029)
 
 ### Dependencies
 - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814)

diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java
@@ -294,8 +294,10 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsFilter;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.extensions.ExtensionsManager;
 import org.opensearch.extensions.action.ExtensionProxyAction;
 import org.opensearch.extensions.action.ExtensionProxyTransportAction;
+import org.opensearch.extensions.rest.RestInitializeExtensionAction;
 import org.opensearch.index.seqno.RetentionLeaseActions;
 import org.opensearch.identity.IdentityService;
 import org.opensearch.indices.SystemIndices;
@@ -453,7 +455,7 @@
 import org.opensearch.rest.action.search.RestPutSearchPipelineAction;
 import org.opensearch.rest.action.search.RestSearchAction;
 import org.opensearch.rest.action.search.RestSearchScrollAction;
-import org.opensearch.rest.extensions.RestSendToExtensionAction;
+import org.opensearch.extensions.rest.RestSendToExtensionAction;
 import org.opensearch.tasks.Task;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.usage.UsageService;
@@ -508,6 +510,7 @@ public class ActionModule extends AbstractModule {
     private final RequestValidators<PutMappingRequest> mappingRequestValidators;
     private final RequestValidators<IndicesAliasesRequest> indicesAliasesRequestRequestValidators;
     private final ThreadPool threadPool;
+    private final ExtensionsManager extensionsManager;
 
     public ActionModule(
         Settings settings,
@@ -521,7 +524,8 @@ public ActionModule(
         CircuitBreakerService circuitBreakerService,
         UsageService usageService,
         SystemIndices systemIndices,
-        IdentityService identityService
+        IdentityService identityService,
+        ExtensionsManager extensionsManager
     ) {
         this.settings = settings;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
@@ -530,6 +534,7 @@ public ActionModule(
         this.settingsFilter = settingsFilter;
         this.actionPlugins = actionPlugins;
         this.threadPool = threadPool;
+        this.extensionsManager = extensionsManager;
         actions = setupActions(actionPlugins);
         actionFilters = setupActionFilters(actionPlugins);
         dynamicActionRegistry = new DynamicActionRegistry();
@@ -947,6 +952,11 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
             registerHandler.accept(new RestDeleteSearchPipelineAction());
         }
 
+        // Extensions API
+        if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) {
+            registerHandler.accept(new RestInitializeExtensionAction(extensionsManager));
+        }
+
         for (ActionPlugin plugin : actionPlugins) {
             for (RestHandler handler : plugin.getRestHandlers(
                 settings,

diff --git a/server/src/main/java/org/opensearch/bootstrap/Security.java b/server/src/main/java/org/opensearch/bootstrap/Security.java
@@ -312,12 +312,11 @@ static void addClasspathPermissions(Permissions policy) throws IOException {
      */
     static void addFilePermissions(Permissions policy, Environment environment) throws IOException {
         // read-only dirs
-        addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.binFile(), "read,readlink", false);
-        addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.libFile(), "read,readlink", false);
-        addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.modulesFile(), "read,readlink", false);
-        addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.pluginsFile(), "read,readlink", false);
-        addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.extensionDir(), "read,readlink", false);
-        addDirectoryPath(policy, "path.conf'", environment.configFile(), "read,readlink", false);
+        addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.binDir(), "read,readlink", false);
+        addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.libDir(), "read,readlink", false);
+        addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.modulesDir(), "read,readlink", false);
+        addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.pluginsDir(), "read,readlink", false);
+        addDirectoryPath(policy, "path.conf'", environment.configDir(), "read,readlink", false);
         // read-write dirs
         addDirectoryPath(policy, "java.io.tmpdir", environment.tmpFile(), "read,readlink,write,delete", false);
         addDirectoryPath(policy, Environment.PATH_LOGS_SETTING.getKey(), environment.logsFile(), "read,readlink,write,delete", false);

diff --git a/server/src/main/java/org/opensearch/env/Environment.java b/server/src/main/java/org/opensearch/env/Environment.java
@@ -312,10 +312,6 @@ public Path pluginsFile() {
         return pluginsFile;
     }
 
-    public Path extensionDir() {
-        return extensionsDir;
-    }
-
     public Path binFile() {
         return binFile;
     }

diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionDependency.java b/server/src/main/java/org/opensearch/extensions/ExtensionDependency.java
@@ -16,6 +16,10 @@
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
 import org.opensearch.common.io.stream.Writeable;
+import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.XContentParser;
+
+import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 /**
  * This class handles the dependent extensions information
@@ -25,6 +29,8 @@
 public class ExtensionDependency implements Writeable {
     private String uniqueId;
     private Version version;
+    private static final String UNIQUE_ID = "uniqueId";
+    private static final String VERSION = "version";
 
     public ExtensionDependency(String uniqueId, Version version) {
         this.uniqueId = uniqueId;
@@ -54,6 +60,39 @@ public void writeTo(StreamOutput out) throws IOException {
         Version.writeVersion(version, out);
     }
 
+    public static ExtensionDependency parse(XContentParser parser) throws IOException {
+        String uniqueId = null;
+        Version version = null;
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            String fieldName = parser.currentName();
+            parser.nextToken();
+
+            switch (fieldName) {
+                case UNIQUE_ID:
+                    uniqueId = parser.text();
+                    break;
+                case VERSION:
+                    try {
+                        version = Version.fromString(parser.text());
+                    } catch (IllegalArgumentException e) {
+                        throw e;
+                    }
+                    break;
+                default:
+                    parser.skipChildren();
+                    break;
+            }
+        }
+        if (Strings.isNullOrEmpty(uniqueId)) {
+            throw new IOException("Required field [uniqueId] is missing in the request for the dependent extension");
+        } else if (version == null) {
+            throw new IOException("Required field [version] is missing in the request for the dependent extension");
+        }
+        return new ExtensionDependency(uniqueId, version);
+
+    }
+
     /**
     * The uniqueId of the dependency extension
     *