Setup OpenSearch plugin dependencies on cluster snapshot (#2734)

* Adding yarn opensearch args to setup opensearch plugin dependencies on snapshot
* Updating the example opensearch snapshot help command with --P arg
* Adding tests to the plugin installation function
* Adding documentation for the new flag in developer guide

Signed-off-by: Manasvini B Suryanarayana <manasvis@amazon.com>

CHANGELOG.md

@@ -44,6 +44,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Save Object Aggregation View] Fix for export all after scroll count response changed in PR#2656 ([#2696](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2696))
 - [Vis Builder] Add an experimental table visualization in vis builder ([#2705](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2705))
 - [Vis Builder] Add field summary popovers ([#2682](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2682))
+- Add yarn opensearch arg to setup plugin dependencies ([#2544](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/2544))
 
 ### 🐛 Bug Fixes
 

DEVELOPER_GUIDE.md

@@ -59,6 +59,30 @@ Dashboards. In a separate terminal you can run the latest snapshot built using:
 $ yarn opensearch snapshot 
 ```
 
+If you would like to download a specific OpenSearch plugin on the cluster snapshot, pass the `--P` flag after `yarn opensearch snapshot`. We can use the flag multiple times to install multiple plugins on the cluster snapshot. The argument value can be URL to the plugin's zip file, maven coordinates of the plugin or for local zip files, use `file:` followed by the absolute or relative path to the plugin zip file. Below is the example help command:
+
+```
+$ yarn opensearch snapshot --P https://repo1.maven.org/maven2/org/opensearch/plugin/opensearch-test-plugin/2.4.0.0/opensearch-test-plugin-2.4.0.0.zip
+```
+
+Following are the list of options that can be passed after `yarn opensearch snapshot` to configure the cluster snapshot.
+Options:
+
+      --license         Run with a 'oss', 'basic', or 'trial' license [default: oss]
+      --version         Version of OpenSearch to download [default: 3.0.0}]
+      --base-path       Path containing cache/installations [default: /home/ubuntu/OpenSearch-Dashboards/.opensearch]
+      --install-path    Installation path, defaults to 'source' within base-path
+      --data-archive    Path to zip or tarball containing an OpenSearch data directory to seed the cluster with.
+      --password        Sets password for opensearch user [default: changeme]
+      -E                Additional key=value settings to pass to OpenSearch
+      --download-only   Download the snapshot but don't actually start it
+      --ssl             Sets up SSL on OpenSearch
+      --P               OpenSearch plugin artifact URL to install it on the cluster. 
+
+```
+$ yarn opensearch snapshot --version 2.2.0 -E cluster.name=test -E path.data=/tmp/opensearch-data --P org.opensearch.plugin:test-plugin:2.2.0.0 --P file:/home/user/opensearch-test-plugin-2.2.0.0.zip
+```
+
 **Warning:** Starting the Dashboards instance before or during the initialization of the OpenSearch Server can cause Dashboards to sometimes misbehave. Ensure that the OpenSearch server instance is up and running first before starting up the Dashboards dev server from the next step.
 
 ### Run OpenSearch Dashboards

packages/osd-opensearch/src/cli_commands/snapshot.js

@@ -49,10 +49,13 @@ exports.help = (defaults = {}) => {
       -E                Additional key=value settings to pass to OpenSearch
       --download-only   Download the snapshot but don't actually start it
       --ssl             Sets up SSL on OpenSearch
+      --P               OpenSearch plugin artifact URL to install it on the cluster. We can use the flag multiple times
+                        to install multiple plugins on the cluster snapshot. The argument value can be url to zip file, maven coordinates of the plugin 
+                        or for local zip files, use file:<followed by the absolute or relative path to the plugin zip file>.
 
     Example:
 
-      opensearch snapshot --version 5.6.8 -E cluster.name=test -E path.data=/tmp/opensearch-data
+      opensearch snapshot --version 2.2.0 -E cluster.name=test -E path.data=/tmp/opensearch-data --P org.opensearch.plugin:test-plugin:2.2.0.0 --P file:/home/user/opensearch-test-plugin-2.2.0.0.zip
   `;
 };
 
@@ -64,6 +67,7 @@ exports.run = async (defaults = {}) => {
       installPath: 'install-path',
       dataArchive: 'data-archive',
       opensearchArgs: 'E',
+      opensearchPlugins: 'P',
     },
 
     string: ['version'],
@@ -83,6 +87,10 @@ exports.run = async (defaults = {}) => {
       await cluster.extractDataDirectory(installPath, options.dataArchive);
     }
 
+    if (options.opensearchPlugins) {
+      await cluster.installOpenSearchPlugins(installPath, options.opensearchPlugins);
+    }
+
     options.bundledJDK = true;
 
     await cluster.run(installPath, options);

packages/osd-opensearch/src/cluster.js

@@ -34,7 +34,7 @@ const execa = require('execa');
 const chalk = require('chalk');
 const path = require('path');
 const { downloadSnapshot, installSnapshot, installSource, installArchive } = require('./install');
-const { OPENSEARCH_BIN } = require('./paths');
+const { OPENSEARCH_BIN, OPENSEARCH_PLUGIN } = require('./paths');
 const { log: defaultLog, parseOpenSearchLog, extractConfigFiles, decompress } = require('./utils');
 const { createCliError } = require('./errors');
 const { promisify } = require('util');
@@ -170,6 +170,29 @@ exports.Cluster = class Cluster {
     this._log.indent(-4);
   }
 
+  /**
+   * Unpacks a tar or zip file containing the OpenSearch plugin directory for an
+   * OpenSearch cluster.
+   *
+   * @param {string} installPath
+   * @param {Array|string} opensearchPlugins Array or string of OpenSearch plugin(s) artifact url
+   */
+  async installOpenSearchPlugins(installPath, opensearchPluginsPath) {
+    if (opensearchPluginsPath) {
+      this._log.info(chalk.bold(`Downloading OpenSearch plugin(s) on the cluster snapshot`));
+      this._log.indent(4);
+      opensearchPluginsPath =
+        typeof opensearchPluginsPath === 'string' ? [opensearchPluginsPath] : opensearchPluginsPath;
+      // Run opensearch-plugin tool script to download OpenSearch plugin artifacts
+      for (const pluginPath of opensearchPluginsPath) {
+        this._log.info(`Installing OpenSearch Plugin from the path: ${pluginPath}`);
+        await execa(OPENSEARCH_PLUGIN, [`install`, `--batch`, pluginPath], { cwd: installPath });
+      }
+      this._log.info(`Plugin installation complete`);
+      this._log.indent(-4);
+    }
+  }
+
   /**
    * Starts OpenSearch and returns resolved promise once started
    *

packages/osd-opensearch/src/integration_tests/cluster.test.js

@@ -292,6 +292,33 @@ describe('#start(installPath)', () => {
   });
 });
 
+describe('#installOpenSearchPlugins()', () => {
+  it('install array of plugins on cluster snapshot', async () => {
+    const cluster = new Cluster({ log });
+    await cluster.installOpenSearchPlugins('foo', ['foo1', 'foo2']);
+    expect(execa).toHaveBeenCalledTimes(2);
+    expect(execa).toHaveBeenCalledWith('./bin/opensearch-plugin', ['install', '--batch', 'foo1'], {
+      cwd: 'foo',
+    });
+    expect(execa).toHaveBeenCalledWith('./bin/opensearch-plugin', ['install', '--batch', 'foo2'], {
+      cwd: 'foo',
+    });
+  });
+  it('installs single plugin on cluster snapshot', async () => {
+    const cluster = new Cluster({ log });
+    await cluster.installOpenSearchPlugins('foo', 'foo1');
+    expect(execa).toHaveBeenCalledTimes(1);
+    expect(execa).toHaveBeenCalledWith('./bin/opensearch-plugin', ['install', '--batch', 'foo1'], {
+      cwd: 'foo',
+    });
+  });
+  it('do not execute plugin installation script when no plugins in the param list', async () => {
+    const cluster = new Cluster({ log });
+    await cluster.installOpenSearchPlugins('foo');
+    expect(execa).toHaveBeenCalledTimes(0);
+  });
+});
+
 describe('#run()', () => {
   it('resolves when bin/opensearch exists with 0', async () => {
     mockOpenSearchBin({ exitCode: 0 });

packages/osd-opensearch/src/paths.js

@@ -44,3 +44,4 @@ exports.OPENSEARCH_BIN = maybeUseBat('bin/opensearch');
 exports.OPENSEARCH_CONFIG = 'config/opensearch.yml';
 
 exports.OPENSEARCH_KEYSTORE_BIN = maybeUseBat('./bin/opensearch-keystore');
+exports.OPENSEARCH_PLUGIN = maybeUseBat('./bin/opensearch-plugin');