feat(cmd-api-server): fully dynamic plugin imports via config file/en…

…v/CLI It leverages the PluginRegistry to its full potential. Had to slightly refactor how the plugin factories are exported from the plugin packages but it is a non-breaking change. Also: decoupled the consortium web service plugins tests from the plugin itself (e.g. moved it to a separate package) so that circular package dependencies are avoided where with cmd-api-server Signed-off-by: Peter Somogyvari <peter.somogyvari@accenture.com>
petermetz · May 19, 2020 · fe396c9 · fe396c9
1 parent 2e19ba8
commit fe396c9
Show file tree

Hide file tree

Showing 23 changed files with 2,822 additions and 47 deletions.
diff --git a/packages/cactus-cmd-api-server/package-lock.json b/packages/cactus-cmd-api-server/package-lock.json
diff --git a/packages/cactus-cmd-api-server/package.json b/packages/cactus-cmd-api-server/package.json
@@ -58,9 +58,11 @@
     "@hyperledger/cactus-core-api": "0.2.0",
     "@hyperledger/cactus-plugin-keychain-memory": "0.2.0",
     "@hyperledger/cactus-plugin-kv-storage-memory": "0.2.0",
+    "@hyperledger/cactus-plugin-web-service-consortium": "0.2.0",
     "body-parser": "1.19.0",
     "compression": "1.7.4",
-    "convict": "5.2.0",
+    "convict": "6.0.0",
+    "convict-format-with-validator": "6.0.0",
     "cors": "2.8.5",
     "express": "4.17.1",
     "express-openapi-validator": "3.10.0",
@@ -74,7 +76,7 @@
   },
   "devDependencies": {
     "@types/compression": "1.7.0",
-    "@types/convict": "5.2.0",
+    "@types/convict": "5.2.1",
     "@types/cors": "2.8.6",
     "@types/express": "4.17.6",
     "@types/joi": "14.3.4",

diff --git a/packages/cactus-cmd-api-server/src/main/typescript/api-server.ts b/packages/cactus-cmd-api-server/src/main/typescript/api-server.ts
@@ -1,6 +1,5 @@
 import path from "path";
 import { Server } from "http";
-import { Config } from "convict";
 import express, {
   Express,
   Request,
@@ -13,10 +12,8 @@ import compression from "compression";
 import bodyParser from "body-parser";
 import cors, { CorsOptions } from "cors";
 import {
-  IPluginKVStorage,
   PluginFactory,
   ICactusPlugin,
-  PluginAspect,
   isIPluginWebService,
   IPluginWebService,
   PluginRegistry,
@@ -73,8 +70,10 @@ export class ApiServer {
   public async getOrInitPluginRegistry(): Promise<PluginRegistry> {
     if (!this.pluginRegistry) {
       if (!this.options.pluginRegistry) {
+        this.log.info(`getOrInitPluginRegistry() initializing a new one...`);
         this.pluginRegistry = await this.initPluginRegistry();
       } else {
+        this.log.info(`getOrInitPluginRegistry() re-using injected one...`);
         this.pluginRegistry = this.options.pluginRegistry;
       }
     }
@@ -83,16 +82,18 @@ export class ApiServer {
 
   public async initPluginRegistry(): Promise<PluginRegistry> {
     const registry = new PluginRegistry({ plugins: [] });
-    // FIXME load the plugins here:
-
-    {
-      const storagePluginPackage = this.options.config.storagePluginPackage;
-      const { PluginFactoryKVStorage } = await import(storagePluginPackage);
-      const storagePluginOptionsJson = this.options.config
-        .storagePluginOptionsJson;
-      const storagePluginOptions = JSON.parse(storagePluginOptionsJson);
-      const pluginFactory = new PluginFactoryKVStorage();
-      const plugin = await pluginFactory.create(storagePluginOptions);
+
+    this.log.info(`Instantiated empty registry, invoking plugin factories...`);
+    for (const pluginImport of this.options.config.plugins) {
+      const { packageName, options } = pluginImport;
+      this.log.info(`Creating plugin from package: ${packageName}`, options);
+      const pluginOptions = { ...options, pluginRegistry: registry };
+      const { createPluginFactory } = await import(packageName);
+      const pluginFactory: PluginFactory<
+        ICactusPlugin,
+        any
+      > = await createPluginFactory();
+      const plugin = await pluginFactory.create(pluginOptions);
       registry.add(plugin);
     }
 

diff --git a/packages/cactus-cmd-api-server/src/main/typescript/config/config-service.ts b/packages/cactus-cmd-api-server/src/main/typescript/config/config-service.ts
@@ -1,12 +1,23 @@
 import { randomBytes } from "crypto";
 import convict, { Schema, Config, SchemaObj } from "convict";
+import { ipaddress } from "convict-format-with-validator";
 import secp256k1 from "secp256k1";
 import { v4 as uuidV4 } from "uuid";
 import {
   LoggerProvider,
   Logger,
   LogLevelDesc,
 } from "@hyperledger/cactus-common";
+import { PluginAspect } from "@hyperledger/cactus-core-api";
+import { FORMAT_PLUGIN_ARRAY } from "./convict-plugin-array-format";
+
+convict.addFormat(FORMAT_PLUGIN_ARRAY);
+convict.addFormat(ipaddress);
+
+export interface IPluginImport {
+  packageName: string;
+  options?: any;
+}
 
 export interface ICactusApiServerOptions {
   configFile: string;
@@ -18,6 +29,7 @@ export interface ICactusApiServerOptions {
   apiHost: string;
   apiPort: number;
   apiCorsDomainCsv: string;
+  plugins: IPluginImport[];
   storagePluginPackage: string;
   storagePluginOptionsJson: string;
   keychainPluginPackage: string;
@@ -60,6 +72,18 @@ export class ConfigService {
 
   private static getConfigSchema(): Schema<ICactusApiServerOptions> {
     return {
+      plugins: {
+        doc: "A collection of plugins to load at runtime.",
+        format: "plugin-array",
+        default: [],
+        env: "PLUGINS",
+        arg: "plugins",
+        pluginSchema: {
+          aspect: "*",
+          packageName: "*",
+          options: {},
+        },
+      } as any,
       configFile: {
         doc:
           "The path to a config file that holds the configuration itself which will be parsed and validated.",
@@ -272,6 +296,22 @@ export class ConfigService {
     const publicKey = Buffer.from(publicKeyBytes).toString("hex");
 
     return {
+      plugins: [
+        {
+          packageName: "@hyperledger/cactus-plugin-kv-storage-memory",
+          options: {},
+        },
+        {
+          packageName: "@hyperledger/cactus-plugin-keychain-memory",
+          options: {},
+        },
+        {
+          packageName: "@hyperledger/cactus-plugin-web-service-consortium",
+          options: {
+            privateKey: "some-fake-key",
+          },
+        },
+      ],
       configFile: ".config.json",
       cactusNodeId: uuidV4(),
       logLevel: "debug",

diff --git a/packages/cactus-cmd-api-server/src/main/typescript/config/convict-plugin-array-format.ts b/packages/cactus-cmd-api-server/src/main/typescript/config/convict-plugin-array-format.ts
@@ -0,0 +1,24 @@
+import convict from "convict";
+
+export const validate = (sources: any[], schema?: any) => {
+  if (!Array.isArray(sources)) {
+    throw new Error("must be of type Array");
+  }
+
+  for (const source of sources) {
+    convict(schema.pluginSchema).load(source).validate();
+  }
+};
+
+export const coerce = (value: string) => {
+  // CLI sends comman separated objects as a JSON string without the array square brackets
+  // ENV sends the proper array that is valid JSON so we have to detect and handle both cases
+  const isJsonArray = value.startsWith("[") && value.endsWith("]");
+  return isJsonArray ? JSON.parse(value) : JSON.parse(`[${value}]`);
+};
+
+export const FORMAT_PLUGIN_ARRAY = {
+  name: "plugin-array",
+  validate,
+  coerce,
+};
diff --git a/packages/cactus-plugin-keychain-memory/src/main/typescript/public-api.ts b/packages/cactus-plugin-keychain-memory/src/main/typescript/public-api.ts
@@ -1,2 +1,10 @@
 export { PluginKeychainMemory } from "./plugin-keychain-memory";
 export { PluginFactoryKeychain } from "./plugin-factory-keychain";
+
+import { PluginFactoryKeychain } from "./plugin-factory-keychain";
+
+export async function createPluginFactory(
+  options?: any
+): Promise<PluginFactoryKeychain> {
+  return new PluginFactoryKeychain();
+}
diff --git a/packages/cactus-plugin-kv-storage-memory/src/main/typescript/public-api.ts b/packages/cactus-plugin-kv-storage-memory/src/main/typescript/public-api.ts
@@ -1,2 +1,10 @@
 export { PluginKVStorageMemory } from "./plugin-kv-storage-memory";
 export { PluginFactoryKVStorage } from "./plugin-factory-kv-storage";
+
+import { PluginFactoryKVStorage } from "./plugin-factory-kv-storage";
+
+export async function createPluginFactory(
+  options?: any
+): Promise<PluginFactoryKVStorage> {
+  return new PluginFactoryKVStorage();
+}
diff --git a/packages/cactus-plugin-ledger-connector-besu/src/main/typescript/public-api.ts b/packages/cactus-plugin-ledger-connector-besu/src/main/typescript/public-api.ts
@@ -1,2 +1,10 @@
 export { PluginLedgerConnectorBesu } from "./plugin-ledger-connector-besu";
 export { PluginFactoryLedgerConnector } from "./plugin-factory-ledger-connector";
+
+import { PluginFactoryLedgerConnector } from "./plugin-factory-ledger-connector";
+
+export async function createPluginFactory(
+  options?: any
+): Promise<PluginFactoryLedgerConnector> {
+  return new PluginFactoryLedgerConnector();
+}
diff --git a/packages/cactus-plugin-ledger-connector-quorum/src/main/typescript/public-api.ts b/packages/cactus-plugin-ledger-connector-quorum/src/main/typescript/public-api.ts
@@ -6,3 +6,10 @@ export {
 } from "./plugin-ledger-connector-quorum";
 export { PluginFactoryLedgerConnector } from "./plugin-factory-ledger-connector";
 export { Contract as Web3EthContract } from "web3-eth-contract";
+
+import { PluginFactoryLedgerConnector } from "./plugin-factory-ledger-connector";
+export async function createPluginFactory(
+  options?: any
+): Promise<PluginFactoryLedgerConnector> {
+  return new PluginFactoryLedgerConnector();
+}
diff --git a/packages/cactus-plugin-web-service-consortium/package.json b/packages/cactus-plugin-web-service-consortium/package.json
@@ -68,7 +68,6 @@
     "typescript-optional": "2.0.1"
   },
   "devDependencies": {
-    "@hyperledger/cactus-cmd-api-server": "0.2.0",
     "@hyperledger/cactus-plugin-kv-storage-memory": "0.2.0",
     "@hyperledger/cactus-sdk": "0.2.0",
     "@types/express": "4.17.6",

diff --git a/packages/cactus-plugin-web-service-consortium/src/main/typescript/public-api.ts b/packages/cactus-plugin-web-service-consortium/src/main/typescript/public-api.ts
@@ -5,3 +5,10 @@ export {
 } from "./plugin-web-service-consortium";
 export { PluginFactoryWebService } from "./plugin-factory-web-service-consortium";
 export * from "./generated/openapi/typescript-axios/index";
+
+import { PluginFactoryWebService } from "./plugin-factory-web-service-consortium";
+export async function createPluginFactory(
+  options?: any
+): Promise<PluginFactoryWebService> {
+  return new PluginFactoryWebService();
+}
diff --git a/...gration/plugin-ledger-connector-quorum/deploy-contract/deploy-contract-via-web-service.ts b/...gration/plugin-ledger-connector-quorum/deploy-contract/deploy-contract-via-web-service.ts
@@ -1,6 +1,6 @@
 // tslint:disable-next-line: no-var-requires
 const tap = require("tap");
-import axios, { AxiosPromise, AxiosInstance, AxiosResponse } from "axios";
+import axios, { AxiosResponse } from "axios";
 import {
   QuorumTestLedger,
   IQuorumGenesisOptions,

diff --git a/packages/cactus-test-plugin-web-service-consortium/README.md b/packages/cactus-test-plugin-web-service-consortium/README.md
@@ -0,0 +1,14 @@
+# `@hyperledger/cactus-test-plugin-web-service-consortium`
+
+
+## Usage
+
+```
+// TODO: DEMONSTRATE API
+```
+
+## FAQ
+
+### **What is a dedicatd test package for?**
+
+This is a dedicated test package meaning that it verifies the integration between two packages that are somehow dependent on each other and therefore these tests cannot be added properly in the child package due to circular dependency issues and it would not be fitting to add it in the parent because the child package's tests should not be held by the parent as a matter of principle.