Add dynamic backend plugins support (janus-idp#522)

* Wire dynamic backend plugins in the backend app

Signed-off-by: David Festal <dfestal@redhat.com>

* Placeholder for built-in-container dynamic plugins

Signed-off-by: David Festal <dfestal@redhat.com>

* Add dynamic plugin installation script and steps

Signed-off-by: David Festal <dfestal@redhat.com>

* Add the requirements.txt file

Signed-off-by: David Festal <dfestal@redhat.com>

* Add changeset

Signed-off-by: David Festal <dfestal@redhat.com>

* Dynamic plugins preliminary documentation

Signed-off-by: David Festal <dfestal@redhat.com>

* Fix small error after review comment

Signed-off-by: David Festal <dfestal@redhat.com>

* PR review fixes

Signed-off-by: David Festal <dfestal@redhat.com>

---------

Signed-off-by: David Festal <dfestal@redhat.com>

.changeset/gentle-eels-rescue.md

@@ -0,0 +1,5 @@
+---
+'backend': minor
+---
+
+Add support for dynamic backend plugins.

docker/Dockerfile

@@ -65,6 +65,9 @@ RUN rm app-config.yaml && mv app-config.example.yaml app-config.yaml
 # hadolint ignore=DL3059
 RUN $YARN build --filter=backend
 
+# Build dynamic plugins
+RUN $YARN --cwd ./dynamic-plugins export-dynamic
+
 # Stage 4 - Build the actual backend image and install production dependencies
 FROM skeleton AS cleanup
 
@@ -90,7 +93,7 @@ RUN $YARN install --frozen-lockfile --production --network-timeout 600000
 FROM registry.access.redhat.com/ubi9/nodejs-18-minimal:1-74.1695740475 AS runner
 USER 0
 
-# Upstream only - install techdocs dependencies
+# Upstream only - install techdocs dependencies directly
 RUN microdnf update -y && \
   microdnf install -y python3 python3-pip && \
   pip3 install mkdocs-techdocs-core==1.* && \
@@ -106,6 +109,14 @@ WORKDIR $CONTAINER_SOURCE/
 # Upstream only
 COPY --from=cleanup --chown=1001:1001 $CONTAINER_SOURCE/ ./
 
+# Copy python script used to gather dynamic plugins
+COPY docker/install-dynamic-plugins.py ./
+RUN chmod a+r ./install-dynamic-plugins.py
+
+# Copy embedded dynamic plugins
+COPY --from=build $CONTAINER_SOURCE/dynamic-plugins/ ./dynamic-plugins/
+RUN chmod -R a+r ./dynamic-plugins/
+
 # The fix-permissions script is important when operating in environments that dynamically use a random UID at runtime, such as OpenShift.
 # The upstream backstage image does not account for this and it causes the container to fail at runtime.
 RUN fix-permissions ./

docker/brew.Dockerfile

@@ -28,7 +28,7 @@ USER 0
 # Install isolated-vm dependencies
 # hadolint ignore=DL3041
 RUN dnf install -y -q --allowerasing --nobest nodejs-devel nodejs-libs \
-  # already installed or installed as deps: 
+  # already installed or installed as deps:
   openssl openssl-devel ca-certificates make cmake cpp gcc gcc-c++ zlib zlib-devel brotli brotli-devel python3 nodejs-packaging && \
   dnf update -y && dnf clean all
 
@@ -77,7 +77,7 @@ COPY $REMOTE_SOURCES/upstream1/cachito.env \
 # hadolint ignore=SC1091
 RUN \
     # debug
-    # ls -l $CONTAINER_SOURCE/cachito.env; \ 
+    # ls -l $CONTAINER_SOURCE/cachito.env; \
     # load envs
     source $CONTAINER_SOURCE/cachito.env; \
     \
@@ -107,6 +107,10 @@ RUN git config --global --add safe.directory ./
 # hadolint ignore=DL3059
 RUN $YARN build --filter=backend
 
+# Build dynamic plugins
+# hadolint ignore=DL3059
+RUN $YARN --cwd ./dynamic-plugins export-dynamic
+
 # Stage 4 - Build the actual backend image and install production dependencies
 
 # Downstream only - files already exist, nothing to copy - debugging
@@ -128,7 +132,11 @@ RUN $YARN install --frozen-lockfile --production --network-timeout 600000
 FROM registry.redhat.io/ubi9/nodejs-18-minimal:1 AS runner
 USER 0
 
-# Downstream only - do not install techdocs dependencies (not required)
+# Downstream only - install techdocs dependencies using cachito sources
+RUN microdnf update -y && \
+  microdnf install -y python3 python3-pip && \
+  pip3 install mkdocs-techdocs-core==1.* && \
+  microdnf clean all
 
 # Env vars
 ENV YARN=./.yarn/releases/yarn-1.22.19.cjs
@@ -140,6 +148,14 @@ WORKDIR $CONTAINER_SOURCE/
 # Downstream only - copy from builder, not cleanup stage
 COPY --from=builder --chown=1001:1001 $CONTAINER_SOURCE/ ./
 
+# Copy python script used to gather dynamic plugins
+COPY docker/install-dynamic-plugins.py ./
+RUN chmod a+r ./install-dynamic-plugins.py
+
+# Copy embedded dynamic plugins
+COPY --from=builder $CONTAINER_SOURCE/dynamic-plugins/ ./dynamic-plugins/
+RUN chmod -R a+r ./dynamic-plugins/
+
 # The fix-permissions script is important when operating in environments that dynamically use a random UID at runtime, such as OpenShift.
 # The upstream backstage image does not account for this and it causes the container to fail at runtime.
 RUN fix-permissions ./

docker/install-dynamic-plugins.py

@@ -0,0 +1,121 @@
+import os
+import sys
+import yaml
+import tarfile
+import shutil
+import subprocess
+
+class InstallException(Exception):
+    """Exception class from which every exception in this library will derive."""
+    pass
+
+def merge(source, destination):
+    for key, value in source.items():
+        if isinstance(value, dict):
+            # get node or create one
+            node = destination.setdefault(key, {})
+            merge(value, node)
+        else:
+            # if key exists in destination trigger an error
+            if key in destination:
+                raise InstallException('Config key ' + key + ' defined for 2 dynamic plugins')
+
+            destination[key] = value
+
+    return destination
+
+def main():
+    dynamicPluginsRoot = sys.argv[1]
+    maxEntrySize = int(os.environ.get('MAX_ENTRY_SIZE', 10000000))
+
+    dynamicPluginsFile = os.path.join(dynamicPluginsRoot, 'dynamic-plugins.yaml')
+    dynamicPluginsGlobalConfigFile = os.path.join(dynamicPluginsRoot, 'app-config.dynamic-plugins.yaml')
+
+    # test if file dynamic-plugins.yaml exists
+    if not os.path.isfile(dynamicPluginsFile):
+        print(f'No {dynamicPluginsFile} file found. Skipping dynamic plugins installation.')
+        with open(dynamicPluginsGlobalConfigFile, 'w') as file:
+            file.write('')
+            file.close()
+        exit(0)
+
+    with open(dynamicPluginsFile, 'r') as file:
+        plugins = yaml.safe_load(file)
+
+    if plugins == '' or plugins is None:
+        print(f'{dynamicPluginsFile} file is empty. Skipping dynamic plugins installation.')
+        with open(dynamicPluginsGlobalConfigFile, 'w') as file:
+            file.write('')
+            file.close()
+        exit(0)
+
+    # test that plugins is a list
+    if not isinstance(plugins, list):
+        raise InstallException(f'{dynamicPluginsFile} content must be a list')
+
+    globalConfig = {
+      'dynamicPlugins': {
+            'rootDirectory': 'dynamic-plugins-root'
+      }
+    }
+
+    # iterate through the list of plugins
+    for plugin in plugins:
+        package = plugin['package']
+        if package.startswith('./'):
+            package = os.path.join(os.getcwd(), package[2:])
+
+        print('\n======= Installing dynamic plugin', package, flush=True)
+
+        print('\t==> Grabbing package archive through `npm pack`', flush=True)
+        completed = subprocess.run(['npm', 'pack', package], capture_output=True, cwd=dynamicPluginsRoot)
+        if completed.returncode != 0:
+            raise InstallException(f'Error while installing plugin { package } with \'npm pack\' : ' + completed.stderr.decode('utf-8'))
+
+        archive = os.path.join(dynamicPluginsRoot, completed.stdout.decode('utf-8').strip())
+        directory = archive.replace('.tgz', '')
+
+        print('\t==> Removing previous plugin directory', directory, flush=True)
+        shutil.rmtree(directory, ignore_errors=True, onerror=None)
+        os.mkdir(directory)
+
+        print('\t==> Extracting package archive', archive, flush=True)
+        file = tarfile.open(archive, 'r:gz') # NOSONAR
+        # extract the archive content but take care of zip bombs
+        for member in file.getmembers():
+            if member.isreg():
+                if not member.name.startswith('package/'):
+                    raise InstallException("NPM package archive archive does not start with 'package/' as it should: " + member.name)
+
+                if member.size > maxEntrySize:
+                    raise InstallException('Zip bomb detected in ' + member.name)
+
+                # Remove the `package/` prefix from the file name
+                member.name = member.name[8:]
+                file.extract(member, path=directory)
+            elif member.isdir():
+                print('\t\tSkipping directory entry', member.name, flush=True)
+            else:
+                raise InstallException('NPM package archive contains a non regular file: ' + member.name)
+
+        file.close()
+
+        print('\t==> Removing package archive', archive, flush=True)
+        os.remove(archive)
+
+        if 'pluginConfig' not in plugin:
+          print('\t==> Successfully installed dynamic plugin', package, flush=True)
+          continue
+
+        # if some plugin configuration is defined, merge it with the global configuration
+
+        print('\t==> Merging plugin-specific configuration', flush=True)
+        config = plugin['pluginConfig']
+        if config is not None and isinstance(config, dict):
+                merge(config, globalConfig)
+
+        print('\t==> Successfully installed dynamic plugin', package, flush=True)
+
+    yaml.safe_dump(globalConfig, open(dynamicPluginsGlobalConfigFile, 'w'))
+
+main()

docker/requirements.txt

@@ -0,0 +1 @@
+PyYAML==6.0.1

dynamic-plugins/.eslintignore

@@ -0,0 +1,4 @@
+**/node_modules/**
+**/dist/**
+**/dist-dynamic/**
+**/templates/**

dynamic-plugins/.eslintrc

@@ -0,0 +1,6 @@
+{
+  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  "parser": "@typescript-eslint/parser",
+  "plugins": ["@typescript-eslint"],
+  "root": true
+}

dynamic-plugins/package.json

@@ -0,0 +1,6 @@
+{
+  "private": true,
+  "scripts": {
+    "export-dynamic": "for D in $(ls -d */ 2>/dev/null); do yarn --cwd \"${D}\" install --frozen-lockfile && yarn --cwd \"${D}\" export-dynamic && rm -Rf ${D}/node_modules ${D}/.yarn ${D}/dist; done"
+  }
+}

packages/backend/package.json

@@ -49,6 +49,9 @@
     "@roadiehq/backstage-plugin-argo-cd-backend": "2.11.3",
     "@roadiehq/scaffolder-backend-argocd": "1.1.17",
     "@roadiehq/scaffolder-backend-module-utils": "1.10.4",
+    "@backstage/backend-plugin-manager": "npm:@janus-idp/backend-plugin-manager@0.0.4-janus.0",
+    "@backstage/plugin-events-backend": "0.2.13",
+    "@backstage/plugin-events-node": "0.2.13",
     "app": "*",
     "better-sqlite3": "8.6.0",
     "dockerode": "3.3.5",

packages/backend/src/index.ts

@@ -29,6 +29,7 @@ import argocd from './plugins/argocd';
 import auth from './plugins/auth';
 import azureDevOps from './plugins/azure-devops';
 import catalog from './plugins/catalog';
+import events from './plugins/events';
 import gitlab from './plugins/gitlab';
 import jenkins from './plugins/jenkins';
 import kubernetes from './plugins/kubernetes';
@@ -39,18 +40,24 @@ import scaffolder from './plugins/scaffolder';
 import search from './plugins/search';
 import sonarqube from './plugins/sonarqube';
 import techdocs from './plugins/techdocs';
-import { PluginEnvironment } from './types';
 import { metricsHandler } from './metrics';
 import { RequestHandler } from 'express';
+import {
+  PluginManager,
+  BackendPluginProvider,
+  LegacyPluginEnvironment as PluginEnvironment,
+} from '@backstage/backend-plugin-manager';
+import { DefaultEventBroker } from '@backstage/plugin-events-backend';
 
-function makeCreateEnv(config: Config) {
+function makeCreateEnv(config: Config, pluginProvider: BackendPluginProvider) {
   const root = getRootLogger();
   const reader = UrlReaders.default({ logger: root, config });
   const discovery = HostDiscovery.fromConfig(config);
   const cacheManager = CacheManager.fromConfig(config);
   const databaseManager = DatabaseManager.fromConfig(config, { logger: root });
   const tokenManager = ServerTokenManager.fromConfig(config, { logger: root });
   const taskScheduler = TaskScheduler.fromConfig(config);
+  const eventBroker = new DefaultEventBroker(root);
 
   const identity = DefaultIdentityClient.create({
     discovery,
@@ -78,6 +85,8 @@ function makeCreateEnv(config: Config) {
       scheduler,
       permissions,
       identity,
+      eventBroker,
+      pluginProvider,
     };
   };
 }
@@ -146,11 +155,13 @@ async function addRouter(args: AddRouter | AddRouterOptional): Promise<void> {
 }
 
 async function main() {
+  const logger = getRootLogger();
   const config = await loadBackendConfig({
     argv: process.argv,
-    logger: getRootLogger(),
+    logger,
   });
-  const createEnv = makeCreateEnv(config);
+  const pluginManager = await PluginManager.fromConfig(config, logger);
+  const createEnv = makeCreateEnv(config, pluginManager);
 
   const appEnv = useHotMemoize(module, () => createEnv('app'));
 
@@ -167,6 +178,7 @@ async function main() {
     createEnv,
     router: scaffolder,
   });
+  await addPlugin({ plugin: 'events', apiRouter, createEnv, router: events });
 
   // Optional plugins
   await addPlugin({
@@ -243,6 +255,21 @@ async function main() {
     isOptional: true,
   });
 
+  for (const plugin of pluginManager.backendPlugins()) {
+    if (plugin.installer.kind === 'legacy') {
+      const pluginRouter = plugin.installer.router;
+      if (pluginRouter !== undefined) {
+        const pluginEnv = useHotMemoize(module, () =>
+          createEnv(pluginRouter.pluginID),
+        );
+        apiRouter.use(
+          `/${pluginRouter.pluginID}`,
+          await pluginRouter.createPlugin(pluginEnv),
+        );
+      }
+    }
+  }
+
   // Add backends ABOVE this line; this 404 handler is the catch-all fallback
   apiRouter.use(notFoundHandler());
 

packages/backend/src/plugins/catalog.ts

@@ -11,7 +11,10 @@ import { KeycloakOrgEntityProvider } from '@janus-idp/backstage-plugin-keycloak-
 import { ManagedClusterProvider } from '@janus-idp/backstage-plugin-ocm-backend';
 import { AapResourceEntityProvider } from '@janus-idp/backstage-plugin-aap-backend';
 import { Router } from 'express';
-import { PluginEnvironment } from '../types';
+import {
+  LegacyBackendPluginInstaller,
+  LegacyPluginEnvironment as PluginEnvironment,
+} from '@backstage/backend-plugin-manager';
 
 export default async function createPlugin(
   env: PluginEnvironment,
@@ -124,6 +127,17 @@ export default async function createPlugin(
   builder.setPlaceholderResolver('asyncapi', jsonSchemaRefPlaceholderResolver);
 
   builder.addProcessor(new ScaffolderEntitiesProcessor());
+
+  env.pluginProvider
+    .backendPlugins()
+    .map(p => p.installer)
+    .filter((i): i is LegacyBackendPluginInstaller => i.kind === 'legacy')
+    .forEach(i => {
+      if (i.catalog) {
+        i.catalog(builder, env);
+      }
+    });
+
   const { processingEngine, router } = await builder.build();
   await processingEngine.start();