feat: implement gitlab catalog backend (#44)

* First pass at Gitlab catalog backend plugin

* Include so its easier to setup

* TSC ignores on Gitlab client import usage

* Run prettier

* Filter users for quicker load

* Update plugins/gitlab-catalog-backend/src/GitlabUserProcessor.ts

Co-authored-by: Zach Hammer <zhammer@seatgeek.com>

* Update plugins/gitlab-catalog-backend/src/GitlabUserProcessor.test.ts

Co-authored-by: Zach Hammer <zhammer@seatgeek.com>

* Update plugins/gitlab-catalog-backend/src/GitlabUserProcessor.ts

Co-authored-by: Zach Hammer <zhammer@seatgeek.com>

* Last linting changes

* Prettier

* README update

* Typo

---------

Co-authored-by: Zach Hammer <zhammer@seatgeek.com>

README.md

@@ -14,6 +14,8 @@ This SeatGeek Backstage Plugins Collection offers the following plugins:
   - [plugins/slack-catalog-backend](plugins/slack-catalog-backend/)
 - **AWS Catalog**: the AWS Catalog Plugin offers catalog integrations with the AWS API.
   - [plugins/aws-catalog-backend](plugins/aws-catalog-backend)
+- **Gitlab Catalog**: the Gitlab Catalog Plugin offers catalog integrations with the Gitlab API.
+  - [plugins/gitlab-catalog-backend](plugins/gitlab-catalog-backend/)
 
 Each of the plugins contain instructions for installation and development within
 their respective locations.

app-config.yaml

@@ -47,3 +47,10 @@ catalog:
       target: ../../mock-catalog/templates/create-postgres-database.yaml
     - type: file
       target: ../../mock-catalog/templates/create-python-module.yaml
+
+slackCatalog:
+  token: ${SLACK_API_TOKEN_CATALOG}
+
+gitlabCatalog:
+  host: ${GITLAB_HOST_CATALOG}
+  token: ${GITLAB_TOKEN_CATALOG}

plugins/gitlab-catalog-backend/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

plugins/gitlab-catalog-backend/README.md

@@ -0,0 +1,51 @@
+# @seatgeek/backstage-plugin-gitlab-catalog-backend
+
+This plugin offers catalog integrations for ingesting data from the Slack API into the Software Catalog.
+
+[![npm latest version](https://img.shields.io/npm/v/@seatgeek/backstage-plugin-gitlab-catalog-backend/latest.svg)](https://www.npmjs.com/package/@seatgeek/backstage-plugin-gitlab-catalog-backend)
+
+## Installation
+
+Install the `@seatgeek/backstage-plugin-gitlab-catalog-backend` package in your backend package:
+
+```shell
+# From your Backstage root directory
+yarn add --cwd packages/backend @seatgeek/backstage-plugin-gitlab-catalog-backend
+```
+
+Add the following config to your `app-config.yaml`:
+
+```yml
+gitlabCatalog:
+  host: ${GITLAB_HOST_CATALOG} # defaults to https://gitlab.com
+  token: ${GITLAB_TOKEN_CATALOG}
+```
+
+Requires `read_user` scope with administrator level permissions to be able to view the email, see [List Users (for administrators)](https://docs.gitlab.com/ee/api/users.html#for-administrators).
+
+## Processors
+
+### `GitlabUserProcessor`
+
+Enriches existing `User` entities with information from Gitlab, notably the user's Gitlab ID, based on the user's `.profile.email`.
+
+#### Installation
+
+Add the following to your `packages/backend/catalog.ts`:
+
+```ts
+import { GitlabUserProcessor } from '@seatgeek/backstage-plugin-gitlab-catalog-backend';
+
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = CatalogBuilder.create(env);
+  builder.addProcessor(
+    // Add the gitlab user processor
+    GitlabUserProcessor.fromConfig(env.config, env.logger),
+  );
+  const { processingEngine, router } = await builder.build();
+  processingEngine.start();
+  return router;
+}
+```

plugins/gitlab-catalog-backend/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@seatgeek/backstage-plugin-gitlab-catalog-backend",
+  "version": "0.0.0-semantically-released",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin"
+  },
+  "scripts": {
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "clean": "backstage-cli package clean",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "^0.20.1",
+    "@backstage/catalog-model": "^1.4.3",
+    "@backstage/config": "^1.1.1",
+    "@backstage/plugin-catalog-common": "^1.0.20",
+    "@gitbeaker/rest": "^40.0.1",
+    "@types/express": "*",
+    "express": "^4.17.1",
+    "express-promise-router": "^4.1.0",
+    "node-fetch": "^2.6.7",
+    "winston": "^3.2.1",
+    "yn": "^4.0.0"
+  },
+  "devDependencies": {
+    "@backstage/cli": "^0.25.1",
+    "@types/supertest": "^2.0.12",
+    "msw": "^1.0.0",
+    "supertest": "^6.2.4"
+  },
+  "files": [
+    "dist"
+  ]
+}

plugins/gitlab-catalog-backend/src/GitlabUserProcessor.test.ts

@@ -0,0 +1,119 @@
+/*
+ * Copyright SeatGeek
+ * Licensed under the terms of the Apache-2.0 license. See LICENSE file in project root for terms.
+ */
+import { SystemEntity, UserEntity } from '@backstage/catalog-model';
+import { Gitlab } from '@gitbeaker/rest';
+import * as winston from 'winston';
+import { GitlabUserProcessor } from './GitlabUserProcessor';
+
+jest.mock('@gitbeaker/rest', () => {
+  return {
+    Gitlab: jest.fn().mockImplementation(() => {
+      return {
+        Users: {
+          all: jest.fn().mockResolvedValue([
+            {
+              id: 123,
+              email: 'rufus@seatgeek.com',
+            },
+            {
+              id: 999,
+              email: 'taylor@seatgeek.com',
+            },
+          ]),
+          // mock other methods as needed
+        },
+      };
+    }),
+  };
+});
+
+describe('GitlabUserProcessor', () => {
+  let processor: GitlabUserProcessor;
+  let mockGitlabClient: InstanceType<typeof Gitlab>;
+  let mockLogger: winston.Logger;
+
+  beforeEach(() => {
+    mockGitlabClient = new Gitlab({ token: `token` });
+    mockLogger = winston.createLogger({
+      transports: [
+        new winston.transports.Console({
+          format: winston.format.simple(),
+          level: 'debug',
+        }),
+      ],
+    });
+
+    processor = new GitlabUserProcessor(mockGitlabClient, mockLogger);
+
+    // Reset the mocks before each test
+    jest.clearAllMocks();
+  });
+
+  test('should add gitlab info', async () => {
+    const before: UserEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: {
+        name: 'rufus',
+      },
+      spec: {
+        profile: {
+          email: 'rufus@seatgeek.com',
+        },
+      },
+    };
+    const result = await processor.postProcessEntity(
+      before,
+      {} as any,
+      () => {},
+    );
+
+    const expected: UserEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: {
+        name: 'rufus',
+        annotations: {
+          'gitlab.com/user_id': '123',
+        },
+      },
+      spec: {
+        profile: {
+          email: 'rufus@seatgeek.com',
+        },
+      },
+    };
+
+    expect(result).toEqual(expected);
+    expect(mockGitlabClient.Users.all).toHaveBeenCalled();
+
+    // make sure that the slack users are only fetched once
+    await processor.postProcessEntity(before, {} as any, () => {});
+    expect(mockGitlabClient.Users.all).toHaveBeenCalledTimes(1);
+  });
+
+  test('should no op if not a user', async () => {
+    const before: SystemEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'System',
+      metadata: {
+        name: 'rufus',
+        annotations: {},
+      },
+      spec: {
+        owner: 'rufus',
+      },
+    };
+
+    const result = await processor.postProcessEntity(
+      before,
+      {} as any,
+      () => {},
+    );
+
+    expect(result).toEqual(before);
+    expect(mockGitlabClient.Users.all).not.toHaveBeenCalled();
+  });
+});

plugins/gitlab-catalog-backend/src/GitlabUserProcessor.ts

@@ -0,0 +1,148 @@
+/*
+ * Copyright SeatGeek
+ * Licensed under the terms of the Apache-2.0 license. See LICENSE file in project root for terms.
+ */
+import { Entity, isUserEntity } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { LocationSpec } from '@backstage/plugin-catalog-common';
+import type {
+  CatalogProcessor,
+  CatalogProcessorEmit,
+} from '@backstage/plugin-catalog-node';
+import { ExpandedUserSchema, Gitlab } from '@gitbeaker/rest';
+import { Logger } from 'winston';
+
+const GITLAB_PER_PAGE_LIMIT = 500;
+const GITLAB_DEFAULT_HOST = 'https://gitlab.com';
+
+/**
+ * The GitlabUserProcessor is used to enrich our User entities with information
+ * from Gitlab: notably the user's Gitlab ID.
+ *
+ * @public
+ */
+export class GitlabUserProcessor implements CatalogProcessor {
+  private readonly gitlab: InstanceType<typeof Gitlab>;
+  private readonly logger: Logger;
+  private cacheLoaded: boolean;
+  private userLookup: Map<string, ExpandedUserSchema>;
+  // guarantee that users are loaded only once
+  private loadUserPromise: Promise<Map<string, ExpandedUserSchema>> | null =
+    null;
+
+  private async fetchUsers(): Promise<Map<string, ExpandedUserSchema>> {
+    if (!this.gitlab) {
+      return new Map();
+    }
+
+    if (!this.cacheLoaded) {
+      // we use a shared promise to make sure that the load is only
+      // executed once even if `GitlabUserProcessor.postProcessEntity`
+      // is called "concurrently".
+      if (!this.loadUserPromise) {
+        this.loadUserPromise = this.loadUsers();
+      }
+
+      // Wait for the data to load
+      await this.loadUserPromise;
+    }
+
+    return this.userLookup;
+  }
+
+  private async loadUsers(): Promise<Map<string, ExpandedUserSchema>> {
+    this.logger.info('Loading gitlab users');
+
+    let members: ExpandedUserSchema[] = [];
+    try {
+      members = (await this.gitlab!.Users.all({
+        perPage: GITLAB_PER_PAGE_LIMIT,
+        active: true,
+        withoutProjectBots: true,
+      })) as ExpandedUserSchema[];
+    } catch (error) {
+      this.logger.error(`Error loading gitlab users: ${error}`);
+      return this.userLookup;
+    }
+
+    members.forEach(user => {
+      if (user.email) {
+        this.userLookup.set(user.email, user);
+      }
+    });
+
+    this.logger.info(`Loaded ${this.userLookup.size} gitlab users`);
+    this.cacheLoaded = true;
+
+    return this.userLookup;
+  }
+
+  static fromConfig(config: Config, logger: Logger): GitlabUserProcessor[] {
+    const token = config.getOptionalString('gitlabCatalog.token');
+    if (!token) {
+      logger.warn(
+        'No token provided for GitlabUserProcessor, skipping Gitlab user lookup',
+      );
+      return [];
+    }
+    let host = config.getOptionalString('gitlabCatalog.host');
+    if (!host) {
+      logger.info(
+        `No host provided for GitlabUserProcessor, defaulting to ${GITLAB_DEFAULT_HOST}`,
+      );
+      host = GITLAB_DEFAULT_HOST;
+    }
+    return [
+      new GitlabUserProcessor(
+        new Gitlab({
+          host: host,
+          token: token,
+        }),
+        logger,
+      ),
+    ];
+  }
+
+  constructor(gitlab: InstanceType<typeof Gitlab>, logger: Logger) {
+    this.gitlab = gitlab;
+    this.logger = logger;
+    this.userLookup = new Map();
+    this.cacheLoaded = false;
+  }
+
+  getProcessorName(): string {
+    return 'GitlabUserProcessor';
+  }
+
+  async postProcessEntity(
+    entity: Entity,
+    _location: LocationSpec,
+    _emit: CatalogProcessorEmit,
+  ): Promise<Entity> {
+    if (!isUserEntity(entity)) {
+      return entity;
+    }
+
+    const email = entity.spec.profile?.email;
+    if (!email) {
+      return entity;
+    }
+
+    const userLookup = await this.fetchUsers();
+    const gitlabUser = userLookup.get(email);
+    if (!gitlabUser) {
+      return entity;
+    }
+
+    if (!entity.metadata.annotations) {
+      entity.metadata.annotations = {};
+    }
+
+    if (gitlabUser.id) {
+      entity.metadata.annotations[`gitlab.com/user_id`] =
+        gitlabUser.id.toString();
+    }
+
+    return entity;
+  }
+}

plugins/gitlab-catalog-backend/src/index.ts

@@ -0,0 +1,5 @@
+/*
+ * Copyright SeatGeek
+ * Licensed under the terms of the Apache-2.0 license. See LICENSE file in project root for terms.
+ */
+export { GitlabUserProcessor } from './GitlabUserProcessor';