-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: implement gitlab catalog backend (#44)
* First pass at Gitlab catalog backend plugin * Include so its easier to setup * TSC ignores on Gitlab client import usage * Run prettier * Filter users for quicker load * Update plugins/gitlab-catalog-backend/src/GitlabUserProcessor.ts Co-authored-by: Zach Hammer <zhammer@seatgeek.com> * Update plugins/gitlab-catalog-backend/src/GitlabUserProcessor.test.ts Co-authored-by: Zach Hammer <zhammer@seatgeek.com> * Update plugins/gitlab-catalog-backend/src/GitlabUserProcessor.ts Co-authored-by: Zach Hammer <zhammer@seatgeek.com> * Last linting changes * Prettier * README update * Typo --------- Co-authored-by: Zach Hammer <zhammer@seatgeek.com>
- Loading branch information
Showing
8 changed files
with
379 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
# @seatgeek/backstage-plugin-gitlab-catalog-backend | ||
|
||
This plugin offers catalog integrations for ingesting data from the Slack API into the Software Catalog. | ||
|
||
[![npm latest version](https://img.shields.io/npm/v/@seatgeek/backstage-plugin-gitlab-catalog-backend/latest.svg)](https://www.npmjs.com/package/@seatgeek/backstage-plugin-gitlab-catalog-backend) | ||
|
||
## Installation | ||
|
||
Install the `@seatgeek/backstage-plugin-gitlab-catalog-backend` package in your backend package: | ||
|
||
```shell | ||
# From your Backstage root directory | ||
yarn add --cwd packages/backend @seatgeek/backstage-plugin-gitlab-catalog-backend | ||
``` | ||
|
||
Add the following config to your `app-config.yaml`: | ||
|
||
```yml | ||
gitlabCatalog: | ||
host: ${GITLAB_HOST_CATALOG} # defaults to https://gitlab.com | ||
token: ${GITLAB_TOKEN_CATALOG} | ||
``` | ||
Requires `read_user` scope with administrator level permissions to be able to view the email, see [List Users (for administrators)](https://docs.gitlab.com/ee/api/users.html#for-administrators). | ||
|
||
## Processors | ||
|
||
### `GitlabUserProcessor` | ||
|
||
Enriches existing `User` entities with information from Gitlab, notably the user's Gitlab ID, based on the user's `.profile.email`. | ||
|
||
#### Installation | ||
|
||
Add the following to your `packages/backend/catalog.ts`: | ||
|
||
```ts | ||
import { GitlabUserProcessor } from '@seatgeek/backstage-plugin-gitlab-catalog-backend'; | ||
export default async function createPlugin( | ||
env: PluginEnvironment, | ||
): Promise<Router> { | ||
const builder = CatalogBuilder.create(env); | ||
builder.addProcessor( | ||
// Add the gitlab user processor | ||
GitlabUserProcessor.fromConfig(env.config, env.logger), | ||
); | ||
const { processingEngine, router } = await builder.build(); | ||
processingEngine.start(); | ||
return router; | ||
} | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
{ | ||
"name": "@seatgeek/backstage-plugin-gitlab-catalog-backend", | ||
"version": "0.0.0-semantically-released", | ||
"main": "src/index.ts", | ||
"types": "src/index.ts", | ||
"license": "Apache-2.0", | ||
"publishConfig": { | ||
"access": "public", | ||
"main": "dist/index.cjs.js", | ||
"types": "dist/index.d.ts" | ||
}, | ||
"backstage": { | ||
"role": "backend-plugin" | ||
}, | ||
"scripts": { | ||
"start": "backstage-cli package start", | ||
"build": "backstage-cli package build", | ||
"lint": "backstage-cli package lint", | ||
"test": "backstage-cli package test", | ||
"clean": "backstage-cli package clean", | ||
"prepack": "backstage-cli package prepack", | ||
"postpack": "backstage-cli package postpack" | ||
}, | ||
"dependencies": { | ||
"@backstage/backend-common": "^0.20.1", | ||
"@backstage/catalog-model": "^1.4.3", | ||
"@backstage/config": "^1.1.1", | ||
"@backstage/plugin-catalog-common": "^1.0.20", | ||
"@gitbeaker/rest": "^40.0.1", | ||
"@types/express": "*", | ||
"express": "^4.17.1", | ||
"express-promise-router": "^4.1.0", | ||
"node-fetch": "^2.6.7", | ||
"winston": "^3.2.1", | ||
"yn": "^4.0.0" | ||
}, | ||
"devDependencies": { | ||
"@backstage/cli": "^0.25.1", | ||
"@types/supertest": "^2.0.12", | ||
"msw": "^1.0.0", | ||
"supertest": "^6.2.4" | ||
}, | ||
"files": [ | ||
"dist" | ||
] | ||
} |
119 changes: 119 additions & 0 deletions
119
plugins/gitlab-catalog-backend/src/GitlabUserProcessor.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,119 @@ | ||
/* | ||
* Copyright SeatGeek | ||
* Licensed under the terms of the Apache-2.0 license. See LICENSE file in project root for terms. | ||
*/ | ||
import { SystemEntity, UserEntity } from '@backstage/catalog-model'; | ||
import { Gitlab } from '@gitbeaker/rest'; | ||
import * as winston from 'winston'; | ||
import { GitlabUserProcessor } from './GitlabUserProcessor'; | ||
|
||
jest.mock('@gitbeaker/rest', () => { | ||
return { | ||
Gitlab: jest.fn().mockImplementation(() => { | ||
return { | ||
Users: { | ||
all: jest.fn().mockResolvedValue([ | ||
{ | ||
id: 123, | ||
email: 'rufus@seatgeek.com', | ||
}, | ||
{ | ||
id: 999, | ||
email: 'taylor@seatgeek.com', | ||
}, | ||
]), | ||
// mock other methods as needed | ||
}, | ||
}; | ||
}), | ||
}; | ||
}); | ||
|
||
describe('GitlabUserProcessor', () => { | ||
let processor: GitlabUserProcessor; | ||
let mockGitlabClient: InstanceType<typeof Gitlab>; | ||
let mockLogger: winston.Logger; | ||
|
||
beforeEach(() => { | ||
mockGitlabClient = new Gitlab({ token: `token` }); | ||
mockLogger = winston.createLogger({ | ||
transports: [ | ||
new winston.transports.Console({ | ||
format: winston.format.simple(), | ||
level: 'debug', | ||
}), | ||
], | ||
}); | ||
|
||
processor = new GitlabUserProcessor(mockGitlabClient, mockLogger); | ||
|
||
// Reset the mocks before each test | ||
jest.clearAllMocks(); | ||
}); | ||
|
||
test('should add gitlab info', async () => { | ||
const before: UserEntity = { | ||
apiVersion: 'backstage.io/v1alpha1', | ||
kind: 'User', | ||
metadata: { | ||
name: 'rufus', | ||
}, | ||
spec: { | ||
profile: { | ||
email: 'rufus@seatgeek.com', | ||
}, | ||
}, | ||
}; | ||
const result = await processor.postProcessEntity( | ||
before, | ||
{} as any, | ||
() => {}, | ||
); | ||
|
||
const expected: UserEntity = { | ||
apiVersion: 'backstage.io/v1alpha1', | ||
kind: 'User', | ||
metadata: { | ||
name: 'rufus', | ||
annotations: { | ||
'gitlab.com/user_id': '123', | ||
}, | ||
}, | ||
spec: { | ||
profile: { | ||
email: 'rufus@seatgeek.com', | ||
}, | ||
}, | ||
}; | ||
|
||
expect(result).toEqual(expected); | ||
expect(mockGitlabClient.Users.all).toHaveBeenCalled(); | ||
|
||
// make sure that the slack users are only fetched once | ||
await processor.postProcessEntity(before, {} as any, () => {}); | ||
expect(mockGitlabClient.Users.all).toHaveBeenCalledTimes(1); | ||
}); | ||
|
||
test('should no op if not a user', async () => { | ||
const before: SystemEntity = { | ||
apiVersion: 'backstage.io/v1alpha1', | ||
kind: 'System', | ||
metadata: { | ||
name: 'rufus', | ||
annotations: {}, | ||
}, | ||
spec: { | ||
owner: 'rufus', | ||
}, | ||
}; | ||
|
||
const result = await processor.postProcessEntity( | ||
before, | ||
{} as any, | ||
() => {}, | ||
); | ||
|
||
expect(result).toEqual(before); | ||
expect(mockGitlabClient.Users.all).not.toHaveBeenCalled(); | ||
}); | ||
}); |
148 changes: 148 additions & 0 deletions
148
plugins/gitlab-catalog-backend/src/GitlabUserProcessor.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,148 @@ | ||
/* | ||
* Copyright SeatGeek | ||
* Licensed under the terms of the Apache-2.0 license. See LICENSE file in project root for terms. | ||
*/ | ||
import { Entity, isUserEntity } from '@backstage/catalog-model'; | ||
import { Config } from '@backstage/config'; | ||
import { LocationSpec } from '@backstage/plugin-catalog-common'; | ||
import type { | ||
CatalogProcessor, | ||
CatalogProcessorEmit, | ||
} from '@backstage/plugin-catalog-node'; | ||
import { ExpandedUserSchema, Gitlab } from '@gitbeaker/rest'; | ||
import { Logger } from 'winston'; | ||
|
||
const GITLAB_PER_PAGE_LIMIT = 500; | ||
const GITLAB_DEFAULT_HOST = 'https://gitlab.com'; | ||
|
||
/** | ||
* The GitlabUserProcessor is used to enrich our User entities with information | ||
* from Gitlab: notably the user's Gitlab ID. | ||
* | ||
* @public | ||
*/ | ||
export class GitlabUserProcessor implements CatalogProcessor { | ||
private readonly gitlab: InstanceType<typeof Gitlab>; | ||
private readonly logger: Logger; | ||
private cacheLoaded: boolean; | ||
private userLookup: Map<string, ExpandedUserSchema>; | ||
// guarantee that users are loaded only once | ||
private loadUserPromise: Promise<Map<string, ExpandedUserSchema>> | null = | ||
null; | ||
|
||
private async fetchUsers(): Promise<Map<string, ExpandedUserSchema>> { | ||
if (!this.gitlab) { | ||
return new Map(); | ||
} | ||
|
||
if (!this.cacheLoaded) { | ||
// we use a shared promise to make sure that the load is only | ||
// executed once even if `GitlabUserProcessor.postProcessEntity` | ||
// is called "concurrently". | ||
if (!this.loadUserPromise) { | ||
this.loadUserPromise = this.loadUsers(); | ||
} | ||
|
||
// Wait for the data to load | ||
await this.loadUserPromise; | ||
} | ||
|
||
return this.userLookup; | ||
} | ||
|
||
private async loadUsers(): Promise<Map<string, ExpandedUserSchema>> { | ||
this.logger.info('Loading gitlab users'); | ||
|
||
let members: ExpandedUserSchema[] = []; | ||
try { | ||
members = (await this.gitlab!.Users.all({ | ||
perPage: GITLAB_PER_PAGE_LIMIT, | ||
active: true, | ||
withoutProjectBots: true, | ||
})) as ExpandedUserSchema[]; | ||
} catch (error) { | ||
this.logger.error(`Error loading gitlab users: ${error}`); | ||
return this.userLookup; | ||
} | ||
|
||
members.forEach(user => { | ||
if (user.email) { | ||
this.userLookup.set(user.email, user); | ||
} | ||
}); | ||
|
||
this.logger.info(`Loaded ${this.userLookup.size} gitlab users`); | ||
this.cacheLoaded = true; | ||
|
||
return this.userLookup; | ||
} | ||
|
||
static fromConfig(config: Config, logger: Logger): GitlabUserProcessor[] { | ||
const token = config.getOptionalString('gitlabCatalog.token'); | ||
if (!token) { | ||
logger.warn( | ||
'No token provided for GitlabUserProcessor, skipping Gitlab user lookup', | ||
); | ||
return []; | ||
} | ||
let host = config.getOptionalString('gitlabCatalog.host'); | ||
if (!host) { | ||
logger.info( | ||
`No host provided for GitlabUserProcessor, defaulting to ${GITLAB_DEFAULT_HOST}`, | ||
); | ||
host = GITLAB_DEFAULT_HOST; | ||
} | ||
return [ | ||
new GitlabUserProcessor( | ||
new Gitlab({ | ||
host: host, | ||
token: token, | ||
}), | ||
logger, | ||
), | ||
]; | ||
} | ||
|
||
constructor(gitlab: InstanceType<typeof Gitlab>, logger: Logger) { | ||
this.gitlab = gitlab; | ||
this.logger = logger; | ||
this.userLookup = new Map(); | ||
this.cacheLoaded = false; | ||
} | ||
|
||
getProcessorName(): string { | ||
return 'GitlabUserProcessor'; | ||
} | ||
|
||
async postProcessEntity( | ||
entity: Entity, | ||
_location: LocationSpec, | ||
_emit: CatalogProcessorEmit, | ||
): Promise<Entity> { | ||
if (!isUserEntity(entity)) { | ||
return entity; | ||
} | ||
|
||
const email = entity.spec.profile?.email; | ||
if (!email) { | ||
return entity; | ||
} | ||
|
||
const userLookup = await this.fetchUsers(); | ||
const gitlabUser = userLookup.get(email); | ||
if (!gitlabUser) { | ||
return entity; | ||
} | ||
|
||
if (!entity.metadata.annotations) { | ||
entity.metadata.annotations = {}; | ||
} | ||
|
||
if (gitlabUser.id) { | ||
entity.metadata.annotations[`gitlab.com/user_id`] = | ||
gitlabUser.id.toString(); | ||
} | ||
|
||
return entity; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
/* | ||
* Copyright SeatGeek | ||
* Licensed under the terms of the Apache-2.0 license. See LICENSE file in project root for terms. | ||
*/ | ||
export { GitlabUserProcessor } from './GitlabUserProcessor'; |