Add support for reading groups and users from the Microsoft Graph API. (

#3293)

* Add support for reading groups and users from the Microsoft Graph API.

* Limit amount of parallel requests

* Add helper for paging in odata collections

* Add tests for the microsoft graph reader

* Output the correct relations between groups and users

.changeset/curly-yaks-invite.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Add support for reading groups and users from the Microsoft Graph API.

app-config.yaml

@@ -140,6 +140,19 @@ catalog:
       #       dn: ou=access,ou=groups,ou=example,dc=example,dc=net
       #       options:
       #         filter: (&(objectClass=some-group-class)(!(groupType=email)))
+    microsoftGraphOrg:
+      ### Example for how to add your Microsoft Graph tenant
+      #providers:
+      #  - target: https://graph.microsoft.com/v1.0/
+      #    authority: https://login.microsoftonline.com/
+      #    tenantId:
+      #      $env: MICROSOFT_GRAPH_TENANT_ID
+      #    clientId:
+      #      $env: MICROSOFT_GRAPH_CLIENT_ID
+      #    clientSecret:
+      #      $env: MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN
+      #    userFilter: accountEnabled eq true and userType eq 'member'
+      #    groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified')
 
   locations:
     # Backstage example components

docs/features/software-catalog/well-known-annotations.md

@@ -190,9 +190,25 @@ metadata:
 ```
 
 The value of these annotations are the corresponding attributes that were found
-when ingestion the entity from LDAP. Not all of them may be present, depending
+when ingesting the entity from LDAP. Not all of them may be present, depending
 on what attributes that the server presented at ingestion time.
 
+### graph.microsoft.com/tenant-id, graph.microsoft.com/group-id, graph.microsoft.com/user-id
+
+```yaml
+# Example:
+metadata:
+  annotations:
+    graph.microsoft.com/tenant-id: 6902611b-ffc1-463f-8af3-4d5285dc057b
+    graph.microsoft.com/group-id: c57e8ba2-6cc4-1039-9ebc-d5f241a7ca21
+    graph.microsoft.com/user-id: 2de244b5-104b-4e8f-a3b8-dce3c31e54b6
+```
+
+The value of these annotations are the corresponding attributes that were found
+when ingesting the entity from the Microsoft Graph API. Not all of them may be
+present, depending on what attributes that the server presented at ingestion
+time.
+
 ### sonarqube.org/project-key
 
 ```yaml

plugins/catalog-backend/package.json

@@ -20,6 +20,7 @@
     "clean": "backstage-cli clean"
   },
   "dependencies": {
+    "@azure/msal-node": "^1.0.0-alpha.8",
     "@backstage/backend-common": "^0.3.0",
     "@backstage/catalog-model": "^0.2.0",
     "@backstage/config": "^0.1.1",
@@ -37,6 +38,7 @@
     "lodash": "^4.17.15",
     "morgan": "^1.10.0",
     "p-limit": "^3.0.2",
+    "qs": "^6.9.4",
     "sqlite3": "^5.0.0",
     "uuid": "^8.0.0",
     "winston": "^3.2.1",

plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { Logger } from 'winston';
+import {
+  MicrosoftGraphClient,
+  MicrosoftGraphProviderConfig,
+  readMicrosoftGraphConfig,
+  readMicrosoftGraphOrg,
+} from './microsoftGraph';
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+
+/**
+ * Extracts teams and users out of an LDAP server.
+ */
+export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor {
+  private readonly providers: MicrosoftGraphProviderConfig[];
+  private readonly logger: Logger;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg');
+    return new MicrosoftGraphOrgReaderProcessor({
+      ...options,
+      providers: c ? readMicrosoftGraphConfig(c) : [],
+    });
+  }
+
+  constructor(options: {
+    providers: MicrosoftGraphProviderConfig[];
+    logger: Logger;
+  }) {
+    this.providers = options.providers;
+    this.logger = options.logger;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'microsoft-graph-org') {
+      return false;
+    }
+
+    const provider = this.providers.find(p =>
+      location.target.startsWith(p.target),
+    );
+    if (!provider) {
+      throw new Error(
+        `There is no Microsoft Graph Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`,
+      );
+    }
+
+    // Read out all of the raw data
+    const startTimestamp = Date.now();
+    this.logger.info('Reading Microsoft Graph users and groups');
+
+    // We create a client each time as we need one that matches the specific provider
+    const client = MicrosoftGraphClient.create(provider);
+    const { users, groups } = await readMicrosoftGraphOrg(
+      client,
+      provider.tenantId,
+      {
+        userFilter: provider.userFilter,
+        groupFilter: provider.groupFilter,
+      },
+    );
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${users.length} users and ${groups.length} groups from Microsoft Graph in ${duration} seconds`,
+    );
+
+    // Done!
+    for (const group of groups) {
+      emit(results.entity(location, group));
+    }
+    for (const user of users) {
+      emit(results.entity(location, user));
+    }
+
+    return true;
+  }
+}

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -27,6 +27,7 @@ export { FileReaderProcessor } from './FileReaderProcessor';
 export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
 export { OwnerRelationProcessor } from './OwnerRelationProcessor';
 export { LocationRefProcessor } from './LocationEntityProcessor';
+export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';
 export { PlaceholderProcessor } from './PlaceholderProcessor';
 export type { PlaceholderResolver } from './PlaceholderProcessor';
 export { StaticLocationProcessor } from './StaticLocationProcessor';