Merge pull request #3874 from backjo/feature/AwsOrganizationProcessor

feat: add catalog ingestion processor for indexing AWS accounts

.changeset/new-horses-protect.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+Add processor for ingesting AWS accounts from AWS Organizations

plugins/catalog-backend/package.json

@@ -36,6 +36,7 @@
     "@octokit/graphql": "^4.5.6",
     "@types/express": "^4.17.6",
     "@types/ldapjs": "^1.0.9",
+    "aws-sdk": "^2.817.0",
     "codeowners-utils": "^1.0.2",
     "core-js": "^3.6.5",
     "cross-fetch": "^3.0.6",

plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts

@@ -0,0 +1,122 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
+
+describe('AwsOrganizationCloudAccountProcessor', () => {
+  describe('readLocation', () => {
+    const processor = new AwsOrganizationCloudAccountProcessor();
+    const location = { type: 'aws-cloud-accounts', target: '' };
+    const emit = jest.fn();
+    const listAccounts = jest.fn();
+
+    processor.organizations.listAccounts = listAccounts;
+    afterEach(() => jest.resetAllMocks());
+
+    it('generates component entities for accounts', async () => {
+      listAccounts.mockImplementation(() => {
+        return {
+          async promise() {
+            return {
+              Accounts: [
+                {
+                  Arn:
+                    'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
+                  Name: 'testaccount',
+                },
+              ],
+              NextToken: undefined,
+            };
+          },
+        };
+      });
+      await processor.readLocation(location, false, emit);
+      expect(emit).toBeCalledWith({
+        type: 'entity',
+        location,
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Component',
+          metadata: {
+            annotations: {
+              'amazonaws.com/arn':
+                'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
+              'amazonaws.com/account-id': '957140518395',
+              'amazonaws.com/organization-id': 'o-1vl18kc5a3',
+            },
+            name: 'testaccount',
+            namespace: 'default',
+          },
+          spec: {
+            type: 'cloud-account',
+            lifecycle: 'unknown',
+            owner: 'unknown',
+          },
+        },
+      });
+    });
+
+    it('filters out accounts not in specified location target', async () => {
+      const location = { type: 'aws-cloud-accounts', target: 'o-1vl18kc5a3' };
+      listAccounts.mockImplementation(() => {
+        return {
+          async promise() {
+            return {
+              Accounts: [
+                {
+                  Arn:
+                    'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
+                  Name: 'testaccount',
+                },
+                {
+                  Arn:
+                    'arn:aws:organizations::192594491037:account/o-zzzzzzzzz/957140518395',
+                  Name: 'testaccount2',
+                },
+              ],
+              NextToken: undefined,
+            };
+          },
+        };
+      });
+      await processor.readLocation(location, false, emit);
+      expect(emit).toBeCalledTimes(1);
+      expect(emit).toBeCalledWith({
+        type: 'entity',
+        location,
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Component',
+          metadata: {
+            annotations: {
+              'amazonaws.com/arn':
+                'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
+              'amazonaws.com/account-id': '957140518395',
+              'amazonaws.com/organization-id': 'o-1vl18kc5a3',
+            },
+            name: 'testaccount',
+            namespace: 'default',
+          },
+          spec: {
+            type: 'cloud-account',
+            lifecycle: 'unknown',
+            owner: 'unknown',
+          },
+        },
+      });
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  ComponentEntityV1alpha1,
+  LocationSpec,
+} from '@backstage/catalog-model';
+import AWS, { Organizations } from 'aws-sdk';
+import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations';
+
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+
+const AWS_ORGANIZATION_REGION = 'us-east-1';
+const LOCATION_TYPE = 'aws-cloud-accounts';
+
+const ACCOUNTID_ANNOTATION: string = 'amazonaws.com/account-id';
+const ARN_ANNOTATION: string = 'amazonaws.com/arn';
+const ORGANIZATION_ANNOTATION: string = 'amazonaws.com/organization-id';
+
+/**
+ * A processor for ingesting AWS Accounts from AWS Organizations.
+ *
+ * If custom authentication is needed, it can be achieved by configuring the global AWS.credentials object.
+ */
+export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
+  organizations: Organizations;
+  constructor() {
+    this.organizations = new AWS.Organizations({
+      region: AWS_ORGANIZATION_REGION,
+    }); // Only available in us-east-1
+  }
+
+  normalizeName(name: string): string {
+    return name
+      .trim()
+      .toLocaleLowerCase()
+      .replace(/[^a-zA-Z0-9\-]/g, '-');
+  }
+
+  extractInformationFromArn(
+    arn: string,
+  ): { accountId: string; organizationId: string } {
+    const parts = arn.split('/');
+
+    return {
+      accountId: parts[parts.length - 1],
+      organizationId: parts[parts.length - 2],
+    };
+  }
+
+  async getAwsAccounts(): Promise<Account[]> {
+    let awsAccounts: Account[] = [];
+    let isInitialAttempt = true;
+    let nextToken = undefined;
+    while (isInitialAttempt || nextToken) {
+      isInitialAttempt = false;
+      const orgAccounts: ListAccountsResponse = await this.organizations
+        .listAccounts({ NextToken: nextToken })
+        .promise();
+      if (orgAccounts.Accounts) {
+        awsAccounts = awsAccounts.concat(orgAccounts.Accounts);
+      }
+      nextToken = orgAccounts.NextToken;
+    }
+
+    return awsAccounts;
+  }
+
+  mapAccountToComponent(account: Account): ComponentEntityV1alpha1 {
+    const { accountId, organizationId } = this.extractInformationFromArn(
+      account.Arn as string,
+    );
+    return {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        annotations: {
+          [ACCOUNTID_ANNOTATION]: accountId,
+          [ARN_ANNOTATION]: account.Arn || '',
+          [ORGANIZATION_ANNOTATION]: organizationId,
+        },
+        name: this.normalizeName(account.Name || ''),
+        namespace: 'default',
+      },
+      spec: {
+        type: 'cloud-account',
+        lifecycle: 'unknown',
+        owner: 'unknown',
+      },
+    };
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== LOCATION_TYPE) {
+      return false;
+    }
+
+    (await this.getAwsAccounts())
+      .map(account => this.mapAccountToComponent(account))
+      .filter(entity => {
+        if (location.target !== '') {
+          if (entity.metadata.annotations) {
+            return (
+              entity.metadata.annotations[ORGANIZATION_ANNOTATION] ===
+              location.target
+            );
+          }
+          return false;
+        }
+        return true;
+      })
+      .forEach((entity: ComponentEntityV1alpha1) => {
+        emit(results.entity(location, entity));
+      });
+
+    return true;
+  }
+}

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -17,6 +17,7 @@
 import * as results from './results';
 
 export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
+export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';
 export { FileReaderProcessor } from './FileReaderProcessor';