From 3b7cf6cb86628181cec993738d2fc4bf44a1280f Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Tue, 11 Jun 2024 20:15:53 +0200 Subject: [PATCH] Add possibility to define entity relationships by using AWS Tags when providing Resource entities directly from AWS. Expose `system`, `domain`, `dependencyOf` and `dependsOn` tag keys to be used. --- .changeset/slimy-tools-grow.md | 5 ++ .../src/providers/AWSDynamoDbTableProvider.ts | 7 +- .../src/providers/AWSEC2Provider.ts | 7 +- .../src/providers/AWSEKSClusterProvider.ts | 7 +- .../src/providers/AWSEntityProvider.ts | 12 ++- .../src/providers/AWSIAMRoleProvider.ts | 7 +- .../providers/AWSLambdaFunctionProvider.ts | 8 +- .../AWSOrganizationAccountsProvider.ts | 7 +- .../src/providers/AWSRDSProvider.ts | 7 +- .../src/providers/AWSS3BucketProvider.ts | 7 +- .../src/utils/tags.test.ts | 31 +++++++- .../src/utils/tags.ts | 75 ++++++++++++++++--- 12 files changed, 154 insertions(+), 26 deletions(-) create mode 100644 .changeset/slimy-tools-grow.md diff --git a/.changeset/slimy-tools-grow.md b/.changeset/slimy-tools-grow.md new file mode 100644 index 000000000..898c6c095 --- /dev/null +++ b/.changeset/slimy-tools-grow.md @@ -0,0 +1,5 @@ +--- +'@roadiehq/catalog-backend-module-aws': minor +--- + +Add possibility to define entity relationships by using AWS Tags when providing Resource entities directly from AWS. Expose `system`, `domain`, `dependencyOf` and `dependsOn` tag keys to be used. diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts index fa087712c..40a3f46a6 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts @@ -21,7 +21,11 @@ import { AWSEntityProvider } from './AWSEntityProvider'; import { ResourceEntity } from '@backstage/catalog-model'; import { ANNOTATION_AWS_DDB_TABLE_ARN } from '../annotations'; import { arnToName } from '../utils/arnToName'; -import { labelsFromTags, ownerFromTags } from '../utils/tags'; +import { + labelsFromTags, + ownerFromTags, + relationShipsFromTags, +} from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; /** @@ -105,6 +109,7 @@ export class AWSDynamoDbTableProvider extends AWSEntityProvider { }, spec: { owner: ownerFromTags(tags, this.getOwnerTag(), groups), + ...relationShipsFromTags(tags), type: 'dynamo-db-table', }, }; diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts index 45f9107bf..30149454e 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts @@ -21,7 +21,11 @@ import { Config } from '@backstage/config'; import { AWSEntityProvider } from './AWSEntityProvider'; import { ANNOTATION_AWS_EC2_INSTANCE_ID } from '../annotations'; import { ARN } from 'link2aws'; -import { labelsFromTags, ownerFromTags } from '../utils/tags'; +import { + labelsFromTags, + ownerFromTags, + relationShipsFromTags, +} from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; /** @@ -104,6 +108,7 @@ export class AWSEC2Provider extends AWSEntityProvider { }, spec: { owner: ownerFromTags(instance.Tags, this.getOwnerTag(), groups), + ...relationShipsFromTags(instance.Tags), type: 'ec2-instance', }, }; diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts index 7fb4f3501..33addad94 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts @@ -24,7 +24,11 @@ import { ANNOTATION_AWS_IAM_ROLE_ARN, } from '../annotations'; import { arnToName } from '../utils/arnToName'; -import { labelsFromTags, ownerFromTags } from '../utils/tags'; +import { + labelsFromTags, + ownerFromTags, + relationShipsFromTags, +} from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; /** @@ -111,6 +115,7 @@ export class AWSEKSClusterProvider extends AWSEntityProvider { this.getOwnerTag(), groups, ), + ...relationShipsFromTags(cluster.cluster?.tags), type: 'eks-cluster', }, }; diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEntityProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEntityProvider.ts index ed477568d..b8b92cb4d 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEntityProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEntityProvider.ts @@ -68,10 +68,14 @@ export abstract class AWSEntityProvider implements EntityProvider { protected getCredentials() { const region = parseArn(this.roleArn).region; - return fromTemporaryCredentials({ - params: { RoleArn: this.roleArn, ExternalId: this.externalId }, - clientConfig: { region: region }, - }); + return region + ? fromTemporaryCredentials({ + params: { RoleArn: this.roleArn, ExternalId: this.externalId }, + clientConfig: { region: region }, + }) + : fromTemporaryCredentials({ + params: { RoleArn: this.roleArn, ExternalId: this.externalId }, + }); } protected async getGroups() { diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts index 8717ae261..8a89a6eb2 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts @@ -22,7 +22,11 @@ import { AWSEntityProvider } from './AWSEntityProvider'; import { ANNOTATION_AWS_IAM_ROLE_ARN } from '../annotations'; import { arnToName } from '../utils/arnToName'; import { ARN } from 'link2aws'; -import { labelsFromTags, ownerFromTags } from '../utils/tags'; +import { + labelsFromTags, + ownerFromTags, + relationShipsFromTags, +} from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; /** @@ -97,6 +101,7 @@ export class AWSIAMRoleProvider extends AWSEntityProvider { spec: { type: 'aws-role', owner: ownerFromTags(role.Tags, this.getOwnerTag(), groups), + ...relationShipsFromTags(role.Tags), }, }; diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts index 3bec0c2fb..96e1b5c7b 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts @@ -25,7 +25,11 @@ import { } from '../annotations'; import { arnToName } from '../utils/arnToName'; import { ARN } from 'link2aws'; -import { labelsFromTags, ownerFromTags } from '../utils/tags'; +import { + labelsFromTags, + ownerFromTags, + relationShipsFromTags, +} from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; /** @@ -119,8 +123,8 @@ export class AWSLambdaFunctionProvider extends AWSEntityProvider { }, spec: { owner: ownerFromTags(tags, this.getOwnerTag(), groups), + ...relationShipsFromTags(tags), type: 'lambda-function', - dependsOn: [], }, }); } diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts index 0840b6725..a70661666 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts @@ -29,7 +29,11 @@ import { ANNOTATION_AWS_ACCOUNT_ARN, } from '../annotations'; import { arnToName } from '../utils/arnToName'; -import { labelsFromTags, ownerFromTags } from '../utils/tags'; +import { + labelsFromTags, + ownerFromTags, + relationShipsFromTags, +} from '../utils/tags'; import { Tag } from '@aws-sdk/client-organizations/dist-types/models/models_0'; import { CatalogApi } from '@backstage/catalog-client'; @@ -119,6 +123,7 @@ export class AWSOrganizationAccountsProvider extends AWSEntityProvider { }, spec: { owner: ownerFromTags(tags, this.getOwnerTag(), groups), + ...relationShipsFromTags(tags), type: 'aws-account', }, }; diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts index cc2c7a410..e9271be24 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts @@ -21,7 +21,11 @@ import { Config } from '@backstage/config'; import { AWSEntityProvider } from './AWSEntityProvider'; import { ANNOTATION_AWS_RDS_INSTANCE_ARN } from '../annotations'; import { ARN } from 'link2aws'; -import { labelsFromTags, ownerFromTags } from '../utils/tags'; +import { + labelsFromTags, + ownerFromTags, + relationShipsFromTags, +} from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; /** @@ -110,6 +114,7 @@ export class AWSRDSProvider extends AWSEntityProvider { this.getOwnerTag(), groups, ), + ...relationShipsFromTags(dbInstance.TagList), type: 'rds-instance', }, }; diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts index 8642aaa54..dd795b671 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts @@ -22,7 +22,11 @@ import { AWSEntityProvider } from './AWSEntityProvider'; import { ANNOTATION_AWS_S3_BUCKET_ARN } from '../annotations'; import { arnToName } from '../utils/arnToName'; import { ARN } from 'link2aws'; -import { labelsFromTags, ownerFromTags } from '../utils/tags'; +import { + labelsFromTags, + ownerFromTags, + relationShipsFromTags, +} from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; /** @@ -92,6 +96,7 @@ export class AWSS3BucketProvider extends AWSEntityProvider { }, spec: { owner: ownerFromTags(tags, this.getOwnerTag(), groups), + ...relationShipsFromTags(tags), type: 's3-bucket', }, }; diff --git a/plugins/backend/catalog-backend-module-aws/src/utils/tags.test.ts b/plugins/backend/catalog-backend-module-aws/src/utils/tags.test.ts index 1d17771e2..fa52c1b6c 100644 --- a/plugins/backend/catalog-backend-module-aws/src/utils/tags.test.ts +++ b/plugins/backend/catalog-backend-module-aws/src/utils/tags.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { ownerFromTags, labelsFromTags } from './tags'; +import { ownerFromTags, labelsFromTags, relationShipsFromTags } from './tags'; import { Entity } from '@backstage/catalog-model'; describe('labelsFromTags and ownerFromTags', () => { @@ -150,3 +150,32 @@ describe('labelsFromTags and ownerFromTags', () => { }); }); }); + +describe('relationShipsFromTags', () => { + it('should return an empty object if tags is undefined', () => { + const output = relationShipsFromTags(); + expect(output).toEqual({}); + }); + + it('should return an empty object if tags is an empty array', () => { + const output = relationShipsFromTags([]); + expect(output).toEqual({}); + }); + + it('should return relationships from an array of tags', () => { + const tags = [{ Key: 'dependsOn', Value: 'Value1' }]; + const output = relationShipsFromTags(tags); + expect(output).toEqual({ dependsOn: ['Value1'] }); + }); + + it('should be case-insensitive when matching tag keys', () => { + const tags = [{ Key: 'dePeNdsOn', Value: 'Value1' }]; + const output = relationShipsFromTags(tags); + expect(output).toEqual({ dependsOn: ['Value1'] }); + }); + it('should work with dependency of tag', () => { + const tags = [{ Key: 'dependencyOf', Value: 'Value1' }]; + const output = relationShipsFromTags(tags); + expect(output).toEqual({ dependencyOf: ['Value1'] }); + }); +}); diff --git a/plugins/backend/catalog-backend-module-aws/src/utils/tags.ts b/plugins/backend/catalog-backend-module-aws/src/utils/tags.ts index b578520b9..1ce8532c2 100644 --- a/plugins/backend/catalog-backend-module-aws/src/utils/tags.ts +++ b/plugins/backend/catalog-backend-module-aws/src/utils/tags.ts @@ -22,29 +22,49 @@ type Tag = { }; const UNKNOWN_OWNER = 'unknown'; +const TAG_DEPENDS_ON = 'dependsOn'; +const TAG_DEPENDENCY_OF = 'dependencyOf'; +const TAG_SYSTEM = 'system'; +const TAG_DOMAIN = 'domain'; + +const dependencyTags = [TAG_DEPENDENCY_OF, TAG_DEPENDS_ON]; +const relationshipTags = [TAG_SYSTEM, TAG_DOMAIN]; + export const labelsFromTags = (tags?: Tag[] | Record) => { if (!tags) { return {}; } if (Array.isArray(tags)) { - return tags?.reduce((acc: Record, tag) => { - if (tag.Key && tag.Value) { - const key = tag.Key.replaceAll(':', '_').replaceAll('/', '-'); - acc[key] = tag.Value.replaceAll('/', '-').substring(0, 63); - } - return acc; - }, {}); + return tags + ?.filter( + tag => + !tag.Key || + ![...dependencyTags, ...relationshipTags] + .map(it => it.toLowerCase()) + .includes(tag.Key.toLowerCase()), + ) + .reduce((acc: Record, tag) => { + if (tag.Key && tag.Value) { + const key = tag.Key.replaceAll(':', '_').replaceAll('/', '-'); + acc[key] = tag.Value.replaceAll('/', '-').substring(0, 63); + } + return acc; + }, {}); } - return Object.entries(tags as Record).reduce( - (acc: Record, [key, value]) => { + return Object.entries(tags as Record) + ?.filter( + ([tagKey]) => + ![...dependencyTags, ...relationshipTags] + .map(it => it.toLowerCase()) + .includes(tagKey.toLowerCase()), + ) + .reduce((acc: Record, [key, value]) => { if (key && value) { const k = key.replaceAll(':', '_').replaceAll('/', '-'); acc[k] = value.replaceAll('/', '-').substring(0, 63); } return acc; - }, - {}, - ); + }, {}); }; export const ownerFromTags = ( @@ -79,3 +99,34 @@ export const ownerFromTags = ( return ownerString ? ownerString : UNKNOWN_OWNER; }; + +export const relationShipsFromTags = ( + tags?: Tag[] | Record, +): Record => { + if (!tags) { + return {}; + } + + const specPartial: Record = {}; + if (Array.isArray(tags)) { + dependencyTags.forEach(tagKey => { + const tagValue = tags?.find( + tag => tag.Key?.toLowerCase() === tagKey?.toLowerCase(), + ); + if (tagValue && tagValue.Value) { + specPartial[tagKey] = [tagValue.Value.split(',')].flat(); + } + }); + + relationshipTags.forEach(tagKey => { + const tagValue = tags?.find( + tag => tag.Key?.toLowerCase() === tagKey?.toLowerCase(), + ); + if (tagValue && tagValue.Value) { + specPartial[tagKey] = tagValue.Value; + } + }); + } + + return specPartial; +};