From d7a975dc2a5ffdb784eef36c4a827d553e7ac1c3 Mon Sep 17 00:00:00 2001 From: "mariano.pizarro" Date: Mon, 11 Apr 2022 12:44:56 -0300 Subject: [PATCH 1/2] feat: Handle TODOs for secrets manager --- README.md | 6 +- src/properties/logger.ts | 1 + src/services/kms/data.ts | 47 ++++++++++++ src/services/kms/format.ts | 28 ++++++-- src/services/kms/schema.graphql | 22 +++++- src/services/lambda/schema.graphql | 1 + src/services/secretsManager/connections.ts | 84 ++++++++++++++++++++++ src/services/secretsManager/data.ts | 50 ++++++++++++- src/services/secretsManager/format.ts | 37 ++++++++-- src/services/secretsManager/index.ts | 3 + src/services/secretsManager/schema.graphql | 27 +++++-- src/types/generated.ts | 32 +++++++-- 12 files changed, 308 insertions(+), 30 deletions(-) create mode 100644 src/services/secretsManager/connections.ts diff --git a/README.md b/README.md index f40bb1b4..0c7d69d5 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,8 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi | iot | | | kinesisFirehose | kinesisStream, s3 | | kinesisStream | kinesisFirehose | -| kms | cloudtrail, cloudwatchLog, codebuild, efs, eksCluster, elastiCacheReplicationGroup, elasticSearchDomain, emrCluster, lambda, rdsClusterSnapshot, sns, sageMakerNotebookInstance, dmsReplicationInstance, redshiftCluster | -| lambda | appSync, cognitoUserPool, kms, securityGroup, subnet, vpc | +| kms | cloudtrail, cloudwatchLog, codebuild, dmsReplicationInstance, efs, eksCluster, elastiCacheReplicationGroup, elasticSearchDomain, emrCluster, lambda, redshiftCluster, rdsClusterSnapshot, sageMakerNotebookInstance, secretsManager, sns | +| lambda | appSync, cognitoUserPool, kms, secretsManager, securityGroup, subnet, vpc | | managedAirflow | iamRole, securityGroups, subnet, s3 | | nacl | vpc | | natGateway | networkInterface, subnet, vpc | @@ -146,7 +146,7 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi | sageMakerNotebookInstance | iamRole, kms, networkInterface, subnet, securityGroup | | sageMakerProject | | | s3 | cloudfront, cloudtrail, kinesisFirehose, managedAirflow | -| secretsManager | | +| secretsManager | lambda, kms | | securityGroup | alb, asg, clientVpnEndpoint, codebuild, dmsReplicationInstance, ecsService, lambda, ec2, elasticSearchDomain, elb, rdsCluster, rdsDbInstance, eksCluster, elastiCacheCluster, managedAirflow, sageMakerNotebookInstance | | ses | | | sns | kms, cloudtrail, cloudwatch | diff --git a/src/properties/logger.ts b/src/properties/logger.ts index cc2a82eb..d7c417c5 100644 --- a/src/properties/logger.ts +++ b/src/properties/logger.ts @@ -427,6 +427,7 @@ export default { gettingRotationStatus: 'Checking rotation status for each key...', gettingPolicies: 'Fetching default Policy for each key...', gettingTags: 'Fetching Tags for each key...', + gettingAliases: 'Fetching Aliases for each key...', /** * EKS diff --git a/src/services/kms/data.ts b/src/services/kms/data.ts index 7d4ca852..15033778 100644 --- a/src/services/kms/data.ts +++ b/src/services/kms/data.ts @@ -6,6 +6,7 @@ import isEmpty from 'lodash/isEmpty' import { AWSError, Request } from 'aws-sdk' import { Config } from 'aws-sdk/lib/config' import KMS, { + AliasListEntry, KeyListEntry, KeyMetadata, ListKeysRequest, @@ -33,6 +34,7 @@ export type AwsKms = KeyListEntry & policy: string Tags: TagMap keyRotationEnabled: boolean + Aliases?: AliasListEntry[] } export default async ({ @@ -50,6 +52,7 @@ export default async ({ const regionPromises = [] const policyPromises = [] const tagPromises = [] + const aliasesPromises = [] /** * Step 1) for all regions, list the kms keys @@ -331,6 +334,50 @@ export default async ({ }) await Promise.all(tagPromises) + + /** + * Step 6) get aliases for each key + */ + + logger.debug(lt.gettingAliases) + + kmsData.map(({ region, KeyId }, idx) => { + const kms = new KMS({ ...config, region, endpoint }) + + const aliasesPromise = new Promise(resolveAliases => + kms.listAliases({ KeyId }, (err, data) => { + if (err) { + errorLog.generateAwsErrorLog({ + functionName: 'kms:listAliases', + err, + }) + resolveAliases() + } + + /** + * No aliases data + */ + + if (isEmpty(data)) { + return resolveAliases() + } + + /** + * Add the aliases to the key + */ + + const { Aliases: aliases } = data || {} + + kmsData[idx].Aliases = aliases + + resolveAliases() + }) + ) + + aliasesPromises.push(aliasesPromise) + }) + + await Promise.all(aliasesPromises) errorLog.reset() resolve(groupBy(kmsData, 'region')) diff --git a/src/services/kms/format.ts b/src/services/kms/format.ts index 615c027a..2bacde34 100644 --- a/src/services/kms/format.ts +++ b/src/services/kms/format.ts @@ -1,8 +1,24 @@ -import t from '../../properties/translations' +import { AliasListEntry } from 'aws-sdk/clients/kms' +import cuid from 'cuid' import { AwsKms } from './data' -import { AwsKms as AwsKmsType } from '../../types/generated' +import { AwsKms as AwsKmsType, AwsKmsAliasListEntry } from '../../types/generated' import { formatTagsFromMap, formatIamJsonPolicy } from '../../utils/format' +export const formatAliases = ( + aliases?: AliasListEntry[] +): AwsKmsAliasListEntry[] => { + return ( + aliases?.map(a => ({ + id: cuid(), + aliasName: a.AliasName, + aliasArn: a.AliasArn, + targetKeyId: a.TargetKeyId, + creationDate: a.CreationDate?.toISOString(), + lastUpdatedDate: a.LastUpdatedDate?.toISOString(), + })) || [] + ) +} + /** * KMS */ @@ -32,6 +48,7 @@ export default ({ Origin: origin, DeletionDate: deletionDate, ValidTo: validTo, + Aliases: aliases = [] } = key return { @@ -47,10 +64,11 @@ export default ({ keyState, customerMasterKeySpec, tags: formatTagsFromMap(Tags), - creationDate: creationDate ? creationDate.toString() : undefined, + creationDate: creationDate?.toISOString(), keyManager, origin, - deletionDate: deletionDate ? deletionDate.toString() : undefined, - validTo: validTo ? validTo.toString() : undefined, + deletionDate: deletionDate?.toISOString(), + validTo: validTo?.toISOString(), + aliases: formatAliases(aliases), } } diff --git a/src/services/kms/schema.graphql b/src/services/kms/schema.graphql index 65d47713..db24cb74 100644 --- a/src/services/kms/schema.graphql +++ b/src/services/kms/schema.graphql @@ -1,3 +1,17 @@ +type awsKmsAliasListEntry + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) { + id: String! @id + aliasName: String @search(by: [hash, regexp]) + aliasArn: String @search(by: [hash, regexp]) + targetKeyId: String @search(by: [hash, regexp]) + creationDate: DateTime @search(by: [day]) + lastUpdatedDate: DateTime @search(by: [day]) +} + type awsKms @key(fields: "id"){ id: String! @id @search(by: [hash]) accountId: String! @search(by: [hash]) @@ -11,11 +25,12 @@ type awsKms @key(fields: "id"){ keyState: String @search(by: [hash, regexp]) customerMasterKeySpec: String @search(by: [hash, regexp]) tags: [awsRawTag] - creationDate: String @search(by: [hash, regexp]) + creationDate: DateTime @search(by: [day]) keyManager: String @search(by: [hash, regexp]) origin: String @search(by: [hash, regexp]) - deletionDate: String @search(by: [hash, regexp]) - validTo: String @search(by: [hash, regexp]) + deletionDate: DateTime @search(by: [day]) + validTo: DateTime @search(by: [day]) + aliases: [awsKmsAliasListEntry] lambda: [awsLambda] @hasInverse(field: kms) #change to plural cloudtrail: [awsCloudtrail] @hasInverse(field: kms) #change to plural redshiftCluster: [awsRedshiftCluster] @hasInverse(field: kms) #change to plural @@ -30,4 +45,5 @@ type awsKms @key(fields: "id"){ dmsReplicationInstances: [awsDmsReplicationInstance] @hasInverse(field: kms) sageMakerNotebookInstances: [awsSageMakerNotebookInstance] @hasInverse(field: kms) rdsClusterSnapshots: [awsRdsClusterSnapshot] @hasInverse(field: kms) + secretsManager: [awsSecretsManager] @hasInverse(field: kms) } diff --git a/src/services/lambda/schema.graphql b/src/services/lambda/schema.graphql index 0335cd6f..e50bb2b9 100644 --- a/src/services/lambda/schema.graphql +++ b/src/services/lambda/schema.graphql @@ -26,6 +26,7 @@ type awsLambda @key(fields: "arn") { vpc: [awsVpc] @hasInverse(field: lambda) cognitoUserPool: [awsCognitoUserPool] @hasInverse(field: lambda) #change to plural appSync: [awsAppSync] @hasInverse(field: lambda) + secretsManager: [awsSecretsManager] @hasInverse(field: lambda) } type awsLambdaEnvironmentVariable diff --git a/src/services/secretsManager/connections.ts b/src/services/secretsManager/connections.ts new file mode 100644 index 00000000..268ef50f --- /dev/null +++ b/src/services/secretsManager/connections.ts @@ -0,0 +1,84 @@ +import { ServiceConnection } from '@cloudgraph/sdk' +import { isEmpty } from 'lodash' +import { RawAwsSecretsManager } from './data' +import { AwsKms } from '../kms/data' +import { RawAwsLambdaFunction } from '../lambda/data' +import services from '../../enums/services' + +export default ({ + service, + data, + region, +}: { + account: string + service: RawAwsSecretsManager + data: Array<{ name: string; data: { [property: string]: any[] } }> + region: string +}): { + [property: string]: ServiceConnection[] +} => { + const connections: ServiceConnection[] = [] + const { + ARN: id, + KmsKeyId: kmsKeyId, + RotationLambdaARN: rotationLambdaARN, + ReplicationStatus: replicationStatus, + } = service + + /** + * Find KMS + * related to this Secrets Manager + */ + const kmsKeyIds: string[] = replicationStatus?.map(rs => rs.KmsKeyId) + const kmsKeys = data.find(({ name }) => name === services.kms) + if (kmsKeys?.data?.[region]) { + const kmsKeyInRegion: AwsKms[] = kmsKeys.data[region].filter( + ({ Arn: arn, KeyId: keyId, Aliases: aliases = [] }: AwsKms) => + kmsKeyId === arn || + kmsKeyIds?.includes(arn) || + kmsKeyIds?.includes(keyId) || + aliases?.some(a => kmsKeyIds?.includes(a.AliasName)) + ) + + if (!isEmpty(kmsKeyInRegion)) { + for (const kms of kmsKeyInRegion) { + const { KeyId: keyId }: AwsKms = kms + connections.push({ + id: keyId, + resourceType: services.kms, + relation: 'child', + field: 'kms', + }) + } + } + } + + /** + * Find Lambda Functions + * related to this Secrets Manager + */ + const lambdas = data.find(({ name }) => name === services.lambda) + + if (rotationLambdaARN && lambdas?.data?.[region]) { + const lambdaInRegion: RawAwsLambdaFunction = lambdas.data[region].find( + ({ FunctionArn: functionArn }: RawAwsLambdaFunction) => + rotationLambdaARN === functionArn + ) + + if (lambdaInRegion) { + const { FunctionArn: functionArn }: RawAwsLambdaFunction = lambdaInRegion + + connections.push({ + id: functionArn, + resourceType: services.lambda, + relation: 'child', + field: 'lambda', + }) + } + } + + const snsResult = { + [id]: connections, + } + return snsResult +} diff --git a/src/services/secretsManager/data.ts b/src/services/secretsManager/data.ts index aeb98b7b..133ff88b 100644 --- a/src/services/secretsManager/data.ts +++ b/src/services/secretsManager/data.ts @@ -1,4 +1,8 @@ -import SM, { SecretListEntry } from 'aws-sdk/clients/secretsmanager' +import SM, { + SecretListEntry, + DescribeSecretResponse, + ReplicationStatusType, +} from 'aws-sdk/clients/secretsmanager' import { AWSError } from 'aws-sdk/lib/error' import CloudGraph from '@cloudgraph/sdk' import groupBy from 'lodash/groupBy' @@ -22,8 +26,33 @@ const endpoint = initTestEndpoint(serviceName) export interface RawAwsSecretsManager extends Omit { region: string Tags: TagMap + ReplicationStatus?: ReplicationStatusType[] } +const getSecretDetails = async ( + sm: SM, + secretId: string +): Promise => + new Promise(resolve => { + sm.describeSecret( + { + SecretId: secretId, + }, + (err: AWSError, data: DescribeSecretResponse) => { + if (err) { + errorLog.generateAwsErrorLog({ + functionName: 'sm:describeSecret', + err, + }) + } + if (!isEmpty(data)) { + resolve(data) + } + resolve({}) + } + ) + }) + export default async ({ regions, config, @@ -34,6 +63,7 @@ export default async ({ new Promise(async resolve => { const smData: RawAwsSecretsManager[] = [] const regionPromises = [] + const secretsDetailsPromises = [] regions.split(',').map(region => { const regionPromise = new Promise(resolveSecretsManagerData => { @@ -73,8 +103,24 @@ export default async ({ }) regionPromises.push(regionPromise) }) - await Promise.all(regionPromises) + + smData.map(({ ARN: arn, region }, idx) => { + const sm = new SM({ ...config, region, endpoint }) + const secretDetailsPromise = new Promise( + async resolveSecretDetails => { + const secretDetails: DescribeSecretResponse = await getSecretDetails( + sm, + arn + ) + smData[idx].ReplicationStatus = secretDetails?.ReplicationStatus || [] + resolveSecretDetails() + } + ) + secretsDetailsPromises.push(secretDetailsPromise) + }) + await Promise.all(secretsDetailsPromises) + errorLog.reset() resolve(groupBy(smData, 'region')) diff --git a/src/services/secretsManager/format.ts b/src/services/secretsManager/format.ts index a52d0900..b00bdc85 100644 --- a/src/services/secretsManager/format.ts +++ b/src/services/secretsManager/format.ts @@ -1,16 +1,36 @@ +import cuid from 'cuid' +import { ReplicationStatusType } from 'aws-sdk/clients/secretsmanager' import { RawAwsSecretsManager } from './data' -import { AwsSecretsManager } from '../../types/generated' +import { + AwsSecretsManager, + AwsSecretsManagerReplicationStatus, +} from '../../types/generated' import { formatTagsFromMap } from '../../utils/format' +export const formatReplicationStatus = ( + replicationStatus?: ReplicationStatusType[] +): AwsSecretsManagerReplicationStatus[] => { + return ( + replicationStatus?.map(rs => ({ + id: cuid(), + region: rs.Region, + kmsKeyId: rs.KmsKeyId, + status: rs.Status, + statusMessage: rs.StatusMessage, + lastAccessedDate: rs.LastAccessedDate?.toISOString(), + })) || [] + ) +} + /** * Secrets Manager */ -export default ({ +export default ({ service, account, - region -}:{ + region, +}: { service: RawAwsSecretsManager account: string region: string @@ -29,6 +49,7 @@ export default ({ DeletedDate: deletedDate, OwningService: owningService, CreatedDate: createdDate, + ReplicationStatus: replicationStatus = [], Tags, } = service @@ -43,12 +64,16 @@ export default ({ tags: formatTagsFromMap(Tags), rotationEnabled, rotationLambdaARN, - rotationRules: { automaticallyAfterDays: rotationRules?.AutomaticallyAfterDays }, + rotationRules: { + id: cuid(), + automaticallyAfterDays: rotationRules?.AutomaticallyAfterDays, + }, lastRotatedDate: lastRotatedDate?.toISOString(), lastChangedDate: lastChangedDate?.toISOString(), lastAccessedDate: lastAccessedDate?.toISOString(), deletedDate: deletedDate?.toISOString(), createdDate: createdDate?.toISOString(), owningService, + replicationStatus: formatReplicationStatus(replicationStatus), } -} \ No newline at end of file +} diff --git a/src/services/secretsManager/index.ts b/src/services/secretsManager/index.ts index e9ebf294..3855a0af 100644 --- a/src/services/secretsManager/index.ts +++ b/src/services/secretsManager/index.ts @@ -2,6 +2,7 @@ import { Service } from '@cloudgraph/sdk' import BaseService from '../base' import format from './format' import getData from './data' +import getConnections from './connections' import mutation from './mutation' export default class SecretsManager extends BaseService implements Service { @@ -9,5 +10,7 @@ export default class SecretsManager extends BaseService implements Service { getData = getData.bind(this) + getConnections = getConnections.bind(this) + mutation = mutation } diff --git a/src/services/secretsManager/schema.graphql b/src/services/secretsManager/schema.graphql index 1c8937be..232d0db2 100644 --- a/src/services/secretsManager/schema.graphql +++ b/src/services/secretsManager/schema.graphql @@ -1,12 +1,26 @@ +type awsSecretsManagerReplicationStatus + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) { + id: String! @id + region: String @search(by: [hash, regexp]) + kmsKeyId: String @search(by: [hash, regexp]) + status: String @search(by: [hash, regexp]) + statusMessage: String @search(by: [hash, regexp]) + lastAccessedDate: DateTime @search(by: [day]) +} + type awsSecretsManagerRotationRule @generate( query: { get: false, query: true, aggregate: false } mutation: { add: false, delete: false } subscription: false ) { - id: ID - automaticallyAfterDays: Int @search - } + id: String! @id + automaticallyAfterDays: Int @search +} type awsSecretsManager @key(fields: "arn") { id: String! @search(by: [hash]) @@ -26,8 +40,7 @@ type awsSecretsManager @key(fields: "arn") { deletedDate: DateTime @search(by: [day]) createdDate: DateTime @search(by: [day]) owningService: String @search(by: [hash, regexp]) + replicationStatus: [awsSecretsManagerReplicationStatus] + kms: [awsKms] @hasInverse(field: secretsManager) + lambda: [awsLambda] @hasInverse(field: secretsManager) } - -# TODO: add connection to kms (also use ReplicationStatus.KmsKeyId) -# TODO: add connection to lambda - diff --git a/src/types/generated.ts b/src/types/generated.ts index 2e759360..f2bd350a 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -3334,13 +3334,14 @@ export type AwsKinesisStream = { export type AwsKms = { accountId: Scalars['String']; + aliases?: Maybe>>; arn: Scalars['String']; cloudtrail?: Maybe>>; cloudwatchLog?: Maybe>>; codebuilds?: Maybe>>; - creationDate?: Maybe; + creationDate?: Maybe; customerMasterKeySpec?: Maybe; - deletionDate?: Maybe; + deletionDate?: Maybe; description?: Maybe; dmsReplicationInstances?: Maybe>>; efs?: Maybe>>; @@ -3360,10 +3361,20 @@ export type AwsKms = { redshiftCluster?: Maybe>>; region?: Maybe; sageMakerNotebookInstances?: Maybe>>; + secretsManager?: Maybe>>; sns?: Maybe>>; tags?: Maybe>>; usage?: Maybe; - validTo?: Maybe; + validTo?: Maybe; +}; + +export type AwsKmsAliasListEntry = { + aliasArn?: Maybe; + aliasName?: Maybe; + creationDate?: Maybe; + id: Scalars['String']; + lastUpdatedDate?: Maybe; + targetKeyId?: Maybe; }; export type AwsLambda = { @@ -3385,6 +3396,7 @@ export type AwsLambda = { reservedConcurrentExecutions?: Maybe; role?: Maybe; runtime?: Maybe; + secretsManager?: Maybe>>; securityGroups?: Maybe>>; sourceCodeSize?: Maybe; subnet?: Maybe>>; @@ -3973,22 +3985,34 @@ export type AwsSecretsManager = { deletedDate?: Maybe; description?: Maybe; id: Scalars['String']; + kms?: Maybe>>; kmsKeyId?: Maybe; + lambda?: Maybe>>; lastAccessedDate?: Maybe; lastChangedDate?: Maybe; lastRotatedDate?: Maybe; name?: Maybe; owningService?: Maybe; region?: Maybe; + replicationStatus?: Maybe>>; rotationEnabled?: Maybe; rotationLambdaARN?: Maybe; rotationRules?: Maybe; tags?: Maybe>>; }; +export type AwsSecretsManagerReplicationStatus = { + id: Scalars['String']; + kmsKeyId?: Maybe; + lastAccessedDate?: Maybe; + region?: Maybe; + status?: Maybe; + statusMessage?: Maybe; +}; + export type AwsSecretsManagerRotationRule = { automaticallyAfterDays?: Maybe; - id?: Maybe; + id: Scalars['String']; }; export type AwsSecurityGroup = { From 0ddf44211fe3f324371e6a21c6d5df3bc69facfb Mon Sep 17 00:00:00 2001 From: "mariano.pizarro" Date: Mon, 11 Apr 2022 15:00:14 -0300 Subject: [PATCH 2/2] feat: Update README file --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 60ac9bfc..c39dd094 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi | iot | | | kinesisFirehose | kinesisStream, s3 | | kinesisStream | kinesisFirehose | -| kms | cloudtrail, cloudwatchLog, codebuild, dmsReplicationInstance, efs, eksCluster, elastiCacheReplicationGroup, elasticSearchDomain, emrCluster, lambda, redshiftCluster, rdsClusterSnapshot, sageMakerNotebookInstance, secretsManager, sns | +| kms | cloudtrail, cloudwatchLog, codebuild, ecsCluster, dmsReplicationInstance, efs, eksCluster, elastiCacheReplicationGroup, elasticSearchDomain, emrCluster, lambda, redshiftCluster, rdsClusterSnapshot, sageMakerNotebookInstance, secretsManager, sns | | lambda | appSync, cognitoUserPool, kms, secretsManager, securityGroup, subnet, vpc | | managedAirflow | iamRole, securityGroups, subnet, s3 | | nacl | vpc | @@ -145,7 +145,7 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi | sageMakerExperiment | | | sageMakerNotebookInstance | iamRole, kms, networkInterface, subnet, securityGroup | | sageMakerProject | | -| s3 | cloudfront, cloudtrail, kinesisFirehose, managedAirflow | +| s3 | cloudfront, cloudtrail, ecsCluster, kinesisFirehose, managedAirflow | | secretsManager | lambda, kms | | securityGroup | alb, asg, clientVpnEndpoint, codebuild, dmsReplicationInstance, ecsService, lambda, ec2, elasticSearchDomain, elb, rdsCluster, rdsDbInstance, eksCluster, elastiCacheCluster, managedAirflow, sageMakerNotebookInstance | | ses | |