diff --git a/README.md b/README.md index b6391c03..3be04b06 100644 --- a/README.md +++ b/README.md @@ -124,14 +124,14 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi | iamServerCertificate | | | iamUser | iamGroup | | iamPolicy | iamRole, iamGroup | -| iamRole | appSync, cloudformationStackSet, codebuild, configurationRecorder, ec2, iamInstanceProfile, iamPolicy, eksCluster, ecsService, flowLog, glueJob, managedAirflow, sageMakerNotebookInstance, systemsManagerInstance guardDutyDetector, lambda, kinesisFirehose, rdsCluster | +| iamRole | appSync, cloudformationStackSet, codebuild, configurationRecorder, ec2, iamInstanceProfile, iamPolicy, eksCluster, ecsService, flowLog, glueJob, managedAirflow, s3, sageMakerNotebookInstance, systemsManagerInstance guardDutyDetector, lambda, kinesisFirehose, rdsCluster | | iamGroup | iamUser, iamPolicy | | igw | vpc | | iot | | | kinesisFirehose | kinesisStream, s3, iamRole | | kinesisStream | kinesisFirehose | | kms | cloudtrail, cloudwatchLog, codebuild, ecsCluster, efs, eksCluster, elastiCacheReplicationGroup, elasticSearchDomain, emrCluster, lambda, rdsClusterSnapshot, sns, sageMakerNotebookInstance, secretsManager, dmsReplicationInstance, redshiftCluster, rdsCluster | -| lambda | appSync, cognitoUserPool, kms, secretsManager, securityGroup, subnet, vpc, iamRole | +| lambda | appSync, cognitoUserPool, kms, s3, secretsManager, securityGroup, subnet, vpc, iamRole | | managedAirflow | iamRole, securityGroups, subnet, s3 | | nacl | vpc | | natGateway | networkInterface, subnet, vpc | @@ -147,12 +147,12 @@ CloudGraph AWS Provider will ask you what regions you would like to crawl and wi | sageMakerExperiment | | | sageMakerNotebookInstance | iamRole, kms, networkInterface, subnet, securityGroup | | sageMakerProject | | -| s3 | cloudfront, cloudtrail, ecsCluster, kinesisFirehose, managedAirflow | +| s3 | cloudfront, cloudtrail, ecsCluster, iamRole, kinesisFirehose, lambda, managedAirflow, sns, sqs | | secretsManager | kms, lambda | | securityGroup | alb, asg, clientVpnEndpoint, codebuild, dmsReplicationInstance, ecsService, lambda, ec2, elasticSearchDomain, elb, rdsCluster, rdsDbInstance, eksCluster, elastiCacheCluster, managedAirflow, sageMakerNotebookInstance | | ses | | -| sns | kms, cloudtrail, cloudwatch | -| sqs | | +| sns | kms, cloudtrail, cloudwatch, s3 | +| sqs | s3 | | subnet | alb, asg, codebuild, dmsReplicationInstance, ec2, ecsService, efsMountTarget, elastiCacheCluster, elasticSearchDomain, elb, lambda, managedAirflow, natGateway, networkInterface, sageMakerNotebookInstance, routeTable, vpc, eksCluster, emrCluster, flowLog | | systemsManagerInstance | ec2, iamRole | | systemsManagerDocument | | diff --git a/src/services/cloudfront/schema.graphql b/src/services/cloudfront/schema.graphql index 522f2aeb..deb2a342 100644 --- a/src/services/cloudfront/schema.graphql +++ b/src/services/cloudfront/schema.graphql @@ -18,7 +18,7 @@ type awsCloudfront implements awsBaseService @key(fields: "id") { origins: [awsCloudfrontOriginData] logging: awsCloudfrontLoggingConfig elb: [awsElb] @hasInverse(field: cloudfrontDistribution) - s3: [awsS3] @hasInverse(field: cloudfrontDistribution) + s3: [awsS3] @hasInverse(field: cloudfrontDistributions) tags: [awsRawTag] webAcl: [awsWafV2WebAcl] @hasInverse(field: cloudfront) } diff --git a/src/services/cloudtrail/schema.graphql b/src/services/cloudtrail/schema.graphql index 250a16cb..902095f6 100644 --- a/src/services/cloudtrail/schema.graphql +++ b/src/services/cloudtrail/schema.graphql @@ -15,7 +15,7 @@ type awsCloudtrail implements awsOptionalService @key(fields: "id") { status: awsCloudtrailStatus eventSelectors: [awsCloudtrailEventSelector] tags: [awsRawTag] - s3: [awsS3] @hasInverse(field: cloudtrail) + s3: [awsS3] @hasInverse(field: cloudtrails) sns: [awsSns] @hasInverse(field: cloudtrail) kms: [awsKms] @hasInverse(field: cloudtrail) cloudwatchLog: [awsCloudwatchLog] @hasInverse(field: cloudtrail) diff --git a/src/services/iamRole/schema.graphql b/src/services/iamRole/schema.graphql index cedfe10e..97c75f36 100644 --- a/src/services/iamRole/schema.graphql +++ b/src/services/iamRole/schema.graphql @@ -22,6 +22,7 @@ type awsIamRole implements awsBaseService @key(fields: "id") { systemsManagerInstances: [awsSystemsManagerInstance] @hasInverse(field: iamRole) iamInstanceProfiles: [awsIamInstanceProfile] @hasInverse(field: iamRole) + s3: [awsS3] @hasInverse(field: iamRole) dynamodb: [awsDynamoDbTable] @hasInverse(field: iamRoles) ec2Instances: [awsEc2] @hasInverse(field: iamRole) cognitoUserPools: [awsCognitoUserPool] @hasInverse(field: iamRole) diff --git a/src/services/lambda/schema.graphql b/src/services/lambda/schema.graphql index 8ed7b52a..c172d4cd 100644 --- a/src/services/lambda/schema.graphql +++ b/src/services/lambda/schema.graphql @@ -21,6 +21,7 @@ type awsLambda implements awsBaseService @key(fields: "arn") { vpc: [awsVpc] @hasInverse(field: lambda) cognitoUserPools: [awsCognitoUserPool] @hasInverse(field: lambdas) appSync: [awsAppSync] @hasInverse(field: lambda) + s3: [awsS3] @hasInverse(field: lambdas) secretsManager: [awsSecretsManager] @hasInverse(field: lambda) iamRole: [awsIamRole] @hasInverse(field: lambda) } diff --git a/src/services/s3/connections.ts b/src/services/s3/connections.ts new file mode 100644 index 00000000..4f927cc2 --- /dev/null +++ b/src/services/s3/connections.ts @@ -0,0 +1,151 @@ +import { ServiceConnection } from '@cloudgraph/sdk' +import isEmpty from 'lodash/isEmpty' +import services from '../../enums/services' +import { RawAwsS3 } from './data' +import { RawAwsIamRole } from '../iamRole/data' +import { RawAwsLambdaFunction } from '../lambda/data' +import { RawAwsSns } from '../sns/data' +import { AwsSqs } from '../sqs/data' +import { globalRegionName } from '../../enums/regions' + +/** + * S3 + */ + +export default ({ + service, + data, + region, +}: { + data: { name: string; data: { [property: string]: any[] } }[] + service: RawAwsS3 + region: string +}): { [key: string]: ServiceConnection[] } => { + const connections: ServiceConnection[] = [] + + const { + Id: id, + AdditionalInfo: { + ReplicationConfig: replicationConfig, + NotificationConfiguration: { + LambdaFunctionConfigurations: lambdaFunctionConfigurations, + TopicConfigurations: topicConfigurations, + QueueConfigurations: queueConfigurations, + }, + }, + } = service + + /** + * Find IAM Roles + * related to this S3 + */ + const roles: { name: string; data: { [property: string]: any[] } } = + data.find(({ name }) => name === services.iamRole) + if (replicationConfig?.Role && roles?.data?.[globalRegionName]) { + const dataAtRegion: RawAwsIamRole[] = roles.data[globalRegionName].filter( + role => role.Arn === replicationConfig.Role + ) + if (!isEmpty(dataAtRegion)) { + for (const instance of dataAtRegion) { + const { Arn: arn }: RawAwsIamRole = instance + + connections.push({ + id: arn, + resourceType: services.iamRole, + relation: 'child', + field: 'iamRole', + }) + } + } + } + + /** + * Find lambda functions + * related to this S3 + */ + const lambdaFunctions: { + name: string + data: { [property: string]: any[] } + } = data.find(({ name }) => name === services.lambda) + + const functionArns = lambdaFunctionConfigurations?.map( + lambdaConfig => lambdaConfig?.LambdaFunctionArn + ) + + if (!isEmpty(functionArns) && lambdaFunctions?.data?.[region]) { + const dataAtRegion: RawAwsLambdaFunction[] = lambdaFunctions.data[ + region + ].filter(({ FunctionArn }: RawAwsLambdaFunction) => + functionArns.includes(FunctionArn) + ) + + if (!isEmpty(dataAtRegion)) { + for (const lambdaFunction of dataAtRegion) { + const { FunctionArn: functionArn }: RawAwsLambdaFunction = + lambdaFunction + connections.push({ + id: functionArn, + resourceType: services.lambda, + relation: 'child', + field: 'lambdas', + }) + } + } + } + + /** + * Find SNS topic + * related to this S3 + */ + const snsTopics = data.find(({ name }) => name === services.sns) + const topicArns = topicConfigurations?.map(topic => topic?.TopicArn) + if (!isEmpty(topicArns) && snsTopics?.data?.[region]) { + const snsTopicsInRegion: RawAwsSns[] = snsTopics.data[region].filter( + ({ TopicArn: topicArn }: RawAwsSns) => topicArns.includes(topicArn) + ) + + if (!isEmpty(snsTopicsInRegion)) { + for (const topic of snsTopicsInRegion) { + const { TopicArn: topicArn }: RawAwsSns = topic + connections.push({ + id: topicArn, + resourceType: services.sns, + relation: 'child', + field: 'sns', + }) + } + } + } + + /** + * Find SQS + * related to this S3 + */ + const sqsQueues = data.find(({ name }) => name === services.sqs) + const sqsArns = queueConfigurations?.map(queue => queue?.QueueArn) + if (!isEmpty(sqsArns) && sqsQueues?.data?.[region]) { + const dataAtRegion: AwsSqs[] = sqsQueues.data[region].filter( + ({ sqsAttributes: { QueueArn: queueArn } }: AwsSqs) => + sqsArns.includes(queueArn) + ) + if (!isEmpty(dataAtRegion)) { + for (const instance of dataAtRegion) { + const { + sqsAttributes: { QueueArn: queueArn }, + }: AwsSqs = instance + + connections.push({ + id: queueArn, + resourceType: services.sqs, + relation: 'child', + field: 'sqs', + }) + } + } + } + + const s3Result = { + [id]: connections, + } + return s3Result +} diff --git a/src/services/s3/data.ts b/src/services/s3/data.ts index f05c8c6e..7262a6e3 100644 --- a/src/services/s3/data.ts +++ b/src/services/s3/data.ts @@ -30,6 +30,7 @@ import S3, { ListBucketsOutput, ListObjectsV2Output, LoggingEnabled, + NotificationConfiguration, Object as S3Object, Owner, Payer, @@ -301,6 +302,29 @@ const getBucketWebsite = async (s3: S3, name: BucketName) => ) }) +const getBucketNotificationConfiguration = async (s3: S3, name: BucketName) => + new Promise(resolve => { + s3.getBucketNotificationConfiguration( + { + Bucket: name, + }, + (err: AWSError, data: NotificationConfiguration) => { + if (err) { + errorLog.generateAwsErrorLog({ + functionName: 's3:getBucketNotificationConfiguration', + err, + }) + } + + if (!isEmpty(data)) { + resolve(data) + } + + resolve({}) + } + ) + }) + const getBucketAdditionalInfo = async (s3: S3, name: BucketName) => new Promise(async resolve => { const promises = [ @@ -318,6 +342,7 @@ const getBucketAdditionalInfo = async (s3: S3, name: BucketName) => getBucketTagging(s3, name), getBucketVersioning(s3, name), getBucketWebsite(s3, name), + getBucketNotificationConfiguration(s3, name), ] const [ @@ -335,6 +360,7 @@ const getBucketAdditionalInfo = async (s3: S3, name: BucketName) => Tags, VersioningInfo, WebsiteInfo, + NotificationConfig, ] = (await Promise.allSettled(promises)).map( /** We force the PromiseFulfilledResult interface * because all promises that we input to Promise.allSettled @@ -364,6 +390,7 @@ const getBucketAdditionalInfo = async (s3: S3, name: BucketName) => Tags: convertAwsTagsToTagMap(Tags), VersioningInfo, StaticWebsiteInfo: WebsiteInfo, + NotificationConfiguration: NotificationConfig, }) }) @@ -455,6 +482,7 @@ export interface RawAwsS3 { ReqPaymentConfig: Payer StaticWebsiteInfo?: GetBucketWebsiteOutput VersioningInfo?: GetBucketVersioningOutput + NotificationConfiguration?: NotificationConfiguration } Tags: TagMap Contents?: S3Object[] diff --git a/src/services/s3/format.ts b/src/services/s3/format.ts index 126bdb13..5377964e 100644 --- a/src/services/s3/format.ts +++ b/src/services/s3/format.ts @@ -3,6 +3,7 @@ import isEmpty from 'lodash/isEmpty' import { GetBucketVersioningOutput, + NotificationConfiguration, Policy, PolicyStatus, PublicAccessBlockConfiguration, @@ -47,6 +48,7 @@ export default ({ ReqPaymentConfig: reqPaymentConfig, StaticWebsiteInfo: staticWebsiteInfo, VersioningInfo: versioningInfo, + NotificationConfiguration: notificationConfiguration, } = { AccelerationConfig: '', BucketOwnerData: { DisplayName: '' }, @@ -62,6 +64,7 @@ export default ({ ReqPaymentConfig: '', StaticWebsiteInfo: {}, VersioningInfo: {}, + NotificationConfiguration: {}, }, } = rawData @@ -172,6 +175,52 @@ export default ({ }) } + let notificationConfigurationData = { + topicConfigurations: [], + queueConfigurations: [], + lambdaFunctionConfigurations: [], + } + + if (!isEmpty(notificationConfiguration)) { + const { + TopicConfigurations: topicConfigurations = [], + QueueConfigurations: queueConfigurations = [], + LambdaFunctionConfigurations: lambdaFunctionConfigurations = [], + }: NotificationConfiguration = notificationConfiguration + notificationConfigurationData = { + topicConfigurations: topicConfigurations?.map(tc => ({ + id: tc.Id || cuid(), + topicArn: tc.TopicArn, + events: tc.Events || [], + filterRules: tc.Filter?.Key?.FilterRules?.map(r => ({ + id: cuid(), + name: r.Name, + value: r.Value, + })) || [], + })) || [], + queueConfigurations: queueConfigurations?.map(qc => ({ + id: qc.Id || cuid(), + queueArn: qc.QueueArn, + events: qc.Events || [], + filterRules: qc.Filter?.Key?.FilterRules?.map(r => ({ + id: cuid(), + name: r.Name, + value: r.Value, + })) || [], + })) || [], + lambdaFunctionConfigurations: lambdaFunctionConfigurations?.map(lc => ({ + id: lc.Id || cuid(), + lambdaFunctionArn: lc.LambdaFunctionArn, + events: lc.Events || [], + filterRules: lc.Filter?.Key?.FilterRules?.map(r => ({ + id: cuid(), + name: r.Name, + value: r.Value, + })) || [], + })) || [], + } + } + // // Format S3 Tags const s3Tags = formatTagsFromMap(tags) @@ -201,6 +250,7 @@ export default ({ ? `${awsBucketItemsLimit}+` : `${total}`, transferAcceleration: accelerationStatus, + notificationConfiguration: notificationConfigurationData, } return s3 } diff --git a/src/services/s3/index.ts b/src/services/s3/index.ts index dfa1a981..7fb03148 100644 --- a/src/services/s3/index.ts +++ b/src/services/s3/index.ts @@ -2,6 +2,7 @@ import { Service } from '@cloudgraph/sdk' import BaseService from '../base' import format from './format' import getData from './data' +import getConnections from './connections' import mutation from './mutation' export default class S3 extends BaseService implements Service { @@ -9,5 +10,7 @@ export default class S3 extends BaseService implements Service { getData = getData.bind(this) + getConnections = getConnections.bind(this) + mutation = mutation } diff --git a/src/services/s3/schema.graphql b/src/services/s3/schema.graphql index edc266f0..823cb7c2 100644 --- a/src/services/s3/schema.graphql +++ b/src/services/s3/schema.graphql @@ -1,3 +1,58 @@ +type awsBucketPolicy + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) { + id: String! @id + policy: awsIamJSONPolicy +} + +type awsS3FilterRule + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash]) + value: String @search(by: [hash]) +} + +interface awsS3ConfigurationBase + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) { + id: String! @id + events: [String] @search(by: [hash]) + filterRules: [awsS3FilterRule] +} + +type awsS3TopicConfiguration implements awsS3ConfigurationBase { + topicArn: String @search(by: [hash]) +} + +type awsS3QueueConfiguration implements awsS3ConfigurationBase { + queueArn: String @search(by: [hash]) +} + +type awsS3LambdaFunctionConfiguration implements awsS3ConfigurationBase { + lambdaFunctionArn: String @search(by: [hash]) +} + +type awsS3NotificationConfiguration + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) { + topicConfigurations: [awsS3TopicConfiguration] + queueConfigurations: [awsS3QueueConfiguration] + lambdaFunctionConfigurations: [awsS3LambdaFunctionConfiguration] +} + type awsS3 implements awsBaseService @key(fields: "arn") { access: String @search(by: [hash, regexp]) bucketOwnerName: String @search(by: [hash, regexp]) @@ -17,23 +72,16 @@ type awsS3 implements awsBaseService @key(fields: "arn") { mfa: String @search(by: [hash, regexp]) versioning: String @search(by: [hash, regexp]) staticWebsiteHosting: String @search(by: [hash, regexp]) + notificationConfiguration: awsS3NotificationConfiguration bucketPolicies: [awsBucketPolicy] kinesisFirehose: [awsKinesisFirehose] @hasInverse(field: s3) tags: [awsRawTag] - cloudfrontDistribution: [awsCloudfront] @hasInverse(field: s3) #change to plural - cloudtrail: [awsCloudtrail] @hasInverse(field: s3) #change to plural + cloudfrontDistributions: [awsCloudfront] @hasInverse(field: s3) + cloudtrails: [awsCloudtrail] @hasInverse(field: s3) managedAirflows: [awsManagedAirflow] @hasInverse(field: s3) + iamRole: [awsIamRole] @hasInverse(field: s3) + lambdas: [awsLambda] @hasInverse(field: s3) + sns: [awsSns] @hasInverse(field: s3) + sqs: [awsSqs] @hasInverse(field: s3) ecsCluster: [awsEcsCluster] @hasInverse(field: s3) } - -# TODO: use getBucketReplication and getBucketNotificationConfiguration to make connections to lambda, sns, iamRole, SQS - -type awsBucketPolicy - @generate( - query: { get: false, query: true, aggregate: false } - mutation: { add: false, delete: false } - subscription: false - ) { - id: String! @id - policy: awsIamJSONPolicy -} diff --git a/src/services/sns/schema.graphql b/src/services/sns/schema.graphql index 64d729f1..b3c38de6 100644 --- a/src/services/sns/schema.graphql +++ b/src/services/sns/schema.graphql @@ -8,6 +8,7 @@ type awsSns implements awsBaseService @key(fields: "arn") { kms: [awsKms] @hasInverse(field: sns) cloudwatch: [awsCloudwatch] @hasInverse(field: sns) cloudFormationStack: [awsCloudFormationStack] @hasInverse(field: sns) + s3: [awsS3] @hasInverse(field: sns) } type awsSnsSubscription diff --git a/src/services/sqs/schema.graphql b/src/services/sqs/schema.graphql index 5825164a..12f96002 100644 --- a/src/services/sqs/schema.graphql +++ b/src/services/sqs/schema.graphql @@ -18,4 +18,5 @@ type awsSqs implements awsBaseService @key(fields: "arn") { fifoThroughputLimit: String @search(by: [hash, regexp]) contentBasedDeduplication: Boolean @search tags: [awsRawTag] + s3: [awsS3] @hasInverse(field: sqs) } diff --git a/src/types/generated.ts b/src/types/generated.ts index dc47ec3b..fbf39425 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -3074,6 +3074,7 @@ export type AwsIamRole = AwsBaseService & { path?: Maybe; rdsClusterIamRoles?: Maybe>>; rdsClusterMonitoringRole?: Maybe>>; + s3?: Maybe>>; sageMakerNotebookInstances?: Maybe>>; systemsManagerInstances?: Maybe>>; tags?: Maybe>>; @@ -3241,6 +3242,7 @@ export type AwsLambda = AwsBaseService & { policyRevisionId?: Maybe; reservedConcurrentExecutions?: Maybe; runtime?: Maybe; + s3?: Maybe>>; secretsManager?: Maybe>>; securityGroups?: Maybe>>; sourceCodeSize?: Maybe; @@ -3704,21 +3706,26 @@ export type AwsS3 = AwsBaseService & { blockPublicPolicy?: Maybe; bucketOwnerName?: Maybe; bucketPolicies?: Maybe>>; - cloudfrontDistribution?: Maybe>>; - cloudtrail?: Maybe>>; + cloudfrontDistributions?: Maybe>>; + cloudtrails?: Maybe>>; corsConfiguration?: Maybe; crossRegionReplication?: Maybe; ecsCluster?: Maybe>>; encrypted?: Maybe; + iamRole?: Maybe>>; ignorePublicAcls?: Maybe; kinesisFirehose?: Maybe>>; + lambdas?: Maybe>>; lifecycle?: Maybe; logging?: Maybe; managedAirflows?: Maybe>>; mfa?: Maybe; + notificationConfiguration?: Maybe; requesterPays?: Maybe; restrictPublicBuckets?: Maybe; size?: Maybe; + sns?: Maybe>>; + sqs?: Maybe>>; staticWebsiteHosting?: Maybe; tags?: Maybe>>; totalNumberOfObjectsInBucket?: Maybe; @@ -3726,6 +3733,36 @@ export type AwsS3 = AwsBaseService & { versioning?: Maybe; }; +export type AwsS3ConfigurationBase = { + events?: Maybe>>; + filterRules?: Maybe>>; + id: Scalars['String']; +}; + +export type AwsS3FilterRule = { + id: Scalars['String']; + name?: Maybe; + value?: Maybe; +}; + +export type AwsS3LambdaFunctionConfiguration = AwsS3ConfigurationBase & { + lambdaFunctionArn?: Maybe; +}; + +export type AwsS3NotificationConfiguration = { + lambdaFunctionConfigurations?: Maybe>>; + queueConfigurations?: Maybe>>; + topicConfigurations?: Maybe>>; +}; + +export type AwsS3QueueConfiguration = AwsS3ConfigurationBase & { + queueArn?: Maybe; +}; + +export type AwsS3TopicConfiguration = AwsS3ConfigurationBase & { + topicArn?: Maybe; +}; + export type AwsSageMakerExperiment = AwsBaseService & { creationTime?: Maybe; displayName?: Maybe; @@ -3899,6 +3936,7 @@ export type AwsSns = AwsBaseService & { displayName?: Maybe; kms?: Maybe>>; policy?: Maybe; + s3?: Maybe>>; subscriptions?: Maybe>>; tags?: Maybe>>; }; @@ -3927,6 +3965,7 @@ export type AwsSqs = AwsBaseService & { queueType?: Maybe; queueUrl?: Maybe; receiveMessageWaitTimeSeconds?: Maybe; + s3?: Maybe>>; sqsManagedSseEnabled?: Maybe; tags?: Maybe>>; visibilityTimeout?: Maybe;