diff --git a/awsautoenableS3Logging/packaged.yaml b/awsautoenableS3Logging/packaged.yaml index 0485124..95d6fe1 100644 --- a/awsautoenableS3Logging/packaged.yaml +++ b/awsautoenableS3Logging/packaged.yaml @@ -24,10 +24,10 @@ Metadata: - s3logging - flowlogs Name: sumologic-s3-logging-auto-enable - SemanticVersion: 1.0.0 + SemanticVersion: 1.0.2 SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/awsautoenableS3Logging - LicenseUrl: s3://appdevstore/AutoEnableS3Logs/v1.0.0/978602b5b9ec16f8bab0e38fd6b3998f - ReadmeUrl: s3://appdevstore/AutoEnableS3Logs/v1.0.0/d05d411471e0bb4db3389f2523f515f0 + LicenseUrl: s3://appdevstore/AutoEnableS3Logs/v1.0.2/978602b5b9ec16f8bab0e38fd6b3998f + ReadmeUrl: s3://appdevstore/AutoEnableS3Logs/v1.0.2/d05d411471e0bb4db3389f2523f515f0 SpdxLicenseId: Apache-2.0 Mappings: Region2ELBAccountId: @@ -92,7 +92,7 @@ Parameters: - S3 - VPC - ALB - AutoTaggingResourceOptions: + AutoEnableResourceOptions: Type: String Description: New - Automatically enables S3 logging for newly created AWS resources to send logs to S3 Buckets. This does not affect AWS resources already collecting @@ -129,10 +129,6 @@ Parameters: Description: True - To remove S3 logging or Vpc flow logs. False - To keep the S3 logging. Type: String - ParentStackName: - Type: String - Default: ParentStackName - Description: Parent Stack Name. Do Not Edit the value. Conditions: enable_alb_logging: Fn::And: @@ -155,25 +151,19 @@ Conditions: auto_enable_existing: Fn::Or: - Fn::Equals: - - Ref: AutoTaggingResourceOptions + - Ref: AutoEnableResourceOptions - Existing - Fn::Equals: - - Ref: AutoTaggingResourceOptions + - Ref: AutoEnableResourceOptions - Both auto_enable_new: Fn::Or: - Fn::Equals: - - Ref: AutoTaggingResourceOptions + - Ref: AutoEnableResourceOptions - New - Fn::Equals: - - Ref: AutoTaggingResourceOptions + - Ref: AutoEnableResourceOptions - Both - do_not_use_parent_stack: - Fn::And: - - Fn::Equals: - - Ref: ParentStackName - - ParentStackName - - Condition: auto_enable_existing Resources: SumoLambdaRole: Type: AWS::IAM::Role @@ -223,20 +213,7 @@ Resources: Type: AWS::Serverless::Function Condition: auto_enable_new Properties: - FunctionName: - Fn::Join: - - '' - - - sumo-s3-enable- - - Fn::Select: - - 0 - - Fn::Split: - - '-' - - Fn::Select: - - 2 - - Fn::Split: - - / - - Ref: AWS::StackId - CodeUri: s3://appdevstore/sumo_app_utils/v2.0.0/sumo_app_utils.zip + CodeUri: s3://appdevstore/sumo_app_utils/v2.0.2/sumo_app_utils.zip Handler: awsresource.enable_s3_logs Runtime: python3.7 Role: @@ -404,11 +381,11 @@ Resources: Id: Main EnableExisitngAWSResourcesLambda: Type: AWS::Serverless::Function - Condition: do_not_use_parent_stack + Condition: auto_enable_existing Properties: Handler: main.handler Runtime: python3.7 - CodeUri: s3://appdevstore/sumo_app_utils/v2.0.0/sumo_app_utils.zip + CodeUri: s3://appdevstore/sumo_app_utils/v2.0.2/sumo_app_utils.zip MemorySize: 128 Timeout: 900 Role: @@ -420,13 +397,9 @@ Resources: Condition: auto_enable_existing Properties: ServiceToken: - Fn::If: - - do_not_use_parent_stack - - Fn::GetAtt: - - EnableExisitngAWSResourcesLambda - - Arn - - Fn::ImportValue: - Fn::Sub: ${ParentStackName}-SumoLogicHelperFunctionARN + Fn::GetAtt: + - EnableExisitngAWSResourcesLambda + - Arn AWSResource: Fn::If: - enable_s3_buckets_logging @@ -464,4 +437,4 @@ Outputs: Fn::GetAtt: - EnableExisitngAWSResourcesLambda - Arn - Condition: do_not_use_parent_stack + Condition: auto_enable_existing diff --git a/awsautoenableS3Logging/sumologic-s3-logging-auto-enable.yaml b/awsautoenableS3Logging/sumologic-s3-logging-auto-enable.yaml index 295cd9d..a1f7edc 100755 --- a/awsautoenableS3Logging/sumologic-s3-logging-auto-enable.yaml +++ b/awsautoenableS3Logging/sumologic-s3-logging-auto-enable.yaml @@ -24,7 +24,7 @@ Metadata: - s3logging - flowlogs Name: sumologic-s3-logging-auto-enable - SemanticVersion: 1.0.0 + SemanticVersion: 1.0.2 SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/awsautoenableS3Logging LicenseUrl: ./LICENSE ReadmeUrl: ./README.md @@ -95,7 +95,7 @@ Parameters: - 'VPC' - 'ALB' - AutoTaggingResourceOptions: + AutoEnableResourceOptions: Type: String Description: "New - Automatically enables S3 logging for newly created AWS resources to send logs to S3 Buckets. This does not affect AWS resources already collecting logs. Existing - Automatically enables S3 logging for existing AWS resources to send logs to S3 Buckets. @@ -132,11 +132,6 @@ Parameters: False - To keep the S3 logging." Type: String - ParentStackName: - Type: String - Default: "ParentStackName" - Description: Parent Stack Name. Do Not Edit the value. - Conditions: enable_alb_logging: !And - !Equals [!Ref AutoEnableLogging, 'ALB'] @@ -149,15 +144,11 @@ Conditions: - !Condition auto_enable_new auto_enable_existing: !Or - - !Equals [ !Ref AutoTaggingResourceOptions, 'Existing' ] - - !Equals [ !Ref AutoTaggingResourceOptions, 'Both' ] + - !Equals [ !Ref AutoEnableResourceOptions, 'Existing' ] + - !Equals [ !Ref AutoEnableResourceOptions, 'Both' ] auto_enable_new: !Or - - !Equals [ !Ref AutoTaggingResourceOptions, 'New' ] - - !Equals [ !Ref AutoTaggingResourceOptions, 'Both' ] - - do_not_use_parent_stack: !And - - !Equals [ !Ref ParentStackName, "ParentStackName"] - - !Condition auto_enable_existing + - !Equals [ !Ref AutoEnableResourceOptions, 'New' ] + - !Equals [ !Ref AutoEnableResourceOptions, 'Both' ] Resources: @@ -210,17 +201,7 @@ Resources: Type: 'AWS::Serverless::Function' Condition: auto_enable_new Properties: - FunctionName: !Join - - "" - - - "sumo-s3-enable-" - - !Select - - 0 - - !Split - - "-" - - !Select - - 2 - - !Split ["/", !Ref "AWS::StackId"] - CodeUri: s3://appdevstore/sumo_app_utils/v2.0.0/sumo_app_utils.zip + CodeUri: s3://appdevstore/sumo_app_utils/v2.0.2/sumo_app_utils.zip Handler: "awsresource.enable_s3_logs" Runtime: python3.7 Role: !GetAtt SumoLambdaRole.Arn @@ -354,11 +335,11 @@ Resources: EnableExisitngAWSResourcesLambda: Type: 'AWS::Serverless::Function' - Condition: do_not_use_parent_stack + Condition: auto_enable_existing Properties: Handler: main.handler Runtime: python3.7 - CodeUri: s3://appdevstore/sumo_app_utils/v2.0.0/sumo_app_utils.zip + CodeUri: s3://appdevstore/sumo_app_utils/v2.0.2/sumo_app_utils.zip MemorySize: 128 Timeout: 900 Role: @@ -370,13 +351,7 @@ Resources: Type: Custom::EnableS3LogsResources Condition: auto_enable_existing Properties: - ServiceToken: !If - - do_not_use_parent_stack - - !GetAtt - - EnableExisitngAWSResourcesLambda - - Arn - - !ImportValue - 'Fn::Sub': '${ParentStackName}-SumoLogicHelperFunctionARN' + ServiceToken: !GetAtt EnableExisitngAWSResourcesLambda.Arn AWSResource: !If [enable_s3_buckets_logging, "s3", !If [enable_vpc_flow_logs_logging, "vpc", "elbv2"] ] BucketName: !Ref BucketName Filter: !Ref FilterExpression @@ -395,4 +370,4 @@ Outputs: EnableExisitngAWSResourcesLambda: Description: "Lambda Function ARN for Existing AWS Resources" Value: !GetAtt EnableExisitngAWSResourcesLambda.Arn - Condition: do_not_use_parent_stack \ No newline at end of file + Condition: auto_enable_existing \ No newline at end of file diff --git a/awsautoenableS3Logging/test/TestTemplate.sh b/awsautoenableS3Logging/test/TestTemplate.sh index f50a4e5..f7c43cf 100644 --- a/awsautoenableS3Logging/test/TestTemplate.sh +++ b/awsautoenableS3Logging/test/TestTemplate.sh @@ -1,10 +1,10 @@ #!/bin/sh -export AWS_REGION="ap-south-1" +export AWS_REGION="us-east-1" export AWS_PROFILE="personal" # App to test export AppName="tag" -export InstallTypes=("s3" "s3exiting" "vpc" "vpcexisting" "alb" "albexisting") +export InstallTypes=("s3both" "vpcboth" "albboth") export BucketName="sumologiclambdahelper-${AWS_REGION}" export FilterExpression=".*" @@ -15,29 +15,42 @@ do if [[ "${InstallType}" == "s3" ]] then - export EnableLogging="S3" - export TaggingResourceOptions="New" + export AutoEnableLogging="S3" + export AutoEnableResourceOptions="New" elif [[ "${InstallType}" == "s3exiting" ]] then - export EnableLogging="S3" - export TaggingResourceOptions="Existing" + export AutoEnableLogging="S3" + export AutoEnableResourceOptions="Existing" + elif [[ "${InstallType}" == "s3both" ]] + then + export AutoEnableLogging="S3" + export AutoEnableResourceOptions="Both" elif [[ "${InstallType}" == "vpc" ]] then - export EnableLogging="VPC" - export TaggingResourceOptions="New" + export AutoEnableLogging="VPC" + export AutoEnableResourceOptions="New" elif [[ "${InstallType}" == "vpcexisting" ]] then - export EnableLogging="VPC" - export TaggingResourceOptions="Existing" + export AutoEnableLogging="VPC" + export AutoEnableResourceOptions="Existing" + elif [[ "${InstallType}" == "vpcboth" ]] + then + export AutoEnableLogging="VPC" + export AutoEnableResourceOptions="Both" elif [[ "${InstallType}" == "alb" ]] then - export EnableLogging="ALB" - export TaggingResourceOptions="New" + export AutoEnableLogging="ALB" + export AutoEnableResourceOptions="New" elif [[ "${InstallType}" == "albexisting" ]] then - export EnableLogging="ALB" - export TaggingResourceOptions="Existing" + export AutoEnableLogging="ALB" + export AutoEnableResourceOptions="Existing" export BucketPrefix=${InstallType}"-LOGS" + elif [[ "${InstallType}" == "albboth" ]] + then + export AutoEnableLogging="ALB" + export AutoEnableResourceOptions="Both" + export BucketPrefix=${InstallType}"-BOTH" else echo "No Valid Choice." fi @@ -45,11 +58,9 @@ do # Stack Name export stackName="${AppName}-${InstallType}" - aws cloudformation deploy --region ${AWS_REGION} --profile ${AWS_PROFILE} --template-file ././../auto_enable_s3_alb.template.yaml \ + aws cloudformation deploy --region ${AWS_REGION} --profile ${AWS_PROFILE} --template-file ./../sumologic-s3-logging-auto-enable.yaml \ --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND CAPABILITY_NAMED_IAM --stack-name "${AppName}-${InstallType}" \ - --parameter-overrides EnableLogging="${EnableLogging}" TaggingResourceOptions="${TaggingResourceOptions}" \ + --parameter-overrides AutoEnableLogging="${AutoEnableLogging}" AutoEnableResourceOptions="${AutoEnableResourceOptions}" \ FilterExpression="${FilterExpression}" BucketName="${BucketName}" BucketPrefix="${BucketPrefix}" & - export ExistingResource="No" - done diff --git a/sumologic-app-utils/Test/SampleTemplate.yaml b/sumologic-app-utils/Test/SampleTemplate.yaml index 04dca44..858140c 100644 --- a/sumologic-app-utils/Test/SampleTemplate.yaml +++ b/sumologic-app-utils/Test/SampleTemplate.yaml @@ -41,281 +41,203 @@ Parameters: Deletes the resources created by the stack. Deletion of updated resources will be skipped." Type: String - CreateExplorerView: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - ExplorerName: - Type: String - Default: "" - ExplorerFields: - Type: CommaDelimitedList - Default: "" - - CreateMetricRule: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - MetricRuleName: - Type: String - Default: "" - MatchExpression: - Type: String - Default: "" - EntityRule: - Type: String - Default: "" - - CreateFER: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - FERName: - Type: String - Default: "" - FERScope: - Type: String - Default: "" - FERExpression: - Type: String - Default: "" - - InstallAPP: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - FolderName: - Type: String - Default: "" - - CreateCollector: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - - UpdateSource: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - - TagResources: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - - AutoEnableS3: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - - CreateDeliveryChannel: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - - CreateTrail: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - -Conditions: - - create_explorer: !Equals [!Ref CreateExplorerView, 'Yes'] - - create_metric_rule: !Equals [!Ref CreateMetricRule, 'Yes'] - - create_fer: !Equals [!Ref CreateFER, 'Yes'] - - install_app: !Equals [!Ref InstallAPP, 'Yes'] - - update_source: !Equals [!Ref UpdateSource, 'Yes'] - - tag_existing_aws_resources: !Equals [!Ref TagResources, 'Yes'] - - create_collector: !Equals [!Ref CreateCollector, 'Yes'] - - auto_enable_existing: !Equals [!Ref AutoEnableS3, 'Yes'] - - create_sns_topic: !Equals [!Ref CreateDeliveryChannel, 'Yes'] +Resources: - creaye_trail: !Equals [!Ref CreateTrail, 'Yes'] + SumoAppUtils: + Type: AWS::Serverless::Application + Properties: + Location: + ApplicationId: arn:aws:serverlessrepo:us-east-1:956882708938:applications/sumologic-app-utils + SemanticVersion: 2.0.3 -Resources: - SumoLogicHelperRole: - Type: AWS::IAM::Role + SumoLogicHelperPolicy: + Type: AWS::IAM::Policy Properties: - AssumeRolePolicyDocument: + PolicyName: SumoLogicLambdaPolicy + Roles: + - !Select ["1", !Split ["/", !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunctionRole]] + PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow - Principal: - Service: lambda.amazonaws.com - Action: sts:AssumeRole - Path: / - Policies: - - PolicyName: AwsObservabilityLambdaExecutePolicies - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - tag:TagResources - - tag:UntagResources - - rds:DescribeDBClusters - - rds:DescribeDBInstances - - rds:AddTagsToResource - - rds:RemoveTagsFromResource - - apigateway:GET - - apigateway:DELETE - - apigateway:POST - - apigateway:PUT - - ec2:DescribeInstances - - ec2:CreateTags - - ec2:DeleteTags - - elasticloadbalancing:DescribeLoadBalancerAttributes - - elasticloadbalancing:DescribeLoadBalancers - - elasticloadbalancing:AddTags - - elasticloadbalancing:RemoveTags - - elasticloadbalancing:ModifyLoadBalancerAttributes - - lambda:ListFunctions - - lambda:TagResource - - lambda:UntagResource - - dynamodb:ListTables - - dynamodb:TagResource - - dynamodb:UntagResource - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - - cloudtrail:CreateTrail - - cloudtrail:DeleteTrail - - cloudtrail:UpdateTrail - - cloudtrail:StartLogging - - s3:GetBucketPolicy - - s3:PutBucketPolicy - - s3:ListAllMyBuckets - - s3:GetBucketLocation - - s3:PutBucketAcl - - s3:GetBucketAcl - - s3:GetBucketLogging - - s3:PutBucketLogging - - config:DescribeDeliveryChannels - - config:PutDeliveryChannel - - config:DeleteDeliveryChannel - - s3:GetBucketPolicy - - s3:PutBucketPolicy - - ec2:DescribeInstances - - ec2:DescribeVpcs - - ec2:DeleteFlowLogs - - ec2:CreateFlowLogs - - ec2:DescribeFlowLogs - - logs:CreateLogDelivery - - logs:DeleteLogDelivery - Resource: '*' - - SumoLogicHelperFunction: - Type: 'AWS::Serverless::Function' + Action: + - tag:TagResources + - tag:UntagResources + - rds:DescribeDBClusters + - rds:DescribeDBInstances + - rds:AddTagsToResource + - rds:RemoveTagsFromResource + - apigateway:GET + - apigateway:DELETE + - apigateway:POST + - apigateway:PUT + - ec2:DescribeInstances + - ec2:CreateTags + - ec2:DeleteTags + - elasticloadbalancing:DescribeLoadBalancerAttributes + - elasticloadbalancing:DescribeLoadBalancers + - elasticloadbalancing:AddTags + - elasticloadbalancing:RemoveTags + - elasticloadbalancing:ModifyLoadBalancerAttributes + - lambda:ListFunctions + - lambda:TagResource + - lambda:UntagResource + - dynamodb:ListTables + - dynamodb:TagResource + - dynamodb:UntagResource + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - cloudtrail:CreateTrail + - cloudtrail:DeleteTrail + - cloudtrail:UpdateTrail + - cloudtrail:StartLogging + - s3:GetBucketPolicy + - s3:PutBucketPolicy + - s3:ListAllMyBuckets + - s3:GetBucketLocation + - s3:PutBucketAcl + - s3:GetBucketAcl + - s3:GetBucketLogging + - s3:PutBucketLogging + - config:DescribeDeliveryChannels + - config:PutDeliveryChannel + - config:DeleteDeliveryChannel + - s3:GetBucketPolicy + - s3:PutBucketPolicy + - ec2:DescribeInstances + - ec2:DescribeVpcs + - ec2:DeleteFlowLogs + - ec2:CreateFlowLogs + - ec2:DescribeFlowLogs + - logs:CreateLogDelivery + - logs:DeleteLogDelivery + Resource: '*' + + SumoCloudTrailExportPolicy: + DependsOn: SumoLogicHelperPolicy + Type: AWS::S3::BucketPolicy Properties: - Handler: main.handler - Runtime: python3.7 - CodeUri: s3://sumologiclambdahelper-us-east-1/sumo_app_utils/v2.0.1/sumo_app_utils.zip - MemorySize: 128 - Timeout: 900 - Role: - Fn::GetAtt: - - SumoLogicHelperRole - - Arn + Bucket: "sumologiclambdahelper-us-east-2" + PolicyDocument: + Statement: + - Sid: AWSCloudTrailAclCheck + Effect: Allow + Principal: + Service: cloudtrail.amazonaws.com + Action: s3:GetBucketAcl + Resource: + - "arn:aws:s3:::sumologiclambdahelper-us-east-2" + - Sid: AWSCloudTrailWrite + Effect: Allow + Principal: + Service: cloudtrail.amazonaws.com + Action: s3:PutObject + Resource: + - "arn:aws:s3:::sumologiclambdahelper-us-east-2/*" + Condition: + StringEquals: + s3:x-amz-acl: bucket-owner-full-control + - Sid: AWSBucketExistenceCheck + Effect: Allow + Principal: + Service: cloudtrail.amazonaws.com + Action: s3:ListBucket + Resource: + - "arn:aws:s3:::sumologiclambdahelper-us-east-2" SumoCloudTrail: - Condition: creaye_trail Type: Custom::AWSTrail + DependsOn: SumoCloudTrailExportPolicy Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction IsLogging: true IsMultiRegionTrail: false - S3BucketName: "sumologiclambdahelper-us-east-1" + S3BucketName: "sumologiclambdahelper-us-east-2" TrailName: "Aws-Observability-onemoreupdae" + RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack - CreateSumoLogicAWSExplorerView: - Type: Custom::SumoLogicAWSExplorer - Condition: create_explorer + SumoHostedCollector: + Type: Custom::Collector Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction + Region: !Ref "AWS::Region" + CollectorType: Hosted RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack - ExplorerName: !Ref ExplorerName - MetadataKeys: !Ref ExplorerFields + CollectorName: "SourabhTestingCollector" + SourceCategory: "testingSource" + Description: "This is again Updated collector" SumoAccessID: !Ref Section1bSumoAccessID SumoAccessKey: !Ref Section1cSumoAccessKey SumoDeployment: !Ref Section1aSumoDeployment - TagAWSResources: - Type: Custom::TagAWSResources - Condition: tag_existing_aws_resources + SumoHTTPSource: + Type: Custom::HTTPSource Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn - RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction Region: !Ref "AWS::Region" - AWSResource: dynamodb - Tags: - mili: "testingagain" - AccountID: !Ref "AWS::AccountId" - Filter: ".*" - - APIGatewayMetricRule: - Type: Custom::SumoLogicMetricRules - Condition: create_metric_rule - Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn + SourceName: "SourabhTestingHTTPSource" RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack - MetricRuleName: !Ref MetricRuleName - MatchExpression: !Ref MatchExpression - ExtractVariables: - entity: !Ref EntityRule + SourceCategory: "TestingCategory" + CollectorId: !GetAtt SumoHostedCollector.COLLECTOR_ID SumoAccessID: !Ref Section1bSumoAccessID SumoAccessKey: !Ref Section1cSumoAccessKey SumoDeployment: !Ref Section1aSumoDeployment - APIGatewayFieldExtractionRule: - Type: Custom::SumoLogicFieldExtractionRule - Condition: create_fer + SumoRole: + Type: AWS::IAM::Role Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + AWS: arn:aws:iam::926226587429:root + Action: sts:AssumeRole + Condition: + StringEquals: + sts:ExternalId: !Sub "${Section1aSumoDeployment}:${Section1dSumoOrganizationId}" + Path: "/" + Policies: + - PolicyName: SumoPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - s3:GetObject + - s3:GetObjectVersion + - s3:ListBucketVersions + - s3:ListBucket + Resource: + "*" + + SumoS3Source: + Type: Custom::AWSSource + Properties: + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction + SourceType: AwsS3Bucket + Region: !Ref "AWS::Region" + SourceName: "SourabhSource" RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack - FieldExtractionRuleName: !Ref FERName - FieldExtractionRuleScope: !Ref FERScope - FieldExtractionRuleParseExpression: !Ref FERExpression - FieldExtractionRuleParseEnabled: !Ref Section1eRemoveSumoResourcesOnDeleteStack + SourceCategory: "TestingSourceCategory" + CollectorId: !GetAtt SumoHostedCollector.COLLECTOR_ID SumoAccessID: !Ref Section1bSumoAccessID SumoAccessKey: !Ref Section1cSumoAccessKey SumoDeployment: !Ref Section1aSumoDeployment + TargetBucketName: "sumologiclambdahelper-us-east-1" + PathExpression: "asdasd" + DateFormat: "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" + DateLocatorRegex: '.*"updatedAt":"(.*)".*' + RoleArn: !GetAtt SumoRole.Arn sumoApp: Type: Custom::App - Condition: install_app Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction Region: !Ref "AWS::Region" AppName: "AWS Observability Api Gateway App" - FolderName: !Ref FolderName + FolderName: "My App without UUID" RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack AppSources: logsrc: "_sourceCategory=fasdfsdad" @@ -325,12 +247,12 @@ Resources: sumoAppwithUID: Type: Custom::App - Condition: install_app + DependsOn: sumoApp Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction Region: !Ref "AWS::Region" AppName: "Another App" - FolderName: !Ref FolderName + FolderName: "My App with UUID" AppId: "924d7e2a-a14a-4b11-8c91-133241be2a51" RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack AppSources: @@ -339,15 +261,40 @@ Resources: SumoAccessKey: !Ref Section1cSumoAccessKey SumoDeployment: !Ref Section1aSumoDeployment + CreateSumoLogicAWSExplorerView: + Type: Custom::SumoLogicAWSExplorer + Properties: + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction + RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack + ExplorerName: "Test EXPLORER" + MetadataKeys: + - "account" + - "region" + SumoAccessID: !Ref Section1bSumoAccessID + SumoAccessKey: !Ref Section1cSumoAccessKey + SumoDeployment: !Ref Section1aSumoDeployment + + APIGatewayMetricRule: + Type: Custom::SumoLogicMetricRules + Properties: + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction + RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack + MetricRuleName: "TestMetricRuleSumoAppUtils" + MatchExpression: "account=asacss FunctionName=*" + ExtractVariables: + entity: "$FunctionName._1" + SumoAccessID: !Ref Section1bSumoAccessID + SumoAccessKey: !Ref Section1cSumoAccessKey + SumoDeployment: !Ref Section1aSumoDeployment + SumoCloudTrailUpdateSource: Type: Custom::SumoLogicUpdateFields - Condition: update_source + DependsOn: SumoHTTPSource Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction Region: !Ref "AWS::Region" - SourceName: "SourabhTestingHTTPSource" + SourceApiUrl: "https://api.sumologic.com/api/v1/collectors/170503459/sources/779514196" RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack - CollectorName: "SourabhTestingCollector" SumoAccessID: !Ref Section1bSumoAccessID SumoAccessKey: !Ref Section1cSumoAccessKey SumoDeployment: !Ref Section1aSumoDeployment @@ -355,89 +302,56 @@ Resources: account: "fsdsd" region: "asfasf" - SumoRole: - Type: AWS::IAM::Role - Condition: create_collector - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - AWS: arn:aws:iam::926226587429:root - Action: sts:AssumeRole - Condition: - StringEquals: - sts:ExternalId: !Sub "${Section1aSumoDeployment}:${Section1dSumoOrganizationId}" - Path: "/" - Policies: - - PolicyName: SumoPolicy - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - s3:GetObject - - s3:GetObjectVersion - - s3:ListBucketVersions - - s3:ListBucket - Resource: - "*" - - SumoHostedCollector: - Condition: create_collector - Type: Custom::Collector + APIGatewayFieldExtractionRule: + Type: Custom::SumoLogicFieldExtractionRule Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn - Region: !Ref "AWS::Region" - CollectorType: Hosted + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack - CollectorName: "SourabhTestingCollector" - SourceCategory: "testingSource" - Description: "This is again Updated collector" + FieldExtractionRuleName: "TestFieldSumoAppUtils" + FieldExtractionRuleScope: "_source=asca" + FieldExtractionRuleParseExpression: "| json \"eventSource\", \"awsRegion\" as eventSource, region" + FieldExtractionRuleParseEnabled: !Ref Section1eRemoveSumoResourcesOnDeleteStack SumoAccessID: !Ref Section1bSumoAccessID SumoAccessKey: !Ref Section1cSumoAccessKey SumoDeployment: !Ref Section1aSumoDeployment - SumoS3Source: - Type: Custom::AWSSource - Condition: create_collector + AccCheck: + Type: Custom::EnterpriseOrTrialAccountCheck Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn - SourceType: AwsS3Bucket + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction Region: !Ref "AWS::Region" - SourceName: "SourabhSource" - RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack - SourceCategory: "TestingSourceCategory" - CollectorId: !GetAtt SumoHostedCollector.COLLECTOR_ID SumoAccessID: !Ref Section1bSumoAccessID SumoAccessKey: !Ref Section1cSumoAccessKey SumoDeployment: !Ref Section1aSumoDeployment - TargetBucketName: "sumologiclambdahelper-us-east-1" - PathExpression: "asdasd" - DateFormat: "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" - DateLocatorRegex: '.*"updatedAt":"(.*)".*' - RoleArn: !GetAtt SumoRole.Arn - SumoHTTPSource: - Condition: create_collector - Type: Custom::HTTPSource + AddAccountField: + Type: Custom::SumoLogicFieldsSchema Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn - Region: !Ref "AWS::Region" - SourceName: "SourabhTestingHTTPSource" - RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack - SourceCategory: "TestingCategory" - CollectorId: !GetAtt SumoHostedCollector.COLLECTOR_ID + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction + FieldName: "test_field" + RemoveOnDeleteStack: true SumoAccessID: !Ref Section1bSumoAccessID SumoAccessKey: !Ref Section1cSumoAccessKey SumoDeployment: !Ref Section1aSumoDeployment + TagAWSResources: + Type: Custom::TagAWSResources + DependsOn: SumoLogicHelperPolicy + Properties: + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction + RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack + Region: !Ref "AWS::Region" + AWSResource: dynamodb + Tags: + mili: "testingagain" + AccountID: !Ref "AWS::AccountId" + Filter: ".*" + ExistingAWSResources: Type: Custom::EnableS3LogsResources - Condition: auto_enable_existing + DependsOn: SumoLogicHelperPolicy Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn + ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction AWSResource: "vpc" BucketName: "sumologiclambdahelper-us-east-1" Filter: ".*" @@ -445,13 +359,10 @@ Resources: AccountID: !Ref "AWS::AccountId" RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack - DeliveryChannelWhenConfigIsTurnedOn: - Type: "Custom::ConfigDeliveryChannel" - Condition: create_sns_topic - Properties: - ServiceToken: !GetAtt SumoLogicHelperFunction.Arn - DeliveryFrequency: Six_Hours - S3BucketName: "sumologiclambdahelper-us-east-1" - S3KeyPrefix: "sourabh" - SnsTopicARN: "arn:aws:sns:us-east-1:668508221233:dynamodb" - RemoveOnDeleteStack: !Ref Section1eRemoveSumoResourcesOnDeleteStack +Outputs: + EnterpriseCheck: + Description: "Check If Account is Enterprise or Not" + Value: !GetAtt AccCheck.is_enterprise + PaidAccountCheck: + Description: "Check If Account is Paid or Not" + Value: !GetAtt AccCheck.is_paid \ No newline at end of file diff --git a/sumologic-app-utils/Test/TestTemplate.sh b/sumologic-app-utils/Test/TestTemplate.sh index 06b9de4..71e8c3c 100644 --- a/sumologic-app-utils/Test/TestTemplate.sh +++ b/sumologic-app-utils/Test/TestTemplate.sh @@ -16,63 +16,6 @@ export Section1cSumoAccessKey="" export Section1dSumoOrganizationId="" export Section1eRemoveSumoResourcesOnDeleteStack=true -export InstallAPP="No" -export CreateExplorerView="No" -export CreateMetricRule="No" -export CreateFER="No" -export UpdateSource="No" -export TagResources="No" -export CreateCollector="No" -export AutoEnableS3="No" -export CreateDeliveryChannel="No" -export CreateTrail="No" - -if [[ "${InstallType}" == "nothing" ]] -then - echo "Deploying nothing." -elif [[ "${InstallType}" == "installapp" ]] -then - export InstallAPP="Yes" - export FolderName="Sourabh Folder" -elif [[ "${InstallType}" == "installexplorerviews" ]] -then - export CreateExplorerView="Yes" - export ExplorerName="ExplorerNameMyNae" - export ExplorerFields="account,${uid_1},${uid_2}" -elif [[ "${InstallType}" == "metricrule" ]] -then - export CreateMetricRule="Yes" - export MetricRuleName="MyMetricRule" - export MatchExpression="account=${uid_1} ${uid_2}=*" - export EntityRule="\$${uid_2}._1" -elif [[ "${InstallType}" == "fer" ]] -then - export CreateFER="Yes" - export FERName="FieldExtraction${uid_1}" - export FERScope="account=${uid_1} namespace=${uid_2}" - export FERExpression="json \"${uid_1}\"" -elif [[ "${InstallType}" == "collector" ]] -then - # It includes source updates for both AWS and HTTP custom resource. - export CreateCollector="Yes" -elif [[ "${InstallType}" == "updatefields" ]] -then - export UpdateSource="Yes" -elif [[ "${InstallType}" == "tagresources" ]] -then - export TagResources="Yes" -elif [[ "${InstallType}" == "autoenable" ]] -then - export AutoEnableS3="Yes" -elif [[ "${InstallType}" == "createchannel" ]] -then - export CreateDeliveryChannel="Yes" -elif [[ "${InstallType}" == "createcloudtrail" ]] -then - export CreateTrail="Yes" -else - echo "No Valid Choice." -fi # Stack Name export stackName="${AppName}-${InstallType}" @@ -81,9 +24,4 @@ aws cloudformation deploy --profile ${AWS_PROFILE} --template-file ./SampleTempl --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND --stack-name ${stackName} \ --parameter-overrides Section1aSumoDeployment="${Section1aSumoDeployment}" Section1bSumoAccessID="${Section1bSumoAccessID}" \ Section1cSumoAccessKey="${Section1cSumoAccessKey}" Section1dSumoOrganizationId="${Section1dSumoOrganizationId}" \ -Section1eRemoveSumoResourcesOnDeleteStack="${Section1eRemoveSumoResourcesOnDeleteStack}" InstallAPP="${InstallAPP}" \ -FolderName="${FolderName}" ExplorerName="${ExplorerName}" ExplorerFields="${ExplorerFields}" MetricRuleName="${MetricRuleName}" \ -MatchExpression="${MatchExpression}" FERName="${FERName}" FERScope="${FERScope}" FERExpression="${FERExpression}" \ -CreateMetricRule="${CreateMetricRule}" CreateFER="${CreateFER}" CreateExplorerView="${CreateExplorerView}" EntityRule="${EntityRule}" \ -CreateCollector="${CreateCollector}" UpdateSource="${UpdateSource}" TagResources="${TagResources}" AutoEnableS3="${AutoEnableS3}" \ -CreateDeliveryChannel="${CreateDeliveryChannel}" CreateTrail="${CreateTrail}" \ No newline at end of file +Section1eRemoveSumoResourcesOnDeleteStack="${Section1eRemoveSumoResourcesOnDeleteStack}" \ No newline at end of file diff --git a/sumologic-app-utils/deploy.sh b/sumologic-app-utils/deploy.sh index 44dca70..31b9af3 100644 --- a/sumologic-app-utils/deploy.sh +++ b/sumologic-app-utils/deploy.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ "$AWS_PROFILE" == "prod" ] +if [ "$AWS_PROFILE" != "prod" ] then SAM_S3_BUCKET="appdevstore" AWS_REGION="us-east-1" @@ -26,7 +26,7 @@ if [ ! -f sumo_app_utils.zip ]; then rm -r python fi -version="2.0.1" +version="2.0.3" aws s3 cp sumo_app_utils.zip s3://$SAM_S3_BUCKET/sumo_app_utils/v"$version"/sumo_app_utils.zip --region $AWS_REGION --acl public-read diff --git a/sumologic-app-utils/packaged_sumo_app_utils.yaml b/sumologic-app-utils/packaged_sumo_app_utils.yaml index 05c022d..1c15533 100644 --- a/sumologic-app-utils/packaged_sumo_app_utils.yaml +++ b/sumologic-app-utils/packaged_sumo_app_utils.yaml @@ -20,17 +20,17 @@ Metadata: - sumologic - serverless Name: sumologic-app-utils - SemanticVersion: 2.0.1 + SemanticVersion: 2.0.3 SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/sumologic-app-utils SpdxLicenseId: Apache-2.0 - ReadmeUrl: s3://appdevstore/sumo_app_utils/v2.0.1/4d5a92c06a7fa9d956a900e51a1f6be4 + ReadmeUrl: s3://appdevstore/sumo_app_utils/v2.0.3/4d5a92c06a7fa9d956a900e51a1f6be4 Resources: SumoAppUtilsFunction: Type: AWS::Serverless::Function Properties: Handler: main.handler Runtime: python3.7 - CodeUri: s3://appdevstore/sumo_app_utils/v2.0.1/sumo_app_utils.zip + CodeUri: s3://appdevstore/sumo_app_utils/v2.0.3/sumo_app_utils.zip MemorySize: 128 Timeout: 300 Policies: diff --git a/sumologic-app-utils/src/awsresource.py b/sumologic-app-utils/src/awsresource.py index c49a368..237a99e 100644 --- a/sumologic-app-utils/src/awsresource.py +++ b/sumologic-app-utils/src/awsresource.py @@ -495,22 +495,10 @@ class EC2Resources(AWSResourcesAbstract): def fetch_resources(self): instances = [] - next_token = None - while next_token != 'END': - if next_token: - response = self.client.describe_instances(MaxResults=1000, NextToken=next_token) - else: - response = self.client.describe_instances(MaxResults=1000) - - for reservation in response['Reservations']: + for page in self.client.get_paginator("describe_instances").paginate(MaxResults=1000): + for reservation in page['Reservations']: if "Instances" in reservation: instances.extend(reservation['Instances']) - - next_token = response["NextToken"] if "NextToken" in response else None - - if not next_token: - next_token = 'END' - return instances def get_arn_list(self, resources): @@ -884,6 +872,7 @@ def enable_s3_logs(self, arns, s3_bucket, s3_prefix, elb_region_account_id): {'Key': 'access_logs.s3.prefix', 'Value': s3_prefix}] for arn in arns: + print("Enable S3 logging for ALB " + arn) response = self.client.describe_load_balancer_attributes(LoadBalancerArn=arn) if "Attributes" in response: for attribute in response["Attributes"]: @@ -917,16 +906,39 @@ def add_bucket_policy(self, bucket_name, elb_region_account_id): else: raise e - bucket_policy = { - 'Sid': 'AwsAlbLogs', - 'Effect': 'Allow', - 'Principal': { - "AWS": "arn:aws:iam::" + elb_region_account_id + ":root" + bucket_policy = [{ + 'Sid': 'AwsAlbLogs', + 'Effect': 'Allow', + 'Principal': { + "AWS": "arn:aws:iam::" + elb_region_account_id + ":root" + }, + 'Action': ['s3:PutObject'], + 'Resource': f'arn:aws:s3:::{bucket_name}/*' }, - 'Action': ['s3:PutObject'], - 'Resource': f'arn:aws:s3:::{bucket_name}/*' - } - existing_policy["Statement"].append(bucket_policy) + { + "Sid": "AWSLogDeliveryAclCheck", + "Effect": "Allow", + "Principal": { + "Service": "delivery.logs.amazonaws.com" + }, + "Action": "s3:GetBucketAcl", + "Resource": "arn:aws:s3:::" + bucket_name + }, + { + "Sid": "AWSLogDeliveryWrite", + "Effect": "Allow", + "Principal": { + "Service": "delivery.logs.amazonaws.com" + }, + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::" + bucket_name + "/*", + "Condition": { + "StringEquals": { + "s3:x-amz-acl": "bucket-owner-full-control" + } + } + }] + existing_policy["Statement"].extend(bucket_policy) s3.put_bucket_policy(Bucket=bucket_name, Policy=json.dumps(existing_policy)) diff --git a/sumologic-app-utils/src/main.py b/sumologic-app-utils/src/main.py index 357189d..65f8cf0 100644 --- a/sumologic-app-utils/src/main.py +++ b/sumologic-app-utils/src/main.py @@ -4,7 +4,7 @@ from resourcefactory import ResourceFactory -helper = CfnResource(json_logging=False, log_level='DEBUG') +helper = CfnResource(json_logging=False, log_level='INFO') def get_resource(event): @@ -15,7 +15,6 @@ def get_resource(event): params = resource.extract_params(event) if isinstance(resource, SumoResource): params["remove_on_delete_stack"] = props.get("RemoveOnDeleteStack") == 'true' - print(params) return resource, resource_type, params diff --git a/sumologic-app-utils/src/sumologic.py b/sumologic-app-utils/src/sumologic.py index 77f0313..bdd59b7 100644 --- a/sumologic-app-utils/src/sumologic.py +++ b/sumologic-app-utils/src/sumologic.py @@ -219,12 +219,20 @@ def install_app(self, app_id, content): def check_app_install_status(self, job_id): return self.get('/apps/install/%s/status' % job_id) + def get_apps(self): + response = self.get('/apps') + return json.loads(response.text) + def create_explorer_view(self, content): return self.post('/topologies', params=content, version='v1alpha') def delete_explorer_view(self, explorer_id): return self.delete('/topologies/%s' % explorer_id, version='v1alpha') + def get_explorer_views(self): + response = self.get('/topologies', version='v1alpha') + return json.loads(response.text) + def create_metric_rule(self, content): return self.post('/metricsRules', params=content) @@ -237,7 +245,7 @@ def create_field_extraction_rule(self, content): def delete_field_extraction_rule(self, fer_name): return self.delete('/extractionRules/%s' % fer_name) - def get_all_field_extraction_rules(self, limit=None, token=None,): + def get_all_field_extraction_rules(self, limit=None, token=None, ): params = {'limit': limit, 'token': token} r = self.get('/extractionRules', params) return json.loads(r.text) @@ -248,3 +256,21 @@ def update_field_extraction_rules(self, fer_id, fer_details): def get_fer_by_id(self, fer_id): response = self.get('/extractionRules/%s' % fer_id) return json.loads(response.text) + + def fetch_metric_data_points(self, content): + return self.post('/metrics/results', params=content) + + def create_new_field(self, content): + response = self.post('/fields', params=content) + return json.loads(response.text) + + def get_all_fields(self): + response = self.get('/fields') + return json.loads(response.text)['data'] + + def get_existing_field(self, field_id): + response = self.get('/fields/%s' % field_id) + return json.loads(response.text) + + def delete_existing_field(self, field_id): + return self.delete('/fields/%s' % field_id) \ No newline at end of file diff --git a/sumologic-app-utils/src/sumoresource.py b/sumologic-app-utils/src/sumoresource.py index 7d591bb..83a44ce 100644 --- a/sumologic-app-utils/src/sumoresource.py +++ b/sumologic-app-utils/src/sumoresource.py @@ -9,6 +9,7 @@ import six from resourcefactory import AutoRegisterResource from sumologic import SumoLogic +from awsresource import AWSResourcesProvider @six.add_metaclass(AutoRegisterResource) @@ -282,6 +283,9 @@ def build_common_source_params(self, props, source_json=None): if 'useAutolineMatching' in props: source_json['useAutolineMatching'] = props['useAutolineMatching'] + # Adding Cutofftimestamp 24 hours. + source_json['cutoffTimestamp'] = int(round(time.time() * 1000)) - 24*60*60*1000 + return source_json @@ -305,7 +309,7 @@ def build_source_params(self, props, source_json = None): } }] }, - "scanInterval": 300000, + "scanInterval": int(props.get("ScanInterval")) if "ScanInterval" in props else 300000, "paused": False, }) return source_json @@ -317,29 +321,23 @@ def _get_path(self, props): if "Region" in props: regions = [props.get("Region")] - if source_type == "AwsMetadata": - return { - "type": "AwsMetadataPath", - "limitToRegions": regions - } - elif source_type == "AwsCloudWatch": - return { - "type": "CloudWatchPath", - "limitToRegions": regions, - "limitToNamespaces": props.get("Namespaces") - } - elif source_type == "AwsInventory": - return { - "type": "AwsInventoryPath", - "limitToRegions": regions, - "limitToNamespaces": props.get("Namespaces") - } - else: + if props.get("TargetBucketName"): return { "type": "S3BucketPathExpression", "bucketName": props.get("TargetBucketName"), "pathExpression": props.get("PathExpression") } + else: + path = {} + if regions: + path["limitToRegions"] = regions + if "Namespaces" in props: + path["limitToNamespaces"] = props.get("Namespaces") + if source_type == "AwsCloudWatch": + path["type"] = "CloudWatchPath" + else: + path["type"] = source_type + "Path" + return path def create(self, collector_id, source_name, props, *args, **kwargs): @@ -424,13 +422,19 @@ def create(self, collector_id, source_name, source_category, fields, message_per raise return {"SUMO_ENDPOINT": endpoint}, source_id - def update(self, collector_id, source_id, source_name, source_category, date_format=None, date_locator=None, *args, + def update(self, collector_id, source_id, source_name, source_category, fields, date_format=None, date_locator=None, *args, **kwargs): sv, etag = self.sumologic_cli.source(collector_id, source_id) sv['source']['category'] = source_category sv['source']['name'] = source_name if date_format: sv['source']["defaultDateFormats"] = [{"format": date_format, "locator": date_locator}] + # Fields condition + existing_fields = sv['source']['fields'] + if fields: + existing_fields.update(fields) + sv['source']['fields'] = existing_fields + resp = self.sumologic_cli.update_source(collector_id, sv, etag) data = resp.json()['source'] print("updated source %s" % data["id"]) @@ -572,6 +576,7 @@ def create_by_import_api(self, appname, source_params, folder_name, *args, **kwa response = self.sumologic_cli.get_personal_folder() folder_id = response.json()['id'] app_folder_id = self._get_app_folder(content, folder_id) + time.sleep(5) response = self.sumologic_cli.import_content(folder_id, content, is_overwrite="true") job_id = response.json()["id"] print("installed app %s: appFolderId: %s personalFolderId: %s jobId: %s" % ( @@ -645,6 +650,14 @@ def extract_params(self, event): class SumoLogicAWSExplorer(SumoResource): + def get_explorer_id(self, explorer_name): + explorer_views = self.sumologic_cli.get_explorer_views() + if explorer_views: + for explorer_view in explorer_views: + if explorer_name == explorer_view["name"]: + return explorer_view["id"] + raise Exception("Explorer View with name %s not found" % explorer_name) + def create_explorer_view(self, explorer_name, hierarchy): content = { "name": explorer_name, @@ -662,7 +675,9 @@ def create_explorer_view(self, explorer_name, hierarchy): for error in errors: if error.get('code') == 'topology:duplicate': print("AWS EXPLORER - Duplicate Exists for Name %s" % explorer_name) - return {"EXPLORER_NAME": explorer_name}, "Duplicate" + # Get the explorer view ID from all explorer. + explorer_id = self.get_explorer_id(explorer_name) + return {"EXPLORER_NAME": explorer_name}, explorer_id raise e def create(self, explorer_name, hierarchy, *args, **kwargs): @@ -671,26 +686,27 @@ def create(self, explorer_name, hierarchy, *args, **kwargs): # No Update API. So, Explorer view can be updated and deleted from the main stack where it was created. # First have to delete the explorer and then create new. Handling delete again due to CF in delete method. def update(self, old_explorer_name, explorer_id, explorer_name, hierarchy, *args, **kwargs): - self.delete(explorer_id, explorer_name, True) + self.delete(explorer_id, old_explorer_name, True) data, explorer_id = self.create_explorer_view(explorer_name, hierarchy) print("AWS EXPLORER - update successful with ID %s" % explorer_id) return data, explorer_id # Handling exception as the Explorer is already deleted. + # handling exception during delete, as update can fail if the previous explorer, metric rule or field has + # already been deleted. This is required in case of multiple installation of + # CF template with same names for metric rule, explorer view or fields def delete(self, explorer_id, explorer_name, remove_on_delete_stack, *args, **kwargs): - if remove_on_delete_stack and explorer_id != "Duplicate": + if remove_on_delete_stack: try: + # Backward Compatibility for 2.0.2 Versions. + # If id is duplicate then get the id from explorer name and delete it. + if explorer_id == "Duplicate": + explorer_id = self.get_explorer_id(explorer_name) response = self.sumologic_cli.delete_explorer_view(explorer_id) print("AWS EXPLORER - Completed the AWS Explorer deletion for Name %s, response - %s" % ( explorer_name, response.text)) except Exception as e: - if hasattr(e, 'response') and e.response.json()["errors"]: - errors = e.response.json()["errors"] - for error in errors: - if error.get('code') == 'topology:does_not_exist': - print("AWS EXPLORER - Completed the AWS Explorer deletion for Name %s," % explorer_name) - else: - raise e + print("AWS EXPLORER - Exception while deleting the Explorer view %s," % e) else: print("AWS EXPLORER - Skipping the AWS Explorer deletion") @@ -744,7 +760,7 @@ def create_metric_rule(self, metric_rule_name, match_expression, variables): if error.get('code') == 'metrics:rule_name_already_exists' \ or error.get('code') == 'metrics:rule_already_exists': print("METRIC RULES - Duplicate Exists for Name %s" % metric_rule_name) - return {"METRIC_RULES": metric_rule_name}, "Duplicate" + return {"METRIC_RULES": metric_rule_name}, metric_rule_name raise e def create(self, metric_rule_name, match_expression, variables, *args, **kwargs): @@ -753,19 +769,21 @@ def create(self, metric_rule_name, match_expression, variables, *args, **kwargs) # No Update API. So, Metric rules can be updated and deleted from the main stack where it was created. def update(self, old_metric_rule_name, job_name, metric_rule_name, match_expression, variables, *args, **kwargs): # Need to add it because CF calls delete method if identifies change in metric rule name. - if metric_rule_name != old_metric_rule_name: - return self.create(metric_rule_name, match_expression, variables) - else: - self.delete(job_name, metric_rule_name, True) - data, job_name = self.create_metric_rule(metric_rule_name, match_expression, variables) - print("METRIC RULES - Update successful with Name %s" % job_name) - return data, job_name - + self.delete(job_name, old_metric_rule_name, True) + data, job_name = self.create_metric_rule(metric_rule_name, match_expression, variables) + print("METRIC RULES - Update successful with Name %s" % job_name) + return data, job_name + + # handling exception during delete, as update can fail if the previous explorer, metric rule or field has + # already been deleted. This is required in case of multiple installation of + # CF template with same names for metric rule, explorer view or fields def delete(self, job_name, metric_rule_name, remove_on_delete_stack, *args, **kwargs): - if remove_on_delete_stack and job_name != "Duplicate": - response = self.sumologic_cli.delete_metric_rule(job_name) - print("METRIC RULES - Completed the Metric Rule deletion for Name %s, response - %s" % ( - job_name, response.text)) + if remove_on_delete_stack: + try: + response = self.sumologic_cli.delete_metric_rule(metric_rule_name) + print("METRIC RULES - Completed the Metric Rule deletion for Name %s, response - %s" % (metric_rule_name, response.text)) + except Exception as e: + print("AWS EXPLORER - Exception while deleting the Metric Rules %s," % e) else: print("METRIC RULES - Skipping the Metric Rule deletion") @@ -800,59 +818,37 @@ class SumoLogicUpdateFields(SumoResource): Fields can also be added to new Sources using AWSSource, HTTPSources classes. Getting collector name, as Calling custom collector resource can update the collector name if stack is updated with different collector name. """ + def add_fields_to_collector(self, collector_id, source_id, fields): + if collector_id and source_id: + sv, etag = self.sumologic_cli.source(collector_id, source_id) - def _get_collector_by_name(self, collector_name): - offset = 0 - page_limit = 300 - all_collectors = self.sumologic_cli.collectors(limit=page_limit, offset=offset) - while all_collectors: - for collector in all_collectors: - if collector["name"] == collector_name: - return collector - offset += page_limit - all_collectors = self.sumologic_cli.collectors(limit=page_limit, offset=offset) - - raise Exception("Collector with name %s not found" % collector_name) - - def add_fields_to_collector(self, collector_name, source_name, fields): - if collector_name and source_name: - collector = self._get_collector_by_name(collector_name) - if collector and 'id' in collector: - collector_id = collector['id'] - sources = self.sumologic_cli.sources(collector_id, limit=300) - source_id = None - for source in sources: - if source["name"] == source_name: - source_id = source["id"] - - sv, etag = self.sumologic_cli.source(collector_id, source_id) - - existing_fields = sv['source']['fields'] + existing_fields = sv['source']['fields'] - new_fields = existing_fields.copy() - new_fields.update(fields) + new_fields = existing_fields.copy() + new_fields.update(fields) - sv['source']['fields'] = new_fields + sv['source']['fields'] = new_fields - resp = self.sumologic_cli.update_source(collector_id, sv, etag) + resp = self.sumologic_cli.update_source(collector_id, sv, etag) - data = resp.json()['source'] - print("Added Fields in Source %s" % data["id"]) + data = resp.json()['source'] + print("Added Fields in Source %s" % data["id"]) - return {"added_fields": fields}, str(source_id) + "#" + str(collector_id) - return {"added_fields": "Not updated"}, "No_Source_Id#No_Collector_Id" + return {"source_name": data["name"]}, str(source_id) + return {"source_name": "Not updated"}, "No_Source_Id" - def create(self, collector_name, source_name, fields, *args, **kwargs): - return self.add_fields_to_collector(collector_name, source_name, fields) + def create(self, collector_id, source_id, fields, *args, **kwargs): + return self.add_fields_to_collector(collector_id, source_id, fields) # Update the new fields to source. - def update(self, old_resource_properties, collector_id, source_id, collector_name, source_name, fields, *args, + def update(self, collector_id, source_id, fields, old_resource_properties, *args, **kwargs): # Fetch the source, get all fields. Merge the Old and New fields and the update source. # If Source name or collector name is changed, it is create again. - if collector_name != old_resource_properties['CollectorName'] or source_name != old_resource_properties[ - 'SourceName']: - return self.create(collector_name, source_name, fields) + if 'SourceApiUrl' in old_resource_properties and \ + old_resource_properties['SourceApiUrl'].rsplit('/', 1)[-1] != source_id or \ + re.search('collectors/(.*)/sources', old_resource_properties['SourceApiUrl']).group(1) != collector_id: + return self.create(collector_id, source_id, fields) else: sv, etag = self.sumologic_cli.source(collector_id, source_id) existing_source_fields = sv['source']['fields'] @@ -865,7 +861,7 @@ def update(self, old_resource_properties, collector_id, source_id, collector_nam resp = self.sumologic_cli.update_source(collector_id, sv, etag) data = resp.json()['source'] print("updated Fields in Source %s" % data["id"]) - return {"updated_fields": fields}, str(source_id) + "#" + str(collector_id) + return {"source_name": data["name"]}, source_id def delete(self, collector_id, source_id, fields, remove_on_delete_stack, *args, **kwargs): if remove_on_delete_stack: @@ -887,10 +883,6 @@ def delete(self, collector_id, source_id, fields, remove_on_delete_stack, *args, def extract_params(self, event): props = event.get("ResourceProperties") - source_id = None - if event.get('PhysicalResourceId'): - _, source_id = event['PhysicalResourceId'].split("/") - old_resource_properties = None if "OldResourceProperties" in event: old_resource_properties = event['OldResourceProperties'] @@ -900,11 +892,9 @@ def extract_params(self, event): fields = props.get("Fields") return { - "collector_name": props.get("CollectorName"), - "source_name": props.get("SourceName"), "fields": fields, - "source_id": source_id.split('#')[0] if source_id else None, - "collector_id": source_id.split('#')[1] if source_id else None, + "collector_id": re.search('collectors/(.*)/sources', props.get("SourceApiUrl")).group(1), + "source_id": props.get("SourceApiUrl").rsplit('/', 1)[-1], "old_resource_properties": old_resource_properties } @@ -956,7 +946,7 @@ def create(self, fer_name, fer_scope, fer_expression, fer_enabled, *args, **kwar change_in_fer = True if change_in_fer: self.sumologic_cli.update_field_extraction_rules(fer_details["id"], fer_details) - return {"FER_RULES": fer_name}, "Duplicate" + return {"FER_RULES": fer_name}, fer_details["id"] raise e def update(self, fer_id, fer_name, fer_scope, fer_expression, fer_enabled, *args, **kwargs): @@ -985,7 +975,7 @@ def update(self, fer_id, fer_name, fer_scope, fer_expression, fer_enabled, *args raise e def delete(self, fer_id, remove_on_delete_stack, *args, **kwargs): - if remove_on_delete_stack and fer_id != "Duplicate": + if remove_on_delete_stack: response = self.sumologic_cli.delete_field_extraction_rule(fer_id) print("FER RULES - Completed the Metric Rule deletion for ID %s, response - %s" % ( fer_id, response.text)) @@ -1008,8 +998,228 @@ def extract_params(self, event): } -if __name__ == '__main__': +class AddFieldsInHostMetricsSources(SumoResource): + """ + This class is specifically designed for Adding fields to HostMetrics Source. + """ + + def batch_size_chunking(self, iterable, size=1): + l = len(iterable) + for idx in range(0, l, size): + data = iterable[idx:min(idx + size, l)] + yield data + + def get_source_and_collector_id(self, instances): + ids = [] + for instance in instances: + ids.append("InstanceId=%s" % instance["InstanceId"]) + query = " or ".join(ids) + content = { + "query": [ + { + "query": "_contentType=HostMetrics (%s) | count by _sourceId, _collectorId" % query, + "rowId": "A" + } + ], + "startTime": int(time.time() * 1000) - 60 * 60 * 1000, + "endTime": int(time.time() * 1000), + "desiredQuantizationInSecs": 600, + "requestedDataPoints": 1 + } + output = self.sumologic_cli.fetch_metric_data_points(content) + responses = json.loads(output.text)["response"] + sources = [] + if responses: + for response in responses: + if "results" in response: + for result in response["results"]: + if "metric" in result and "dimensions" in result["metric"]: + output = {} + for dimension in result["metric"]["dimensions"]: + if dimension["key"] == "_collectorId" or dimension["key"] == "_sourceId": + output[dimension["key"]] = str(int(dimension["value"], 16)) + sources.append(output) + return sources + + def add_remove_fields(self, region_value, account_id, new_fields, old_fields=None): + # Get all EC2 Instance ID's + ec2_resource = AWSResourcesProvider.get_provider("ec2", region_value, account_id) + instance_ids = ec2_resource.fetch_resources() + chucked_data = self.batch_size_chunking(instance_ids, 10) + for instances in chucked_data: + sources = self.get_source_and_collector_id(instances) + for source in sources: + collector_id = source["_collectorId"] + source_id = source["_sourceId"] + sv, etag = self.sumologic_cli.source(collector_id, source_id) + existing_source_fields = sv['source']['fields'] + if old_fields: + for k in old_fields: + existing_source_fields.pop(k, None) + if new_fields: + existing_source_fields.update(new_fields) + + sv['source']['fields'] = existing_source_fields + resp = self.sumologic_cli.update_source(collector_id, sv, etag) + data = resp.json()['source'] + print("updated Fields in Source %s" % data["id"]) + + def create(self, region_value, account_id, fields, add_fields, *args, **kwargs): + if add_fields: + self.add_remove_fields(region_value, account_id, fields) + else: + print("Skipping Adding Fields to Sources for Region %s", region_value) + return {"Fields_Added": "Successful"}, region_value + def update(self, old_properties, region_value, account_id, fields, add_fields, *args, **kwargs): + if add_fields: + if old_properties['Region'] != region_value: + data, region_value = self.create(region_value, account_id, fields, add_fields) + else: + old_fields = None + if 'Fields' in old_properties and old_properties['Fields']: + old_fields = old_properties['Fields'] + self.add_remove_fields(region_value, account_id, fields, old_fields) + else: + print("Skipping Adding Fields to Sources for Region %s", region_value) + return {"Fields_Updated": "Successful"}, region_value + + def delete(self, remove_on_delete_stack, region_value, account_id, fields, add_fields, *args, **kwargs): + if add_fields: + if remove_on_delete_stack: + self.add_remove_fields(region_value, account_id, None, fields) + else: + print("UPDATE FIELDS - Skipping the Fields deletion") + else: + print("Skipping Adding Fields to Sources for Region %s", region_value) + + def extract_params(self, event): + props = event.get("ResourceProperties") + + fields = {} + if "Fields" in props: + fields = props.get("Fields") + + add_fields = True + if props.get("AddFields") == "No": + add_fields = False + + old_resource_properties = None + if "OldResourceProperties" in event: + old_resource_properties = event['OldResourceProperties'] + + return { + "region_value": props.get("Region"), + "account_id": props.get("AccountID"), + "fields": fields, + "add_fields": add_fields, + "remove_on_delete_stack": props.get("RemoveOnDeleteStack"), + "old_properties": old_resource_properties + } + + +class SumoLogicFieldsSchema(SumoResource): + + def get_field_id(self, field_name): + all_fields = self.sumologic_cli.get_all_fields() + if all_fields: + for field in all_fields: + if field_name == field["fieldName"]: + return field["fieldId"] + raise Exception("Field Name with name %s not found" % field_name) + + def add_field(self, field_name): + content = { + "fieldName": field_name, + } + try: + response = self.sumologic_cli.create_new_field(content) + field_id = response["fieldId"] + print("FIELD NAME - creation successful with Field Id %s" % field_id) + return {"FIELD_NAME": response["fieldName"]}, field_id + except Exception as e: + if hasattr(e, 'response') and e.response.json()["errors"]: + errors = e.response.json()["errors"] + for error in errors: + if error.get('code') == 'field:already_exists': + print("FIELD NAME - Duplicate Exists for Name %s" % field_name) + # Get the Field ID from the existing fields. + field_id = self.get_field_id(field_name) + return {"FIELD_NAME": field_name}, field_id + raise e + + def create(self, field_name, *args, **kwargs): + return self.add_field(field_name) + + # No Update API. So, Fields will be added and deleted from the main stack. + def update(self, field_id, field_name, old_field_name, *args, **kwargs): + # Create a new field when field name changes. Delete will happen for old Field. No Update API, so no updates. + if field_name != old_field_name: + return self.create(field_name) + return {"FIELD_NAME": field_name}, field_id + + # handling exception during delete, as update can fail if the previous explorer, metric rule or field has + # already been deleted. This is required in case of multiple installation of + # CF template with same names for metric rule, explorer view or fields + def delete(self, field_id, field_name, remove_on_delete_stack, *args, **kwargs): + if remove_on_delete_stack: + # Backward Compatibility for 2.0.2 Versions. + # Check for field_id is duplicate, then get the field ID from name and delete the field. + try: + if field_id == "Duplicate": + field_id = self.get_field_id(field_name) + response = self.sumologic_cli.delete_existing_field(field_id) + print("FIELD NAME - Completed the Field deletion for ID %s, response - %s" % (field_id, response.text)) + except Exception as e: + print("AWS EXPLORER - Exception while deleting the Field %s," % e) + else: + print("FIELD NAME - Skipping the Field deletion") + + def extract_params(self, event): + props = event.get("ResourceProperties") + + field_id = None + if event.get('PhysicalResourceId'): + _, field_id = event['PhysicalResourceId'].split("/") + + # Get previous Metric Rule Name + old_field_name = None + if "OldResourceProperties" in event and "FieldName" in event['OldResourceProperties']: + old_field_name = event["OldResourceProperties"]['FieldName'] + + return { + "field_name": props.get("FieldName"), + "field_id": field_id, + "old_field_name": old_field_name + } + + +class EnterpriseOrTrialAccountCheck(SumoResource): + + def check_account(self): + is_enterprise = self.is_enterprise_or_trial_account() + is_paid = "Yes" + if not is_enterprise: + all_apps = self.sumologic_cli.get_apps() + if "apps" in all_apps and len(all_apps['apps']) <= 2: + is_paid = "No" + return {"is_enterprise": "Yes" if is_enterprise else "No", "is_paid": is_paid}, is_enterprise + + def create(self, *args, **kwargs): + return self.check_account() + + def update(self, *args, **kwargs): + return self.check_account() + + def delete(self, *args, **kwargs): + print("In Delete method for Enterprise or Trial account") + + def extract_params(self, event): + props = event.get("ResourceProperties") + return props + + +if __name__ == '__main__': props = { "SumoAccessID": "", "SumoAccessKey": "", diff --git a/sumologic-app-utils/sumo_app_utils.yaml b/sumologic-app-utils/sumo_app_utils.yaml index 0ff206a..07d4361 100644 --- a/sumologic-app-utils/sumo_app_utils.yaml +++ b/sumologic-app-utils/sumo_app_utils.yaml @@ -17,7 +17,7 @@ Metadata: - sumologic - serverless Name: sumologic-app-utils - SemanticVersion: 2.0.1 + SemanticVersion: 2.0.3 SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/sumologic-app-utils SpdxLicenseId: Apache-2.0 ReadmeUrl: ./README.md @@ -29,7 +29,7 @@ Resources: Properties: Handler: main.handler Runtime: python3.7 - CodeUri: s3://appdevstore/sumo_app_utils/v2.0.1/sumo_app_utils.zip + CodeUri: s3://appdevstore/sumo_app_utils/v2.0.3/sumo_app_utils.zip MemorySize: 128 Timeout: 300 Policies: diff --git a/sumologic-app-utils/sumo_app_utils.zip b/sumologic-app-utils/sumo_app_utils.zip index fdfc727..cc66473 100644 Binary files a/sumologic-app-utils/sumo_app_utils.zip and b/sumologic-app-utils/sumo_app_utils.zip differ