From d07b2028ab110a8d92d7361c8adf6597ab408461 Mon Sep 17 00:00:00 2001 From: Oleksii Date: Mon, 4 Feb 2019 17:52:06 +0200 Subject: [PATCH] Rework identification API to be asynchronous User issues a POST request with payload to /identify, unique request_id is generated and returned to user. User then can issue GET request to /identify/ to get response. This allows to scan big accounts with a lot of resources and do not exceed 30 seconds response timeout for lambda integration with gateway. New DDB was added for tracking status of user requests. API lambda(entrypoint) now triggers existing describe lambdas to do a scan. Every describe lambda retrieves request_id from payload and updates corresponding record in requests table after it finishes the scan of the region. This allows to track the progress of scan, by comparing current progress and total number of scans (features * regions) and determine when the scan is finished. Remediation is not supported for now. Add "scan account " command to slack bot. Currently it supports only full account scan. --- deployment/cf-templates/api.json | 2 +- deployment/cf-templates/ddb.json | 22 ++ deployment/configs/config.json | 15 ++ .../identification/lambdas/api/authorizer.py | 7 +- .../identification/lambdas/api/entrypoint.py | 213 +++++++++++++----- .../describe_cloudtrails.py | 10 +- .../describe_ebs_public_snapshots.py | 9 +- .../describe_ebs_unencrypted_volumes.py | 9 +- .../describe_iam_key_rotation.py | 9 +- .../describe_iam_accesskey_details.py | 9 +- .../describe_rds_public_snapshots.py | 7 +- .../describe_rds_instance_encryption.py | 7 +- .../describe_s3_bucket_acl.py | 7 +- .../describe_s3_bucket_policy.py | 7 +- .../describe_s3_encryption.py | 7 +- .../describe_sec_grps_unrestricted_access.py | 7 +- .../describe_sqs_public_policy.py | 7 +- hammer/library/aws/utility.py | 43 +++- hammer/library/config.py | 22 +- hammer/library/ddb_issues.py | 8 + hammer/reporting-remediation/bot/commands.py | 56 +++++ hammer/tools/ddb_inject_credentials.py | 7 + 22 files changed, 412 insertions(+), 78 deletions(-) diff --git a/deployment/cf-templates/api.json b/deployment/cf-templates/api.json index 82ff8613..5d5d5f8d 100644 --- a/deployment/cf-templates/api.json +++ b/deployment/cf-templates/api.json @@ -209,7 +209,7 @@ "HttpMethod": "ANY", "Integration": { "Type": "AWS_PROXY", - "IntegrationHttpMethod": "ANY", + "IntegrationHttpMethod": "POST", "Uri": {"Fn::Join": ["", ["arn:aws:apigateway:", {"Ref": "AWS::Region"}, ":lambda:path/2015-03-31/functions/", {"Fn::GetAtt": ["LambdaApi", "Arn"]}, "/invocations"] ]} diff --git a/deployment/cf-templates/ddb.json b/deployment/cf-templates/ddb.json index 2ab4843c..aef007e1 100755 --- a/deployment/cf-templates/ddb.json +++ b/deployment/cf-templates/ddb.json @@ -428,6 +428,28 @@ }, "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "rds-unencrypted" ] ]} } + }, + "DynamoDBApiRequests": { + "Type": "AWS::DynamoDB::Table", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "request_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "request_id", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": "10", + "WriteCapacityUnits": "2" + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "credentials" ] ]} + } } } } diff --git a/deployment/configs/config.json b/deployment/configs/config.json index 74087148..b46bae42 100755 --- a/deployment/configs/config.json +++ b/deployment/configs/config.json @@ -40,6 +40,9 @@ "654321210987": "slave2" } }, + "api": { + "ddb.table_name": "hammer-api-requests" + }, "whitelisting_procedure_url": "", "credentials": { "ddb.table_name": "hammer-credentials" @@ -47,6 +50,7 @@ "s3_bucket_acl": { "enabled": true, "ddb.table_name": "hammer-s3-public-bucket-acl", + "topic_name": "hammer-describe-s3-acl-lambda", "reporting": false, "remediation": false, "remediation_retention_period": 0 @@ -54,6 +58,7 @@ "secgrp_unrestricted_access": { "enabled": true, "ddb.table_name": "hammer-security-groups-unrestricted", + "topic_name": "hammer-describe-security-groups-lambda", "restricted_ports": [ 21, 22, @@ -75,6 +80,7 @@ "user_inactivekeys": { "enabled": true, "ddb.table_name": "hammer-iam-user-keys-inactive", + "topic_name": "hammer-describe-iam-user-inactive-keys-lambda", "ignore_accounts": ["654321210987"], "inactive_criteria_days": 1, "reporting": false, @@ -84,6 +90,7 @@ "user_keysrotation": { "enabled": true, "ddb.table_name": "hammer-iam-user-keys-rotation", + "topic_name": "hammer-describe-iam-user-keys-rotation-lambda", "rotation_criteria_days": 10, "reporting": false, "remediation": false, @@ -92,6 +99,7 @@ "s3_bucket_policy": { "enabled": true, "ddb.table_name": "hammer-s3-public-bucket-policy", + "topic_name": "hammer-describe-s3-policy-lambda", "reporting": false, "remediation": false, "remediation_retention_period": 7 @@ -99,17 +107,20 @@ "cloudtrails": { "enabled": true, "ddb.table_name": "hammer-cloudtrails", + "topic_name": "hammer-describe-cloudtrails-lambda", "reporting": false }, "ebs_unencrypted_volume": { "enabled": true, "ddb.table_name": "hammer-ebs-volumes-unencrypted", + "topic_name": "hammer-describe-ebs-unencrypted-volumes-lambda", "accounts": ["123456789012", "210987654321"], "reporting": false }, "ebs_public_snapshot": { "enabled": true, "ddb.table_name": "hammer-ebs-snapshots-public", + "topic_name": "hammer-describe-ebs-public-snapshots-lambda", "reporting": false, "remediation": false, "remediation_retention_period": 0 @@ -117,6 +128,7 @@ "rds_public_snapshot": { "enabled": true, "ddb.table_name": "hammer-rds-public-snapshots", + "topic_name": "hammer-describe-rds-public-snapshots-lambda", "reporting": false, "remediation": false, "remediation_retention_period": 0 @@ -124,6 +136,7 @@ "sqs_public_access": { "enabled": true, "ddb.table_name": "hammer-sqs-public-access", + "topic_name": "hammer-describe-sqs-public-policy-lambda", "reporting": true, "remediation": false, "remediation_retention_period": 0 @@ -131,6 +144,7 @@ "s3_encryption": { "enabled": true, "ddb.table_name": "hammer-s3-unencrypted", + "topic_name": "hammer-describe-s3-encryption-lambda", "reporting": true, "remediation": false, "remediation_retention_period": 0 @@ -138,6 +152,7 @@ "rds_encryption": { "enabled": true, "ddb.table_name": "hammer-rds-unencrypted", + "topic_name": "hammer-describe-rds-encryption-lambda", "reporting": true } } diff --git a/hammer/identification/lambdas/api/authorizer.py b/hammer/identification/lambdas/api/authorizer.py index be68c974..79f3262e 100644 --- a/hammer/identification/lambdas/api/authorizer.py +++ b/hammer/identification/lambdas/api/authorizer.py @@ -28,6 +28,11 @@ def lambda_handler(event, context): policy.restApiId = apiGatewayArnTmp[0] policy.region = tmp[3] policy.stage = apiGatewayArnTmp[1] + # a quick hack to allow GET calls to /identify/{request_id}, request_id is hex string + # rewrite this solution to more generic variant + if len(apiGatewayArnTmp) == 5: + full_path = '/identify/' + apiGatewayArnTmp[4] + policy.allowMethod(HttpVerb.GET, full_path) policy.allowMethod(HttpVerb.POST, '/identify') policy.allowMethod(HttpVerb.POST, '/remediate') @@ -190,4 +195,4 @@ def build(self): policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Allow', self.allowMethods)) policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Deny', self.denyMethods)) - return policy \ No newline at end of file + return policy diff --git a/hammer/identification/lambdas/api/entrypoint.py b/hammer/identification/lambdas/api/entrypoint.py index 2bc68a47..16d203b9 100644 --- a/hammer/identification/lambdas/api/entrypoint.py +++ b/hammer/identification/lambdas/api/entrypoint.py @@ -1,12 +1,16 @@ +import functools import json import logging -import functools -import importlib +import uuid +import time +import boto3 -from library.logger import set_logging +from library.aws.utility import Account, DDB, Sns from library.config import Config -from library.aws.utility import Account +from library.ddb_issues import Operations as IssueOperations +from library.logger import set_logging +from library import utility from responses import bad_request, server_error @@ -21,45 +25,45 @@ def wrapper(event, context): return wrapper -@logger -def lambda_handler(event, context): - try: - payload = json.loads(event.get('body', "{}")) - except Exception: - logging.exception("failed to parse payload") - return bad_request(text="malformed payload") +def get_sns_topic_arn(config, topic_name): + # it assumes that lambda and sns are in the same region + region = config.aws.region + account_id = boto3.client('sts').get_caller_identity()['Account'] + return f"arn:aws:sns:{region}:{account_id}:{topic_name}" - if not payload: - return bad_request(text="empty payload") - account_id = payload.get("account_id", None) - region = payload.get("region", None) - security_feature = payload.get("security_feature", None) - tags = payload.get("tags", None) - ids = payload.get("ids", None) +GLOBAL_SECURITY_FEATURES = ['s3_bucket_acl', 'user_inactivekeys', 'user_keysrotation', 's3_bucket_policy', + 's3_encryption'] - config = Config() - action = event.get("path", "")[1:] - # do not forget to allow path in authorizer.py while extending this list - if action == "identify": - role = config.aws.role_name_identification - elif action == "remediate": - role = config.aws.role_name_reporting - else: - return bad_request(text="wrong action") +def start_scan(account_id, regions, security_features, tags, ids): + config = Config() account_name = config.aws.accounts.get(account_id, None) + if not account_id: + return bad_request(text="account_id is required parameter") + if account_name is None: return bad_request(text=f"account '{account_id}' is not defined") - if not all([account_id, security_feature]): - return bad_request(text="wrong payload, missing required parameter") - valid_security_features = [ module.section for module in config.modules ] - if security_feature not in valid_security_features: - return bad_request(text=f"wrong security feature - '{security_feature}', available choices - {valid_security_features}") + for security_feature in security_features: + if security_feature not in valid_security_features: + return bad_request( + text=f"wrong security feature - '{security_feature}', available choices - {valid_security_features}") + + if not security_features: + security_features = valid_security_features + + all_regions = config.aws.regions + + for region in regions: + if region not in all_regions: + return bad_request(text=f"Region '{region} is not supported") + # empty list means we want to scan all supported regions + if not regions: + regions = all_regions if ids is not None and not isinstance(ids, list): return bad_request(text=f"'ids' parameter must be list") @@ -67,28 +71,131 @@ def lambda_handler(event, context): if tags is not None and not isinstance(tags, dict): return bad_request(text=f"'tags' parameter must be dict") - account = Account(id=account_id, - name=account_name, - region=region, - role_name=role) - if account.session is None: - return server_error(text=f"Failed to create session in {account}") - - try: - module = importlib.import_module(security_feature) - handler = getattr(module, action) - except (ModuleNotFoundError, AttributeError): - logging.exception("Module or attribute was not found") - response = f"{action} for '{security_feature}' resources in '{region}' of '{account_id} / {account_name}' is not implemented yet" - else: - try: - response = handler(security_feature, account, config, ids, tags) - except Exception: - text=f"{security_feature}:{action} execution error" - logging.exception(text) - return server_error(text=text) + main_account = Account(region=config.aws.region) + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + regional_services = set(security_features) - set(GLOBAL_SECURITY_FEATURES) + global_services = set(security_features).intersection(set(GLOBAL_SECURITY_FEATURES)) + total = len(regional_services) * len(regions) + len(global_services) + request_params = { + "account_id": account_id, + "regions": regions, + "security_features": security_features, + "tags": tags + } + request_id = uuid.uuid4().hex + + DDB.add_request(api_table, request_id, request_params, total) + + for security_feature in security_features: + topic_name = config.get_module_config_by_name(security_feature).sns_topic_name + topic_arn = get_sns_topic_arn(config, topic_name) + payload = { + "account_id": account_id, + "account_name": account_name, + "regions": regions, + "sns_arn": topic_arn, + "request_id": request_id + } + Sns.publish(topic_arn, payload) + + response = {'request_id': request_id} return { "statusCode": 200, "body": json.dumps(response, indent=4) if isinstance(response, dict) else response - } \ No newline at end of file + } + + +def start_remediate(account_id, regions, security_features, tags, ids): + return { + "statusCode": 200, + "body": json.dumps({}, indent=4) + } + + +def collect_results(request_info, main_account): + security_features = request_info['request_params']['security_features'] + regions = request_info['request_params']['regions'] + scan_account_id = request_info['request_params']['account_id'] + tags = request_info['request_params']['tags'] + response = dict({'global': {}}) + for region in regions: + response[region] = {} + for sec_feature in security_features: + if sec_feature not in GLOBAL_SECURITY_FEATURES: + response[region][sec_feature] = [] + else: + response['global'][sec_feature] = [] + + config = Config() + for security_feature in security_features: + sec_feature_config = config.get_module_config_by_name(security_feature) + ddb_table = main_account.resource("dynamodb").Table(sec_feature_config.ddb_table_name) + for issue in IssueOperations.get_account_open_issues(ddb_table, scan_account_id): + if issue.contains_tags(tags) and ( + issue.issue_details.region in regions or security_feature in GLOBAL_SECURITY_FEATURES): + issue_region = issue.issue_details.region if issue.issue_details.region else 'global' + response[issue_region][security_feature].append({'id': issue.issue_id, + 'issue_details': issue.issue_details.as_dict()}) + return response + + +def get_scan_results(request_id): + config = Config() + main_account = Account(region=config.aws.region) + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + request_info = DDB.get_request_data(api_table, request_id) + if not request_info: + status_code = 404 + body = {"message": "Request id has not been found."} + elif request_info['progress'] == request_info['total']: + status_code = 200 + body = { + "scan_status": "COMPLETE", + "scan_results": collect_results(request_info, main_account) + } + elif time.time() - request_info['updated'] <= 300: + status_code = 200 + body = { + "scan_status": "IN_PROGRESS" + } + else: + status_code = 200 + body = { + "scan_status": "FAILED" + } + return { + "statusCode": status_code, + "body": json.dumps(body, indent=4, default=utility.jsonEncoder) + } + + +@logger +def lambda_handler(event, context): + try: + body = event.get('body') if event.get('body') else "{}" + payload = json.loads(body) + except Exception: + logging.exception("failed to parse payload") + return bad_request(text="malformed payload") + + account_id = payload.get("account_id", None) + regions = payload.get("regions", []) + security_features = payload.get("security_features", []) + tags = payload.get("tags", None) + ids = payload.get("ids", None) + + action = event.get("path", "")[1:] + method = event.get("httpMethod") + # do not forget to allow path in authorizer.py while extending this list + if action.startswith('identify'): + if method == "POST": + return start_scan(account_id, regions, security_features, tags, ids) + if method == "GET": + # get request id from url path + request_id = action.split('/')[1] + return get_scan_results(request_id) + elif action == "remediate": + return start_remediate(account_id, regions, security_features, tags, ids) + else: + return bad_request(text="wrong action") diff --git a/hammer/identification/lambdas/cloudtrails-issues-identification/describe_cloudtrails.py b/hammer/identification/lambdas/cloudtrails-issues-identification/describe_cloudtrails.py index de285db8..b02ea0ec 100755 --- a/hammer/identification/lambdas/cloudtrails-issues-identification/describe_cloudtrails.py +++ b/hammer/identification/lambdas/cloudtrails-issues-identification/describe_cloudtrails.py @@ -8,7 +8,7 @@ from library.aws.utility import Account from library.ddb_issues import IssueStatus, CloudTrailIssue from library.ddb_issues import Operations as IssueOperations -from library.aws.utility import Sns +from library.aws.utility import DDB, Sns def lambda_handler(event, context): @@ -21,6 +21,8 @@ def lambda_handler(event, context): account_name = payload['account_name'] # get the last region from the list to process region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -63,6 +65,10 @@ def lambda_handler(event, context): # issue exists in ddb and was fixed elif region in open_issues: IssueOperations.set_status_resolved(ddb_table, open_issues[region]) + # track the progress of API request to scan specific account/region/feature + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check CloudTrail in '{region}' for '{account_id} ({account_name})'") @@ -73,4 +79,4 @@ def lambda_handler(event, context): except Exception: logging.exception("Failed to chain CloudTrail checking") - logging.debug(f"Checked CloudTrail in '{region}' for '{account_id} ({account_name})'") \ No newline at end of file + logging.debug(f"Checked CloudTrail in '{region}' for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/ebs-public-snapshots-identification/describe_ebs_public_snapshots.py b/hammer/identification/lambdas/ebs-public-snapshots-identification/describe_ebs_public_snapshots.py index 13b6889e..dee609e9 100755 --- a/hammer/identification/lambdas/ebs-public-snapshots-identification/describe_ebs_public_snapshots.py +++ b/hammer/identification/lambdas/ebs-public-snapshots-identification/describe_ebs_public_snapshots.py @@ -8,7 +8,7 @@ from library.aws.utility import Account from library.ddb_issues import IssueStatus, EBSPublicSnapshotIssue from library.ddb_issues import Operations as IssueOperations -from library.aws.utility import Sns +from library.aws.utility import DDB, Sns def lambda_handler(event, context): @@ -21,6 +21,8 @@ def lambda_handler(event, context): account_name = payload['account_name'] # get the last region from the list to process region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -69,6 +71,9 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated EBS snapshots for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check public EBS snapshots in '{region}' for '{account_id} ({account_name})'") @@ -79,4 +84,4 @@ def lambda_handler(event, context): except Exception: logging.exception("Failed to chain public EBS snapshots checking") - logging.debug(f"Checked public EBS snapshots in '{region}' for '{account_id} ({account_name})'") \ No newline at end of file + logging.debug(f"Checked public EBS snapshots in '{region}' for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/ebs-unencrypted-volume-identification/describe_ebs_unencrypted_volumes.py b/hammer/identification/lambdas/ebs-unencrypted-volume-identification/describe_ebs_unencrypted_volumes.py index decd773c..6c295aff 100755 --- a/hammer/identification/lambdas/ebs-unencrypted-volume-identification/describe_ebs_unencrypted_volumes.py +++ b/hammer/identification/lambdas/ebs-unencrypted-volume-identification/describe_ebs_unencrypted_volumes.py @@ -8,7 +8,7 @@ from library.aws.utility import Account from library.ddb_issues import IssueStatus, EBSUnencryptedVolumeIssue from library.ddb_issues import Operations as IssueOperations -from library.aws.utility import Sns +from library.aws.utility import DDB, Sns def lambda_handler(event, context): @@ -21,6 +21,8 @@ def lambda_handler(event, context): account_name = payload['account_name'] # get the last region from the list to process region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -71,6 +73,9 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated volumes for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check unencrypted EBS volumes in '{region}' for '{account_id} ({account_name})'") @@ -81,4 +86,4 @@ def lambda_handler(event, context): except Exception: logging.exception("Failed to chain unencrypted EBS volumes checking") - logging.debug(f"Checked unencrypted EBS volumes in '{region}' for '{account_id} ({account_name})'") \ No newline at end of file + logging.debug(f"Checked unencrypted EBS volumes in '{region}' for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/iam-keyrotation-issues-identification/describe_iam_key_rotation.py b/hammer/identification/lambdas/iam-keyrotation-issues-identification/describe_iam_key_rotation.py index 7d56ef5d..b85e0bc2 100755 --- a/hammer/identification/lambdas/iam-keyrotation-issues-identification/describe_iam_key_rotation.py +++ b/hammer/identification/lambdas/iam-keyrotation-issues-identification/describe_iam_key_rotation.py @@ -5,7 +5,7 @@ from library.logger import set_logging from library.config import Config from library.aws.iam import IAMKeyChecker -from library.aws.utility import Account +from library.aws.utility import Account, DDB from library.ddb_issues import IssueStatus, IAMKeyRotationIssue from library.ddb_issues import Operations as IssueOperations @@ -18,6 +18,8 @@ def lambda_handler(event, context): payload = json.loads(event["Records"][0]["Sns"]["Message"]) account_id = payload['account_id'] account_name = payload['account_name'] + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -68,8 +70,11 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated keys for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check IAM user keys rotation for '{account_id} ({account_name})'") return - logging.debug(f"Checked IAM user keys rotation for '{account_id} ({account_name})'") \ No newline at end of file + logging.debug(f"Checked IAM user keys rotation for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/iam-user-inactive-keys-identification/describe_iam_accesskey_details.py b/hammer/identification/lambdas/iam-user-inactive-keys-identification/describe_iam_accesskey_details.py index 463fd1a7..c1db9fac 100755 --- a/hammer/identification/lambdas/iam-user-inactive-keys-identification/describe_iam_accesskey_details.py +++ b/hammer/identification/lambdas/iam-user-inactive-keys-identification/describe_iam_accesskey_details.py @@ -5,7 +5,7 @@ from library.logger import set_logging from library.config import Config from library.aws.iam import IAMKeyChecker -from library.aws.utility import Account +from library.aws.utility import Account, DDB from library.ddb_issues import IssueStatus, IAMKeyInactiveIssue from library.ddb_issues import Operations as IssueOperations @@ -18,6 +18,8 @@ def lambda_handler(event, context): payload = json.loads(event["Records"][0]["Sns"]["Message"]) account_id = payload['account_id'] account_name = payload['account_name'] + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -69,8 +71,11 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated keys for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check IAM user inactive keys for '{account_id} ({account_name})'") return - logging.debug(f"Checked IAM user inactive keys for '{account_id} ({account_name})'") \ No newline at end of file + logging.debug(f"Checked IAM user inactive keys for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/rds-public-snapshots-identification/describe_rds_public_snapshots.py b/hammer/identification/lambdas/rds-public-snapshots-identification/describe_rds_public_snapshots.py index 0f292fa7..6d155389 100755 --- a/hammer/identification/lambdas/rds-public-snapshots-identification/describe_rds_public_snapshots.py +++ b/hammer/identification/lambdas/rds-public-snapshots-identification/describe_rds_public_snapshots.py @@ -8,7 +8,7 @@ from library.aws.utility import Account from library.ddb_issues import IssueStatus, RdsPublicSnapshotIssue from library.ddb_issues import Operations as IssueOperations -from library.aws.utility import Sns +from library.aws.utility import DDB, Sns def lambda_handler(event, context): @@ -21,6 +21,8 @@ def lambda_handler(event, context): account_name = payload['account_name'] # get the last region from the list to process region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -71,6 +73,9 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated RDS snapshots for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check public RDS snapshots in '{region}' for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/rds-unencrypted-instance-identification/describe_rds_instance_encryption.py b/hammer/identification/lambdas/rds-unencrypted-instance-identification/describe_rds_instance_encryption.py index e0efe3d5..bc84e972 100644 --- a/hammer/identification/lambdas/rds-unencrypted-instance-identification/describe_rds_instance_encryption.py +++ b/hammer/identification/lambdas/rds-unencrypted-instance-identification/describe_rds_instance_encryption.py @@ -8,7 +8,7 @@ from library.aws.utility import Account from library.ddb_issues import IssueStatus, RdsEncryptionIssue from library.ddb_issues import Operations as IssueOperations -from library.aws.utility import Sns +from library.aws.utility import DDB, Sns def lambda_handler(event, context): @@ -21,6 +21,8 @@ def lambda_handler(event, context): account_name = payload['account_name'] # get the last region from the list to process region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -71,6 +73,9 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated RDS encryption for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check RDS encryption in '{region}' for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/s3-acl-issues-identification/describe_s3_bucket_acl.py b/hammer/identification/lambdas/s3-acl-issues-identification/describe_s3_bucket_acl.py index 7db9e3b9..6f8f20fa 100755 --- a/hammer/identification/lambdas/s3-acl-issues-identification/describe_s3_bucket_acl.py +++ b/hammer/identification/lambdas/s3-acl-issues-identification/describe_s3_bucket_acl.py @@ -4,7 +4,7 @@ from library.logger import set_logging from library.config import Config from library.aws.s3 import S3BucketsAclChecker -from library.aws.utility import Account +from library.aws.utility import Account, DDB from library.ddb_issues import IssueStatus, S3AclIssue from library.ddb_issues import Operations as IssueOperations @@ -17,6 +17,8 @@ def lambda_handler(event, context): payload = json.loads(event["Records"][0]["Sns"]["Message"]) account_id = payload['account_id'] account_name = payload['account_name'] + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -67,6 +69,9 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated buckets for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check s3 acls for '{account_id} ({account_name})'") return diff --git a/hammer/identification/lambdas/s3-policy-issues-identification/describe_s3_bucket_policy.py b/hammer/identification/lambdas/s3-policy-issues-identification/describe_s3_bucket_policy.py index 82055a45..2ac13ae0 100755 --- a/hammer/identification/lambdas/s3-policy-issues-identification/describe_s3_bucket_policy.py +++ b/hammer/identification/lambdas/s3-policy-issues-identification/describe_s3_bucket_policy.py @@ -4,7 +4,7 @@ from library.logger import set_logging from library.config import Config from library.aws.s3 import S3BucketsPolicyChecker -from library.aws.utility import Account +from library.aws.utility import Account, DDB from library.ddb_issues import IssueStatus, S3PolicyIssue from library.ddb_issues import Operations as IssueOperations @@ -17,6 +17,8 @@ def lambda_handler(event, context): payload = json.loads(event["Records"][0]["Sns"]["Message"]) account_id = payload['account_id'] account_name = payload['account_name'] + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -67,6 +69,9 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated buckets for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check s3 policies for '{account_id} ({account_name})'") return diff --git a/hammer/identification/lambdas/s3-unencrypted-bucket-issues-identification/describe_s3_encryption.py b/hammer/identification/lambdas/s3-unencrypted-bucket-issues-identification/describe_s3_encryption.py index 5cf93f81..ecf8e766 100644 --- a/hammer/identification/lambdas/s3-unencrypted-bucket-issues-identification/describe_s3_encryption.py +++ b/hammer/identification/lambdas/s3-unencrypted-bucket-issues-identification/describe_s3_encryption.py @@ -4,7 +4,7 @@ from library.logger import set_logging from library.config import Config from library.aws.s3 import S3EncryptionChecker -from library.aws.utility import Account +from library.aws.utility import Account, DDB from library.ddb_issues import IssueStatus, S3EncryptionIssue from library.ddb_issues import Operations as IssueOperations @@ -17,6 +17,8 @@ def lambda_handler(event, context): payload = json.loads(event["Records"][0]["Sns"]["Message"]) account_id = payload['account_id'] account_name = payload['account_name'] + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -66,6 +68,9 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated buckets for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check S3 encryption for '{account_id} ({account_name})'") return diff --git a/hammer/identification/lambdas/sg-issues-identification/describe_sec_grps_unrestricted_access.py b/hammer/identification/lambdas/sg-issues-identification/describe_sec_grps_unrestricted_access.py index 2eec5952..701e9cf1 100755 --- a/hammer/identification/lambdas/sg-issues-identification/describe_sec_grps_unrestricted_access.py +++ b/hammer/identification/lambdas/sg-issues-identification/describe_sec_grps_unrestricted_access.py @@ -8,7 +8,7 @@ from library.aws.utility import Account from library.ddb_issues import IssueStatus, SecurityGroupIssue from library.ddb_issues import Operations as IssueOperations -from library.aws.utility import Sns +from library.aws.utility import DDB, Sns def lambda_handler(event, context): @@ -21,6 +21,8 @@ def lambda_handler(event, context): account_name = payload['account_name'] # get the last region from the list to process region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -79,6 +81,9 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated security groups for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check insecure services in '{region}' for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/sqs-public-policy-identification/describe_sqs_public_policy.py b/hammer/identification/lambdas/sqs-public-policy-identification/describe_sqs_public_policy.py index 864c9e82..63a02b12 100644 --- a/hammer/identification/lambdas/sqs-public-policy-identification/describe_sqs_public_policy.py +++ b/hammer/identification/lambdas/sqs-public-policy-identification/describe_sqs_public_policy.py @@ -7,7 +7,7 @@ from library.aws.utility import Account from library.ddb_issues import IssueStatus, SQSPolicyIssue from library.ddb_issues import Operations as IssueOperations -from library.aws.utility import Sns +from library.aws.utility import DDB, Sns def lambda_handler(event, context): @@ -21,6 +21,8 @@ def lambda_handler(event, context): # get the last region from the list to process region = payload['regions'].pop() # region = payload['region'] + # if request_id is present in payload, it means this lambda was called from the API + request_id = payload.get('request_id', None) except Exception: logging.exception(f"Failed to parse event\n{event}") return @@ -71,6 +73,9 @@ def lambda_handler(event, context): # all other unresolved issues in DDB are for removed/remediated queues for issue in open_issues.values(): IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) except Exception: logging.exception(f"Failed to check SQS policies for '{account_id} ({account_name})'") return diff --git a/hammer/library/aws/utility.py b/hammer/library/aws/utility.py index 917bbbe3..7a32d883 100755 --- a/hammer/library/aws/utility.py +++ b/hammer/library/aws/utility.py @@ -1,12 +1,13 @@ -import logging -import json import boto3 +import decimal +from functools import lru_cache +import json +import logging import socket import time -import botocore.config -from functools import lru_cache +import botocore.config from botocore.exceptions import ClientError from boto3.session import Session @@ -197,6 +198,40 @@ def publish(arn, payload): ) +class DDB: + @staticmethod + def track_progress(table, request_id): + table.update_item( + Key={ + 'request_id': request_id + }, + UpdateExpression='SET updated=:upd, progress=progress + :val', + ExpressionAttributeValues={':upd': int(time.time()), ':val': 1}) + + @staticmethod + def _convert_item(item): + for k in item.keys(): + if isinstance(item[k], decimal.Decimal): + item[k] = int(item[k]) + return item + + @staticmethod + def get_request_data(table, request_id): + item = table.get_item(Key={'request_id': request_id}) + if 'Item' in item: + return DDB._convert_item(item['Item']) + + @staticmethod + def add_request(table, request_id, request_params, total): + table.put_item(Item={ + 'request_id': request_id, + 'request_params': request_params, + 'progress': 0, + 'total': total, + 'updated': int(time.time()) + }) + + class AWSMetric(object): """ Encapsulates AWS CloudWatch metric diff --git a/hammer/library/config.py b/hammer/library/config.py index 3f66749b..273099ac 100755 --- a/hammer/library/config.py +++ b/hammer/library/config.py @@ -87,10 +87,10 @@ def __init__(self, self.api = ApiConfig({ 'credentials': self.json_load_from_ddb(self._config["credentials"]["ddb.table_name"], self.aws.region, - "api") + "api"), + 'table': self._config["api"]["ddb.table_name"] }) - def get_bu_by_name(self, name): """ Guess BU value from the issue name @@ -115,6 +115,11 @@ def modules(self): def now(self): return datetime.now(timezone.utc) + def get_module_config_by_name(self, name): + for module in self.modules: + if module.name == name: + return module + def json_load_from_file(self, filename, default=None): """ Loads json from config file to dictionary. @@ -283,6 +288,14 @@ def __init__(self, config): def token(self): return self._config.get("credentials", {}).get("token", None) + @property + def url(self): + return self._config.get("credentials", {}).get("url", None) + + @property + def ddb_table_name(self): + return self._config['table'] + class SlackConfig(object): """ Base class for Slack logging """ @@ -446,6 +459,7 @@ def __init__(self, config, section): self._fixnow = config["fixnow"].get(section, {}) # main accounts dict self._accounts = config["aws"]["accounts"] + self.name = section def module_accounts(self, option): """ @@ -510,6 +524,10 @@ def ddb_table_name(self): """ :return: DDB table name to use for storing issue details """ return self._config["ddb.table_name"] + @property + def sns_topic_name(self): + return self._config['topic_name'] + @property def reporting(self): """ :return: boolean, if reporting for issue should be enabled """ diff --git a/hammer/library/ddb_issues.py b/hammer/library/ddb_issues.py index 433b9b5a..548cf4c0 100755 --- a/hammer/library/ddb_issues.py +++ b/hammer/library/ddb_issues.py @@ -130,6 +130,14 @@ def from_dict(item, issue_class=None): issue.jira_details = Details(item['jira_details']) return issue + def contains_tags(self, tags): + if not tags: + return True + for k in tags: + if k not in self.issue_details.tags: + return False + return True + class SecurityGroupIssue(Issue): def __init__(self, *args): diff --git a/hammer/reporting-remediation/bot/commands.py b/hammer/reporting-remediation/bot/commands.py index c58b8d31..3297c3a6 100644 --- a/hammer/reporting-remediation/bot/commands.py +++ b/hammer/reporting-remediation/bot/commands.py @@ -1,5 +1,7 @@ import json import re +import requests +import time from slackbot.bot import respond_to @@ -123,3 +125,57 @@ def whitelisted(message, term): return message.reply(response) + + +message_mapping = { + 's3_bucket_policy': '*These buckets have public policy:* ', + 's3_bucket_acl': '*These buckets contain public ACLs:* ', + 's3_encryption': '*These buckets are unencrypted:* ', + 'user_inactivekeys': '*Users with inactive keys:* ', + 'user_keysrotation': '*Users with keys to rotate:* ', + 'secgrp_unrestricted_access': '*Insecure services:* ', + 'cloudtrails': '*These trails are disabled or contain delivery errors:* ', + 'ebs_unencrypted_volume': '*These EBS volumes are unencrypted:* ', + 'ebs_public_snapshot': '*These EBS snapshots are public:* ', + 'rds_public_snapshot': '*These RDS snapshots are public:* ', + 'sqs_public_access': '*These SQS are publicly accessible:* ', + 'rds_encryption': '*These RDS instances are unencrypted:* ' +} + + +def format_scan_account_result(scan_result): + result = "" + for region in scan_result: + if not any(v for k, v in scan_result[region].items()): + continue + result += f"*Found these issues in {region} region:* \n" + for sec_feature in scan_result[region]: + if scan_result[region][sec_feature]: + result += message_mapping[sec_feature] + issues_id = [issue['id'] for issue in scan_result[region][sec_feature]] + issues = ','.join(issues_id) + result += '[' + issues + ']\n' + return result + + +@respond_to('^scan account (?P.*)$', re.IGNORECASE) +def scan_account(message, account_num): + api_token = config.api.token + api_url = config.api.url + '/identify' + headers = {'Auth': api_token} + resp = requests.post(api_url, json={'account_id': account_num}, headers=headers) + if resp.status_code != 200: + message.reply(f'Failed to start scan for account {account_num}, {resp.text}') + return + message.reply(f'Scan for account {account_num} has been started. When the scan is finished,' + f'you will be notified with results.') + request_id = resp.json()['request_id'] + time_start = time.time() + while time.time() - time_start < 300: + resp = requests.get(api_url + '/' + request_id, headers=headers) + if resp.json()['scan_status'] == 'COMPLETE': + return message.reply(format_scan_account_result(resp.json()['scan_results'])) + if resp.json()['scan_status'] == 'FAILED': + return message.reply(f'Scan of account {account_num} is failed. Please try again later.') + time.sleep(5) + return message.reply('Sorry, but current scan takes too long to finish.') diff --git a/hammer/tools/ddb_inject_credentials.py b/hammer/tools/ddb_inject_credentials.py index 0e006fcb..0d91a85c 100755 --- a/hammer/tools/ddb_inject_credentials.py +++ b/hammer/tools/ddb_inject_credentials.py @@ -14,6 +14,9 @@ parser.add_argument("--hammer-api-token", dest="hammer_api_token", nargs='?', const=-1, type=str, help="Hammer API token") + parser.add_argument("--hammer-api-url", + dest="hammer_api_url", nargs='?', const=-1, type=str, + help="Hammer API url") parser.add_argument("--slack-api-token", dest="slack_api_token", default=None, help="Slack API token") @@ -40,6 +43,7 @@ if args.slack_api_token is not None: creds["slack"] = {"api_token": args.slack_api_token} + if all(x is not None for x in [args.jira_key_cert_file, args.jira_consumer_key, args.jira_access_token, @@ -58,6 +62,9 @@ # generate new secret if secret value is not set creds["api"] = {"token": secrets.token_hex() if args.hammer_api_token == -1 else args.hammer_api_token} + if args.hammer_api_url != None: + creds["api"]["url"] = args.hammer_api_url + if not creds: print(f"no credentials detected, please check CLI arguments") exit(1)