From 68ff414df4411fb380e958f29ebd62b526f416bc Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 19 Feb 2019 11:20:53 -0500 Subject: [PATCH 001/154] Added ConfigService, broken atm --- ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js | 2 ++ ScoutSuite/providers/aws/configs/services.py | 9 +++++++++ ScoutSuite/providers/aws/metadata.json | 9 +++++++++ 3 files changed, 20 insertions(+) diff --git a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js index 585fd4a82..8df89cadb 100644 --- a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js +++ b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js @@ -988,6 +988,8 @@ function make_title (title) { return 'CloudWatch'; } else if (title == 'cloudformation') { return 'CloudFormation'; + } else if (title == 'config') { + return 'Config'; } else if (title == 'awslambda') { return 'Lambda'; } else if (title == 'dynamodb') { diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 0f7ec191c..3b7b95065 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -26,6 +26,10 @@ from ScoutSuite.providers.base.configs.services import BaseServicesConfig from ScoutSuite.utils import format_service_name +try: + from ScoutSuite.providers.aws.services.config_private import ConfigConfig +except ImportError: + pass try: from ScoutSuite.providers.aws.services.dynamodb_private import DynamoDBConfig except ImportError: @@ -38,6 +42,7 @@ class AWSServicesConfig(BaseServicesConfig): :ivar cloudtrail: CloudTrail configuration :ivar cloudwatch: CloudWatch configuration: TODO + :ivar config: Config configuration :ivar dynamodb: DynomaDB configuration :ivar ec2: EC2 configuration :ivar iam: IAM configuration @@ -79,6 +84,10 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.dynamodb = DynamoDBConfig(metadata['database']['dynamodb'], thread_config) except NameError as e: pass + try: + self.config = ConfigConfig(metadata['management']['config'], thread_config) + except NameError as e: + pass def _is_provider(self, provider_name): return provider_name == 'aws' diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index ed650617b..4cc5fcf1f 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -58,6 +58,15 @@ "path": "services.cloudwatch.statistics" } } + }, + "config": { + "resources": { + "recorders": { + "api_call": "describe_configuration_recorder_status", + "response": "Recorders", + "path": "services.config.regions.id.recorders" + } + } } }, "messaging": { From c6f6aace2da95dc105487ce4dec5d30b6e7c8eb4 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 19 Feb 2019 11:48:25 -0500 Subject: [PATCH 002/154] Renamed Config to ConfigService --- ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js | 4 ++-- ScoutSuite/providers/aws/configs/services.py | 6 +++--- ScoutSuite/utils.py | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js index 8df89cadb..c6909a4b1 100644 --- a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js +++ b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js @@ -988,8 +988,8 @@ function make_title (title) { return 'CloudWatch'; } else if (title == 'cloudformation') { return 'CloudFormation'; - } else if (title == 'config') { - return 'Config'; + } else if (title == 'configservice') { + return 'ConfigService'; } else if (title == 'awslambda') { return 'Lambda'; } else if (title == 'dynamodb') { diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 3b7b95065..820f3c672 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -27,7 +27,7 @@ from ScoutSuite.utils import format_service_name try: - from ScoutSuite.providers.aws.services.config_private import ConfigConfig + from ScoutSuite.providers.aws.services.configservice_private import ConfigServiceConfig except ImportError: pass try: @@ -42,7 +42,7 @@ class AWSServicesConfig(BaseServicesConfig): :ivar cloudtrail: CloudTrail configuration :ivar cloudwatch: CloudWatch configuration: TODO - :ivar config: Config configuration + :ivar configservice: ConfigService configuration :ivar dynamodb: DynomaDB configuration :ivar ec2: EC2 configuration :ivar iam: IAM configuration @@ -85,7 +85,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): except NameError as e: pass try: - self.config = ConfigConfig(metadata['management']['config'], thread_config) + self.configservice = ConfigServiceConfig(metadata['management']['config'], thread_config) except NameError as e: pass diff --git a/ScoutSuite/utils.py b/ScoutSuite/utils.py index f5ab11c95..328129527 100644 --- a/ScoutSuite/utils.py +++ b/ScoutSuite/utils.py @@ -15,7 +15,9 @@ 'cloudformation': 'CloudFormation', 'cloudtrail': 'CloudTrail', 'cloudwatch': 'CloudWatch', + 'configservice': 'ConfigService', 'directconnect': 'Direct Connect', + 'dynamodb': 'DynamoDB', 'elasticache': 'ElastiCache', 'lambda': 'Lambda', 'redshift': 'RedShift', From c98f510683b4ce763ecb14772d8c08d69dcf761a Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 19 Feb 2019 12:35:16 -0500 Subject: [PATCH 003/154] Replaced plain text replacement with regex --- ScoutSuite/providers/aws/configs/regions.py | 5 ++--- ScoutSuite/providers/base/configs/base.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 2ed181690..08aa99815 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -50,9 +50,8 @@ def __init__(self, service_metadata=None, thread_config=4): self.regions = {} self.thread_config = thread_configs[thread_config] - self.service = \ - type(self).__name__.replace('Config', '').lower() # TODO: use regex with EOS instead of plain replace - + self.service = re.sub(r'Config$', "", type(self).__name__).lower() + # Booleans that define if threads should keep running self.run_q_threads = True self.run_qr_threads = True diff --git a/ScoutSuite/providers/base/configs/base.py b/ScoutSuite/providers/base/configs/base.py index 813d0b14e..554ea4ce9 100644 --- a/ScoutSuite/providers/base/configs/base.py +++ b/ScoutSuite/providers/base/configs/base.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import copy +import re from threading import Thread @@ -40,8 +41,7 @@ def __init__(self, thread_config=4, **kwargs): self.library_type = None if not hasattr(self, 'library_type') else self.library_type - self.service = type(self).__name__.replace('Config', - '').lower() # TODO: use regex with EOS instead of plain replace + self.service = re.sub(r'Config$', "", type(self).__name__).lower() self.thread_config = thread_configs[thread_config] # Booleans that define if threads should keep running From 87e645cc190be1b1c492ced586716c85ff9a2617 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 19 Feb 2019 13:25:03 -0500 Subject: [PATCH 004/154] Pre third name change --- ScoutSuite/providers/aws/metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 4cc5fcf1f..4612bb014 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -62,7 +62,7 @@ "config": { "resources": { "recorders": { - "api_call": "describe_configuration_recorder_status", + "api_call": "describe_configuration_recorder_status999", "response": "Recorders", "path": "services.config.regions.id.recorders" } From ef60c17a1cad019478da5b8d6e8d47a989045472 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 19 Feb 2019 15:23:26 -0500 Subject: [PATCH 005/154] Now properly detecting if config is on/off and added view --- ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js | 4 ++-- ScoutSuite/providers/aws/configs/services.py | 6 +++--- ScoutSuite/providers/aws/metadata.json | 4 ++-- ScoutSuite/utils.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js index c6909a4b1..8df89cadb 100644 --- a/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js +++ b/ScoutSuite/output/data/inc-scoutsuite/scoutsuite.js @@ -988,8 +988,8 @@ function make_title (title) { return 'CloudWatch'; } else if (title == 'cloudformation') { return 'CloudFormation'; - } else if (title == 'configservice') { - return 'ConfigService'; + } else if (title == 'config') { + return 'Config'; } else if (title == 'awslambda') { return 'Lambda'; } else if (title == 'dynamodb') { diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 820f3c672..3b7b95065 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -27,7 +27,7 @@ from ScoutSuite.utils import format_service_name try: - from ScoutSuite.providers.aws.services.configservice_private import ConfigServiceConfig + from ScoutSuite.providers.aws.services.config_private import ConfigConfig except ImportError: pass try: @@ -42,7 +42,7 @@ class AWSServicesConfig(BaseServicesConfig): :ivar cloudtrail: CloudTrail configuration :ivar cloudwatch: CloudWatch configuration: TODO - :ivar configservice: ConfigService configuration + :ivar config: Config configuration :ivar dynamodb: DynomaDB configuration :ivar ec2: EC2 configuration :ivar iam: IAM configuration @@ -85,7 +85,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): except NameError as e: pass try: - self.configservice = ConfigServiceConfig(metadata['management']['config'], thread_config) + self.config = ConfigConfig(metadata['management']['config'], thread_config) except NameError as e: pass diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 4612bb014..357bd0a18 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -62,8 +62,8 @@ "config": { "resources": { "recorders": { - "api_call": "describe_configuration_recorder_status999", - "response": "Recorders", + "api_call": "describe_configuration_recorders", + "response": "ConfigurationRecorders", "path": "services.config.regions.id.recorders" } } diff --git a/ScoutSuite/utils.py b/ScoutSuite/utils.py index 328129527..24ccc0b20 100644 --- a/ScoutSuite/utils.py +++ b/ScoutSuite/utils.py @@ -15,7 +15,7 @@ 'cloudformation': 'CloudFormation', 'cloudtrail': 'CloudTrail', 'cloudwatch': 'CloudWatch', - 'configservice': 'ConfigService', + 'config': 'Config', 'directconnect': 'Direct Connect', 'dynamodb': 'DynamoDB', 'elasticache': 'ElastiCache', From 10075b11ef766f9e26f20f0cfb7bb68145e39e22 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 19 Feb 2019 16:17:37 -0500 Subject: [PATCH 006/154] Trying to push metadata change --- ScoutSuite/providers/aws/metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 357bd0a18..4f7dc8b38 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -62,7 +62,7 @@ "config": { "resources": { "recorders": { - "api_call": "describe_configuration_recorders", + "api_call": "describe_configuration_recorders111", "response": "ConfigurationRecorders", "path": "services.config.regions.id.recorders" } From a2e97d058d36f72791eb9f38b2e64dd256a7b027 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 19 Feb 2019 16:17:59 -0500 Subject: [PATCH 007/154] Trying to fix back metatada --- ScoutSuite/providers/aws/metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 4f7dc8b38..357bd0a18 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -62,7 +62,7 @@ "config": { "resources": { "recorders": { - "api_call": "describe_configuration_recorders111", + "api_call": "describe_configuration_recorders", "response": "ConfigurationRecorders", "path": "services.config.regions.id.recorders" } From 4ca5306dcb945cfab7735dd71c896811b947e5eb Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 19 Feb 2019 17:07:15 -0500 Subject: [PATCH 008/154] Moving KMS stuff to private repo --- .../aws/services.kms.regions.id.keys.html | 29 ----------- ScoutSuite/providers/aws/configs/services.py | 11 +++-- ScoutSuite/providers/aws/services/kms.py | 49 ------------------- 3 files changed, 8 insertions(+), 81 deletions(-) delete mode 100644 ScoutSuite/output/data/html/partials/aws/services.kms.regions.id.keys.html delete mode 100644 ScoutSuite/providers/aws/services/kms.py diff --git a/ScoutSuite/output/data/html/partials/aws/services.kms.regions.id.keys.html b/ScoutSuite/output/data/html/partials/aws/services.kms.regions.id.keys.html deleted file mode 100644 index d84a053b4..000000000 --- a/ScoutSuite/output/data/html/partials/aws/services.kms.regions.id.keys.html +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 0f7ec191c..2da906d8b 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -14,7 +14,6 @@ from ScoutSuite.providers.aws.services.elbv2 import ELBv2Config from ScoutSuite.providers.aws.services.emr import EMRConfig from ScoutSuite.providers.aws.services.iam import IAMConfig -from ScoutSuite.providers.aws.services.kms import KMSConfig from ScoutSuite.providers.aws.services.rds import RDSConfig from ScoutSuite.providers.aws.services.redshift import RedshiftConfig from ScoutSuite.providers.aws.services.route53 import Route53Config, Route53DomainsConfig @@ -30,7 +29,10 @@ from ScoutSuite.providers.aws.services.dynamodb_private import DynamoDBConfig except ImportError: pass - +try: + from ScoutSuite.providers.aws.services.kms import KMSConfig +except ImportError: + pass class AWSServicesConfig(BaseServicesConfig): """ @@ -63,7 +65,6 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config) self.emr = EMRConfig(metadata['analytics']['emr'], thread_config) self.iam = IAMConfig(thread_config) - self.kms = KMSConfig(metadata['security']['kms'], thread_config) self.awslambda = LambdaConfig(metadata['compute']['awslambda'], thread_config) self.redshift = RedshiftConfig(metadata['database']['redshift'], thread_config) self.rds = RDSConfig(metadata['database']['rds'], thread_config) @@ -79,6 +80,10 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.dynamodb = DynamoDBConfig(metadata['database']['dynamodb'], thread_config) except NameError as e: pass + try: + self.kms = KMSConfig(metadata['security']['kms'], thread_config) + except NameError as e: + pass def _is_provider(self, provider_name): return provider_name == 'aws' diff --git a/ScoutSuite/providers/aws/services/kms.py b/ScoutSuite/providers/aws/services/kms.py deleted file mode 100644 index 2892609ac..000000000 --- a/ScoutSuite/providers/aws/services/kms.py +++ /dev/null @@ -1,49 +0,0 @@ -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - - -class KMSRegionConfig(RegionConfig): - """ - KMS Configuration for a single AWS region - """ - - def parse_key(self, global_params, region, key): - """ - Parse a single Key Management Service - - :param global_params: Parameters shared for all regions - :param region: Name of the AWS region - :param key: Key - """ - - customer_master_key = {'id': key.pop('KeyId'), 'arn': key.pop('KeyArn')} - - api_client = api_clients[region] - - rotation_status = api_client.get_key_rotation_status(KeyId=customer_master_key['id']) - customer_master_key['rotation_enabled'] = rotation_status['KeyRotationEnabled'] - - key_information = api_client.describe_key(KeyId=customer_master_key['id']) - customer_master_key['description'] = key_information['KeyMetadata']['Description'] - - alias = api_client.list_aliases(KeyId=customer_master_key['id']) - if len(alias['Aliases']): - customer_master_key['name'] = alias['Aliases'][0]['AliasName'][6:len(alias['Aliases'][0]['AliasName'])] - else: - customer_master_key['name'] = 'unnamed key' - - self.keys[len(self.keys)] = customer_master_key - - -######################################## -# KMSConfig -######################################## - -class KMSConfig(RegionalServiceConfig): - """ - KMS configuration for all AWS regions - """ - - region_config_class = KMSRegionConfig - - def __init__(self, service_metadata, thread_config=4): - super(KMSConfig, self).__init__(service_metadata, thread_config) From 860ade2ce1aa18070f78d244a3824f5c979256f4 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 19 Feb 2019 17:24:22 -0500 Subject: [PATCH 009/154] Typo --- ScoutSuite/providers/aws/configs/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 2da906d8b..7ff917660 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -30,7 +30,7 @@ except ImportError: pass try: - from ScoutSuite.providers.aws.services.kms import KMSConfig + from ScoutSuite.providers.aws.services.kms_private import KMSConfig except ImportError: pass From 4f05ccb4eeb247c1bde65a8abd5f894c0f00a462 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Wed, 20 Feb 2019 09:51:14 -0500 Subject: [PATCH 010/154] Added regions to config metadata --- ScoutSuite/providers/aws/metadata.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json index 357bd0a18..0b4c2b7bd 100644 --- a/ScoutSuite/providers/aws/metadata.json +++ b/ScoutSuite/providers/aws/metadata.json @@ -61,6 +61,10 @@ }, "config": { "resources": { + "regions": { + "cols": 2, + "path": "services.config.regions" + }, "recorders": { "api_call": "describe_configuration_recorders", "response": "ConfigurationRecorders", From 563615b65d4b9bb7a6427e2ddf3f743c5ca75832 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 08:34:08 -0500 Subject: [PATCH 011/154] [Opinel] Removed CLI --- opinel/utils/cli_parser.py | 179 ------------------------------------- 1 file changed, 179 deletions(-) delete mode 100644 opinel/utils/cli_parser.py diff --git a/opinel/utils/cli_parser.py b/opinel/utils/cli_parser.py deleted file mode 100644 index dbb8d6ce2..000000000 --- a/opinel/utils/cli_parser.py +++ /dev/null @@ -1,179 +0,0 @@ -# -*- coding: utf-8 -*- - -import argparse -import json -import os -import sys -import tempfile - -opinel_arg_dir = os.path.join(os.path.expanduser('~'), '.aws/opinel') - -class OpinelArgumentParser(object): - """ - """ - - def __init__(self, tool_name = ''): - self.parser = argparse.ArgumentParser() - self.default_args = read_default_args(tool_name) - - def add_argument(self, argument_name, help = None, dest = None, nargs = None, default = None, action = None, choices = None): - - # Built-in, common arguments - if argument_name == 'debug': - self.parser.add_argument('--debug', - dest='debug', - default=False, - action='store_true', - help='Print the stack trace when exception occurs' if not help else help) - elif argument_name == 'dry-run': - self.parser.add_argument('--dry-run', - dest='dry_run', - default=False, - action='store_true', - help='Executes read-only actions (check status, describe*, get*, list*...)' if not help else help) - elif argument_name == 'profile': - default = os.environ.get('AWS_PROFILE', 'default') - default_origin = " (from AWS_PROFILE)." if 'AWS_PROFILE' in os.environ else "." - self.parser.add_argument('--profile', - dest='profile', - default=[default], - nargs='+', - help='Name of the profile. Defaults to %(default)s' + default_origin if not help else help) - elif argument_name == 'regions': - self.parser.add_argument('--regions', - dest='regions', - default=[], - nargs='+', - help='Name of regions to run the tool in, defaults to all' if not help else help) - elif argument_name == 'partition-name': - self.parser.add_argument('--partition-name', - dest='partition_name', - default='aws', - help='Switch out of the public AWS partition (e.g. US gov or China)') - elif argument_name == 'vpc': - self.parser.add_argument('--vpc', - dest='vpc', - default=[], - nargs='+', - help='Name of VPC to run the tool in, defaults to all' if not help else help) - elif argument_name == 'force': - self.parser.add_argument('--force', - dest='force_write', - default=False, - action='store_true', - help='Overwrite existing files' if not help else help) - elif argument_name == 'ip-ranges': - self.parser.add_argument('--ip-ranges', - dest='ip_ranges', - default=[], - nargs='+', - help='Config file(s) that contain your known IP ranges.' if not help else help) - elif argument_name == 'ip-ranges-name-key': - self.parser.add_argument('--ip-ranges-name-key', - dest='ip_ranges_name_key', - default='name', - help='Name of the key containing the display name of a known CIDR.' if not help else help) - elif argument_name == 'mfa-serial': - self.parser.add_argument('--mfa-serial', - dest='mfa_serial', - default=None, - help='ARN of the user\'s MFA device' if not help else help) - elif argument_name == 'mfa-code': - self.parser.add_argument('--mfa-code', - dest='mfa_code', - default=None, - help='Six-digit code displayed on the MFA device.' if not help else help) - elif argument_name == 'csv-credentials': - self.parser.add_argument('--csv-credentials', - dest='csv_credentials', - default=None, - help='Path to a CSV file containing the access key ID and secret key' if not help else help) - elif argument_name == 'user-name': - self.parser.add_argument('--user-name', - dest='user_name', - default=[], - nargs='+', - help='Name of the user.' if not help else help) - elif argument_name == 'bucket-name': - self.parser.add_argument('--bucket-name', - dest='bucket_name', - default=[None], - help='Name of the s3 bucket.' if not help else help) - elif argument_name == 'group-name': - self.parser.add_argument('--group-name', - dest='group_name', - default=[], - nargs='+', - help='Name of the IAM group.' if not help else help) - - # Default - elif help != None and default != None and (nargs != None or action != None): - dest = argument_name.replace('-', '_') if not dest else dest - if nargs: - if not choices: - self.parser.add_argument('--%s' % argument_name, - dest = dest, - default = self.default_args[dest] if dest in self.default_args else default, - nargs = nargs, - help = help) - else: - self.parser.add_argument('--%s' % argument_name, - dest = dest, - default = self.default_args[dest] if dest in self.default_args else default, - nargs = nargs, - choices = choices, - help = help) - elif action: - self.parser.add_argument('--%s' % argument_name, - dest = dest, - default = self.default_args[dest] if dest in self.default_args else default, - action = action, - help = help) - - # Error - else: - raise Exception('Invalid parameter name %s' % argument_name) - - - def parse_args(self): - args = self.parser.parse_args() - return args - - -def read_default_args(tool_name): - """ - Read default argument values for a given tool - - :param tool_name: Name of the script to read the default arguments for - :return: Dictionary of default arguments (shared + tool-specific) - """ - global opinel_arg_dir - - profile_name = 'default' - # h4ck to have an early read of the profile name - for i, arg in enumerate(sys.argv): - if arg == '--profile' and len(sys.argv) >= i + 1: - profile_name = sys.argv[i + 1] - #if not os.path.isdir(opinel_arg_dir): - # os.makedirs(opinel_arg_dir) - if not os.path.isdir(opinel_arg_dir): - try: - os.makedirs(opinel_arg_dir) - except: - # Within AWS Lambda, home directories are not writable. This attempts to detect that... - # ...and uses the /tmp folder, which *is* writable in AWS Lambda - opinel_arg_dir = os.path.join(tempfile.gettempdir(), '.aws/opinel') - if not os.path.isdir(opinel_arg_dir): - os.makedirs(opinel_arg_dir) - opinel_arg_file = os.path.join(opinel_arg_dir, '%s.json' % profile_name) - default_args = {} - if os.path.isfile(opinel_arg_file): - with open(opinel_arg_file, 'rt') as f: - all_args = json.load(f) - for target in all_args: - if tool_name.endswith(target): - default_args.update(all_args[target]) - for k in all_args['shared']: - if k not in default_args: - default_args[k] = all_args['shared'][k] - return default_args From 649307767e0c4a1b7d57c771cefdfc90950dc5ea Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 08:36:44 -0500 Subject: [PATCH 012/154] [Opinel] Removed data/requirements.txt --- opinel/data/requirements.txt | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 opinel/data/requirements.txt diff --git a/opinel/data/requirements.txt b/opinel/data/requirements.txt deleted file mode 100644 index 8ba8bd8a8..000000000 --- a/opinel/data/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -boto3>=1.4.4 -requests>=2.4.0,<3.0.0 -netaddr>=0.7.11 -iampoliciesgonewild>=1.0.6.2 -pyyaml>=3.12 From f9f5cf03accb30d38891eee06c3d0fc5b2dcdac6 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Thu, 21 Feb 2019 08:43:10 -0500 Subject: [PATCH 013/154] Removed dead code and merged try-catch of private imports --- ScoutSuite/providers/aws/configs/services.py | 35 +------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 9fdf53bf3..23290e87c 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -27,13 +27,7 @@ try: from ScoutSuite.providers.aws.services.config_private import ConfigConfig -except ImportError: - pass -try: from ScoutSuite.providers.aws.services.dynamodb_private import DynamoDBConfig -except ImportError: - pass -try: from ScoutSuite.providers.aws.services.kms_private import KMSConfig except ImportError: pass @@ -83,37 +77,10 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): try: self.config = ConfigConfig(metadata['management']['config'], thread_config) - except NameError as e: - pass - try: self.dynamodb = DynamoDBConfig(metadata['database']['dynamodb'], thread_config) - except NameError as e: - pass - try: self.kms = KMSConfig(metadata['security']['kms'], thread_config) except NameError as e: pass def _is_provider(self, provider_name): - return provider_name == 'aws' - - # TODO is this ever called? - # def single_service_pass(self): - # pass - - # TODO is this ever called? - # def multi_service_pass(self): - # pass - - # TODO is this ever called? - # def postprocessing(self): - # for service in self.services: - # method_name = '%s_postprocessing' % service - # if method_name in globals(): - # try: - # printInfo('Post-processing %s config...' % format_service_name(service)) - # method = globals()[method_name] - # method(aws_config) - # except Exception as e: - # printException(e) - # pass + return provider_name == 'aws' \ No newline at end of file From a82b9c8b6fcec1cb2ad319ef6146d6a693ec1b5b Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 08:58:50 -0500 Subject: [PATCH 014/154] [Opinel] Moved console to ScoutSuite --- ScoutSuite/__listall__.py | 2 +- ScoutSuite/__main__.py | 2 +- ScoutSuite/core/exceptions.py | 2 +- ScoutSuite/core/processingengine.py | 2 +- ScoutSuite/core/rule.py | 3 +-- ScoutSuite/core/rule_definition.py | 2 +- ScoutSuite/core/ruleset.py | 2 +- ScoutSuite/core/utils.py | 2 +- ScoutSuite/output/html.py | 4 ++-- ScoutSuite/output/js.py | 2 +- ScoutSuite/output/utils.py | 2 +- ScoutSuite/providers/aws/configs/regions.py | 2 +- ScoutSuite/providers/aws/configs/services.py | 3 --- ScoutSuite/providers/aws/provider.py | 4 ++-- ScoutSuite/providers/aws/services/ec2.py | 3 +-- ScoutSuite/providers/aws/services/iam.py | 2 +- ScoutSuite/providers/aws/services/rds.py | 2 +- ScoutSuite/providers/aws/services/s3.py | 2 +- ScoutSuite/providers/azure/provider.py | 2 +- ScoutSuite/providers/azure/utils.py | 2 +- ScoutSuite/providers/base/configs/base.py | 2 +- ScoutSuite/providers/base/configs/browser.py | 2 +- ScoutSuite/providers/base/configs/services.py | 4 ++-- ScoutSuite/providers/base/provider.py | 3 +-- ScoutSuite/providers/gcp/configs/base.py | 2 +- ScoutSuite/providers/gcp/configs/services.py | 3 --- ScoutSuite/providers/gcp/provider.py | 2 +- ScoutSuite/providers/gcp/services/cloudresourcemanager.py | 2 +- ScoutSuite/providers/gcp/services/cloudsql.py | 3 +-- ScoutSuite/providers/gcp/services/cloudstorage.py | 2 +- ScoutSuite/providers/gcp/services/computeengine.py | 3 --- ScoutSuite/providers/gcp/services/iam.py | 3 +-- ScoutSuite/providers/gcp/services/stackdriverlogging.py | 2 -- ScoutSuite/providers/gcp/services/stackdrivermonitoring.py | 2 -- ScoutSuite/providers/gcp/utils.py | 2 +- {opinel/utils => core}/console.py | 3 ++- opinel/services/cloudformation.py | 5 ++--- opinel/services/iam.py | 2 +- opinel/services/organizations.py | 2 +- opinel/utils/aws.py | 2 +- opinel/utils/conditions.py | 2 +- opinel/utils/credentials.py | 4 ++-- opinel/utils/fs.py | 2 +- opinel/utils/globals.py | 2 +- opinel/utils/profiles.py | 3 +-- opinel/utils/threads.py | 2 +- tests/test-rules-processingengine.py | 2 +- tests/test-rules-ruleset.py | 2 +- tests/test-scoutsuite.py | 1 - tests/test-utils_cloudwatch.py | 2 +- tests/test-utils_sns.py | 2 +- 51 files changed, 51 insertions(+), 71 deletions(-) rename {opinel/utils => core}/console.py (99%) diff --git a/ScoutSuite/__listall__.py b/ScoutSuite/__listall__.py index f5a6c7ad0..e4598fe0b 100644 --- a/ScoutSuite/__listall__.py +++ b/ScoutSuite/__listall__.py @@ -7,7 +7,7 @@ try: from opinel.utils.globals import check_requirements - from opinel.utils.console import configPrintException, printError, printException, printInfo + from core.console import configPrintException, printError, printException, printInfo except Exception as e: print('Error: Scout2 depends on the opinel package. Install all the requirements with the following command:') print(' $ pip install -r requirements.txt') diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 1acc5d39f..5504cb50e 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -6,7 +6,7 @@ import os import webbrowser -from opinel.utils.console import configPrintException, printInfo, printDebug +from core.console import configPrintException, printInfo, printDebug from opinel.utils.profiles import AWSProfiles from ScoutSuite import AWSCONFIG diff --git a/ScoutSuite/core/exceptions.py b/ScoutSuite/core/exceptions.py index 79035b5f7..d76976750 100644 --- a/ScoutSuite/core/exceptions.py +++ b/ScoutSuite/core/exceptions.py @@ -3,7 +3,7 @@ Exceptions handling """ -from opinel.utils.console import printDebug +from core.console import printDebug from ScoutSuite import EXCEPTIONS from ScoutSuite.output.js import JavaScriptReaderWriter diff --git a/ScoutSuite/core/processingengine.py b/ScoutSuite/core/processingengine.py index a4bea6769..c26559891 100644 --- a/ScoutSuite/core/processingengine.py +++ b/ScoutSuite/core/processingengine.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printDebug, printError, printException +from core.console import printDebug, printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.core.utils import recurse diff --git a/ScoutSuite/core/rule.py b/ScoutSuite/core/rule.py index 2fff065e1..6b48b4889 100644 --- a/ScoutSuite/core/rule.py +++ b/ScoutSuite/core/rule.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- import json -import os import re from opinel.utils.fs import read_ip_ranges -from opinel.utils.console import printDebug, printError, printException +from core.console import printError from ScoutSuite.utils import format_service_name diff --git a/ScoutSuite/core/rule_definition.py b/ScoutSuite/core/rule_definition.py index d0a6a1657..1e92ad6d6 100644 --- a/ScoutSuite/core/rule_definition.py +++ b/ScoutSuite/core/rule_definition.py @@ -3,7 +3,7 @@ import json import os -from opinel.utils.console import printDebug, printError, printException +from core.console import printError class RuleDefinition(object): diff --git a/ScoutSuite/core/ruleset.py b/ScoutSuite/core/ruleset.py index 7b14eb552..a32c5191a 100644 --- a/ScoutSuite/core/ruleset.py +++ b/ScoutSuite/core/ruleset.py @@ -4,7 +4,7 @@ import os import tempfile -from opinel.utils.console import printDebug, printError, prompt_4_yes_no +from core.console import printDebug, printError, prompt_4_yes_no from ScoutSuite.core.rule import Rule from ScoutSuite.core.rule_definition import RuleDefinition diff --git a/ScoutSuite/core/utils.py b/ScoutSuite/core/utils.py index af7229ccf..1b25face9 100644 --- a/ScoutSuite/core/utils.py +++ b/ScoutSuite/core/utils.py @@ -8,7 +8,7 @@ import re from opinel.utils.conditions import pass_condition -from opinel.utils.console import printError, printException +from core.console import printError, printException from ScoutSuite.core import condition_operators from ScoutSuite.providers.base.configs.browser import get_value_at diff --git a/ScoutSuite/output/html.py b/ScoutSuite/output/html.py index 5a7321a35..9b01fd079 100644 --- a/ScoutSuite/output/html.py +++ b/ScoutSuite/output/html.py @@ -7,10 +7,10 @@ import zipfile import dateutil.tz -from opinel.utils.console import printInfo, printException +from core.console import printInfo, printException from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, \ - GENERATOR_FILE, REPORT_TITLE + GENERATOR_FILE from ScoutSuite.output.js import JavaScriptReaderWriter from ScoutSuite.output.utils import get_filename, prompt_4_overwrite diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index 7292384d4..5df552290 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -6,7 +6,7 @@ import json import os -from opinel.utils.console import printException, printInfo +from core.console import printException, printInfo from ScoutSuite import DEFAULT_REPORT_DIR from ScoutSuite.output.utils import get_filename, prompt_4_overwrite diff --git a/ScoutSuite/output/utils.py b/ScoutSuite/output/utils.py index 748ec5e4e..20ef42211 100644 --- a/ScoutSuite/output/utils.py +++ b/ScoutSuite/output/utils.py @@ -4,7 +4,7 @@ import os import sys -from opinel.utils.console import printError +from core.console import printError from six.moves import input diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 08aa99815..1d4943b23 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -13,7 +13,7 @@ from queue import Queue from opinel.utils.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response -from opinel.utils.console import printException, printInfo +from core.console import printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.base.configs import resource_id_map diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 9fdf53bf3..f2ac13ba6 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError, printException, printInfo, printDebug - from ScoutSuite.providers.aws.services.awslambda import LambdaConfig from ScoutSuite.providers.aws.services.cloudformation import CloudFormationConfig from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig @@ -23,7 +21,6 @@ from ScoutSuite.providers.aws.services.sqs import SQSConfig from ScoutSuite.providers.aws.services.vpc import VPCConfig from ScoutSuite.providers.base.configs.services import BaseServicesConfig -from ScoutSuite.utils import format_service_name try: from ScoutSuite.providers.aws.services.config_private import ConfigConfig diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 4054370a4..5a57fa55a 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -6,7 +6,7 @@ try: from opinel.utils.aws import get_aws_account_id, get_partition_name - from opinel.utils.console import configPrintException, printInfo, printDebug + from core.console import configPrintException, printInfo, printDebug from opinel.utils.credentials import read_creds from opinel.utils.globals import check_requirements from opinel.utils.profiles import AWSProfiles @@ -16,7 +16,7 @@ print(e) sys.exit(42) -from opinel.utils.console import printDebug, printError, printException, printInfo +from core.console import printDebug, printError, printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.services import AWSServicesConfig diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 915589599..62de742d9 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -8,13 +8,12 @@ import base64 from opinel.utils.aws import get_name -from opinel.utils.console import printException, printInfo +from core.console import printException, printInfo from opinel.utils.fs import load_data from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.providers.base.configs.browser import get_attribute_at -from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.utils import get_keys, ec2_classic from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index aa6c5d4db..a60f85d4a 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -2,7 +2,7 @@ from botocore.exceptions import ClientError from opinel.utils.aws import connect_service, handle_truncated_response -from opinel.utils.console import printError, printException +from core.console import printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index 2f8a190a4..55c127590 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from opinel.utils.aws import handle_truncated_response -from opinel.utils.console import printError, printException +from core.console import printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index d3a0be048..8456a1549 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -8,7 +8,7 @@ from botocore.exceptions import ClientError from opinel.services.s3 import get_s3_bucket_location from opinel.utils.aws import handle_truncated_response -from opinel.utils.console import printError, printException, printInfo +from core.console import printError, printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/azure/provider.py b/ScoutSuite/providers/azure/provider.py index 41a249740..5937c48a7 100644 --- a/ScoutSuite/providers/azure/provider.py +++ b/ScoutSuite/providers/azure/provider.py @@ -5,7 +5,7 @@ from getpass import getpass -from opinel.utils.console import printError, printException +from core.console import printError, printException from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.azure.configs.services import AzureServicesConfig diff --git a/ScoutSuite/providers/azure/utils.py b/ScoutSuite/providers/azure/utils.py index 7096b0b26..717bf5654 100644 --- a/ScoutSuite/providers/azure/utils.py +++ b/ScoutSuite/providers/azure/utils.py @@ -2,7 +2,7 @@ import re -from opinel.utils.console import printException, printInfo +from core.console import printException from azure.mgmt.storage import StorageManagementClient from azure.mgmt.monitor import MonitorManagementClient diff --git a/ScoutSuite/providers/base/configs/base.py b/ScoutSuite/providers/base/configs/base.py index 554ea4ce9..cdd62d216 100644 --- a/ScoutSuite/providers/base/configs/base.py +++ b/ScoutSuite/providers/base/configs/base.py @@ -21,7 +21,7 @@ from ScoutSuite.providers.azure.utils import azure_connect_service from opinel.utils.aws import build_region_list -from opinel.utils.console import printException, printInfo +from core.console import printException, printInfo from ScoutSuite.output.console import FetchStatusLogger from ScoutSuite.utils import format_service_name diff --git a/ScoutSuite/providers/base/configs/browser.py b/ScoutSuite/providers/base/configs/browser.py index 59aade272..3dee54d71 100644 --- a/ScoutSuite/providers/base/configs/browser.py +++ b/ScoutSuite/providers/base/configs/browser.py @@ -2,7 +2,7 @@ import copy -from opinel.utils.console import printError, printException +from core.console import printError, printException ######################################## diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 6fcf89400..5a89a9ba2 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError, printException, printDebug -from opinel.utils.aws import get_aws_account_id, get_partition_name +from core.console import printError, printException, printDebug +from opinel.utils.aws import get_partition_name class BaseServicesConfig(object): diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index 80cba7087..a6e6aa78c 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -7,8 +7,7 @@ import sys import copy -from opinel.utils.console import printException, printInfo -from opinel.utils.globals import manage_dictionary +from core.console import printException, printInfo from ScoutSuite import __version__ as scout2_version from ScoutSuite.providers.base.configs.browser import get_object_at diff --git a/ScoutSuite/providers/gcp/configs/base.py b/ScoutSuite/providers/gcp/configs/base.py index 756d95601..3db702c33 100644 --- a/ScoutSuite/providers/gcp/configs/base.py +++ b/ScoutSuite/providers/gcp/configs/base.py @@ -14,7 +14,7 @@ from google.cloud import container_v1 from googleapiclient.errors import HttpError -from opinel.utils.console import printException, printError +from core.console import printException, printError from ScoutSuite.providers.base.configs.base import BaseConfig from ScoutSuite.providers.gcp.utils import gcp_connect_service diff --git a/ScoutSuite/providers/gcp/configs/services.py b/ScoutSuite/providers/gcp/configs/services.py index 514b6265b..0693601a4 100644 --- a/ScoutSuite/providers/gcp/configs/services.py +++ b/ScoutSuite/providers/gcp/configs/services.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError, printException, printDebug - from ScoutSuite.providers.base.configs.services import BaseServicesConfig from ScoutSuite.providers.gcp.services.cloudstorage import CloudStorageConfig from ScoutSuite.providers.gcp.services.cloudsql import CloudSQLConfig from ScoutSuite.providers.gcp.services.iam import IAMConfig from ScoutSuite.providers.gcp.services.stackdriverlogging import StackdriverLoggingConfig -from ScoutSuite.providers.gcp.services.stackdrivermonitoring import StackdriverMonitoringConfig from ScoutSuite.providers.gcp.services.computeengine import ComputeEngineConfig from ScoutSuite.providers.gcp.services.cloudresourcemanager import CloudResourceManager diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py index ede8e9c3c..adf6721be 100644 --- a/ScoutSuite/providers/gcp/provider.py +++ b/ScoutSuite/providers/gcp/provider.py @@ -5,7 +5,7 @@ import google.auth import googleapiclient -from opinel.utils.console import printError, printException, printInfo +from core.console import printError, printException, printInfo from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.gcp.configs.services import GCPServicesConfig diff --git a/ScoutSuite/providers/gcp/services/cloudresourcemanager.py b/ScoutSuite/providers/gcp/services/cloudresourcemanager.py index 385a505d3..c1ec32399 100644 --- a/ScoutSuite/providers/gcp/services/cloudresourcemanager.py +++ b/ScoutSuite/providers/gcp/services/cloudresourcemanager.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from opinel.utils.console import printError, printException, printInfo +from core.console import printException class CloudResourceManager(GCPBaseConfig): diff --git a/ScoutSuite/providers/gcp/services/cloudsql.py b/ScoutSuite/providers/gcp/services/cloudsql.py index de24abd6c..bb252e1a8 100644 --- a/ScoutSuite/providers/gcp/services/cloudsql.py +++ b/ScoutSuite/providers/gcp/services/cloudsql.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError +from core.console import printError -import operator from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/services/cloudstorage.py b/ScoutSuite/providers/gcp/services/cloudstorage.py index b7c1cacfd..72f260783 100644 --- a/ScoutSuite/providers/gcp/services/cloudstorage.py +++ b/ScoutSuite/providers/gcp/services/cloudstorage.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError +from core.console import printError from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/services/computeengine.py b/ScoutSuite/providers/gcp/services/computeengine.py index 72c82b28a..4a03d8677 100644 --- a/ScoutSuite/providers/gcp/services/computeengine.py +++ b/ScoutSuite/providers/gcp/services/computeengine.py @@ -2,9 +2,6 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from googleapiclient.errors import HttpError -import json -from opinel.utils.console import printException, printError class ComputeEngineConfig(GCPBaseConfig): targets = ( diff --git a/ScoutSuite/providers/gcp/services/iam.py b/ScoutSuite/providers/gcp/services/iam.py index 8a58a5f9d..ddce638fd 100644 --- a/ScoutSuite/providers/gcp/services/iam.py +++ b/ScoutSuite/providers/gcp/services/iam.py @@ -2,10 +2,9 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from opinel.utils.console import printError, printException, printInfo +from core.console import printError from googleapiclient import discovery -from ScoutSuite.providers.gcp.utils import gcp_connect_service class IAMConfig(GCPBaseConfig): diff --git a/ScoutSuite/providers/gcp/services/stackdriverlogging.py b/ScoutSuite/providers/gcp/services/stackdriverlogging.py index 5bb5878ec..53998933b 100644 --- a/ScoutSuite/providers/gcp/services/stackdriverlogging.py +++ b/ScoutSuite/providers/gcp/services/stackdriverlogging.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError - from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/services/stackdrivermonitoring.py b/ScoutSuite/providers/gcp/services/stackdrivermonitoring.py index d8d0e94ed..ddb003b55 100644 --- a/ScoutSuite/providers/gcp/services/stackdrivermonitoring.py +++ b/ScoutSuite/providers/gcp/services/stackdrivermonitoring.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError - from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/utils.py b/ScoutSuite/providers/gcp/utils.py index c2ea17de3..adc81dbcd 100644 --- a/ScoutSuite/providers/gcp/utils.py +++ b/ScoutSuite/providers/gcp/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printException +from core.console import printException from google.cloud import storage from google.cloud import logging as stackdriver_logging diff --git a/opinel/utils/console.py b/core/console.py similarity index 99% rename from opinel/utils/console.py rename to core/console.py index 6b4a71357..efd4b1b21 100644 --- a/opinel/utils/console.py +++ b/core/console.py @@ -7,7 +7,8 @@ try: input = raw_input -except NameError: pass +except NameError: + pass ######################################## diff --git a/opinel/services/cloudformation.py b/opinel/services/cloudformation.py index 0478e681c..480a6a2e0 100644 --- a/opinel/services/cloudformation.py +++ b/opinel/services/cloudformation.py @@ -1,14 +1,13 @@ # -*- coding: utf-8 -*- -import json import os import re import time from opinel.utils.aws import connect_service, handle_truncated_response -from opinel.utils.console import printDebug, printInfo, printError, printException, prompt_4_yes_no +from core.console import printDebug, printInfo, printError, printException from opinel.utils.fs import read_file -from opinel.utils.globals import snake_to_camel, snake_to_words +from opinel.utils.globals import snake_to_camel re_iam_capability = re.compile('.*?AWS::IAM.*?', re.DOTALL | re.MULTILINE) diff --git a/opinel/services/iam.py b/opinel/services/iam.py index 41c8a4347..a59df2d44 100644 --- a/opinel/services/iam.py +++ b/opinel/services/iam.py @@ -4,7 +4,7 @@ from opinel.utils.aws import handle_truncated_response from opinel.utils.credentials import generate_password -from opinel.utils.console import printInfo, printError, printException +from core.console import printInfo, printError, printException diff --git a/opinel/services/organizations.py b/opinel/services/organizations.py index fee03f458..355efbfe2 100644 --- a/opinel/services/organizations.py +++ b/opinel/services/organizations.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from opinel.utils.aws import handle_truncated_response -from opinel.utils.console import printDebug, printInfo +from core.console import printDebug, printInfo def get_organization_account_ids(api_client, exceptions = [], quiet = True): diff --git a/opinel/utils/aws.py b/opinel/utils/aws.py index 6872d129c..705bb61e4 100644 --- a/opinel/utils/aws.py +++ b/opinel/utils/aws.py @@ -5,7 +5,7 @@ from collections import Counter import time -from opinel.utils.console import printInfo, printException +from core.console import printInfo, printException diff --git a/opinel/utils/conditions.py b/opinel/utils/conditions.py index 6e1982205..a3cac7fca 100644 --- a/opinel/utils/conditions.py +++ b/opinel/utils/conditions.py @@ -6,7 +6,7 @@ import netaddr import re -from opinel.utils.console import printError +from core.console import printError from iampoliciesgonewild import get_actions_from_statement, _expand_wildcard_action diff --git a/opinel/utils/credentials.py b/opinel/utils/credentials.py index 126b0a6b7..0bb0b151c 100644 --- a/opinel/utils/credentials.py +++ b/opinel/utils/credentials.py @@ -11,8 +11,8 @@ import requests # TODO: get rid of that and make sure urllib2 validates certs ? import string -from opinel.utils.console import printException, printError, printInfo -from opinel.utils.console import prompt_4_mfa_code +from core.console import printException, printError, printInfo +from core.console import prompt_4_mfa_code from opinel.utils.fs import save_blob_as_json from opinel.utils.aws import connect_service diff --git a/opinel/utils/fs.py b/opinel/utils/fs.py index 8c2ee07d6..7978a8ec4 100644 --- a/opinel/utils/fs.py +++ b/opinel/utils/fs.py @@ -6,7 +6,7 @@ import os import yaml -from opinel.utils.console import printError, printException, prompt_4_overwrite +from core.console import printError, printException, prompt_4_overwrite from opinel.utils.conditions import pass_condition diff --git a/opinel/utils/globals.py b/opinel/utils/globals.py index 5164e7cd4..34072085c 100644 --- a/opinel/utils/globals.py +++ b/opinel/utils/globals.py @@ -6,7 +6,7 @@ import re from opinel import __version__ as OPINEL_VERSION -from opinel.utils.console import printError +from core.console import printError ######################################## diff --git a/opinel/utils/profiles.py b/opinel/utils/profiles.py index 4b398eb58..081337f6b 100644 --- a/opinel/utils/profiles.py +++ b/opinel/utils/profiles.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- -import fileinput import os import re from opinel.utils.aws import get_aws_account_id -from opinel.utils.console import printDebug +from core.console import printDebug from opinel.utils.credentials import read_creds aws_dir = os.path.join(os.path.expanduser('~'), '.aws') diff --git a/opinel/utils/threads.py b/opinel/utils/threads.py index 9309b715a..d78742a54 100644 --- a/opinel/utils/threads.py +++ b/opinel/utils/threads.py @@ -8,7 +8,7 @@ # Python3 from queue import Queue -from opinel.utils.console import printException +from core.console import printException diff --git a/tests/test-rules-processingengine.py b/tests/test-rules-processingengine.py index f95d66d76..f93da05dd 100644 --- a/tests/test-rules-processingengine.py +++ b/tests/test-rules-processingengine.py @@ -4,7 +4,7 @@ import os import tempfile -from opinel.utils.console import configPrintException, printError +from core.console import configPrintException, printError from ScoutSuite.core.processingengine import ProcessingEngine from ScoutSuite.core.ruleset import Ruleset diff --git a/tests/test-rules-ruleset.py b/tests/test-rules-ruleset.py index fe5593d5f..a639ef7a5 100644 --- a/tests/test-rules-ruleset.py +++ b/tests/test-rules-ruleset.py @@ -3,7 +3,7 @@ import os from mock import patch -from opinel.utils.console import configPrintException, printDebug +from core.console import configPrintException, printDebug from ScoutSuite.core.rule import Rule from ScoutSuite.core.ruleset import Ruleset diff --git a/tests/test-scoutsuite.py b/tests/test-scoutsuite.py index 362bad187..1af03246d 100644 --- a/tests/test-scoutsuite.py +++ b/tests/test-scoutsuite.py @@ -3,7 +3,6 @@ import mock from nose.plugins.attrib import attr -from opinel.utils.console import configPrintException from opinel.utils.credentials import read_creds_from_environment_variables from ScoutSuite.__main__ import * diff --git a/tests/test-utils_cloudwatch.py b/tests/test-utils_cloudwatch.py index 1341d8a46..46d9d44f3 100644 --- a/tests/test-utils_cloudwatch.py +++ b/tests/test-utils_cloudwatch.py @@ -1,5 +1,5 @@ from ScoutSuite.utils_cloudwatch import * -from opinel.utils.console import configPrintException +from core.console import configPrintException # diff --git a/tests/test-utils_sns.py b/tests/test-utils_sns.py index 34e747a1e..7f3d0fce4 100644 --- a/tests/test-utils_sns.py +++ b/tests/test-utils_sns.py @@ -1,5 +1,5 @@ from ScoutSuite.utils_sns import * -from opinel.utils.console import configPrintException +from core.console import configPrintException # From 7f67ccc6c67e68c8f699ac603cf86a00e168fba4 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 09:01:59 -0500 Subject: [PATCH 015/154] [Opinel] Moved cli_parser to core --- Scout.py | 2 +- ScoutSuite/__listall__.py | 2 +- ScoutSuite/__main__.py | 2 +- ScoutSuite/{ => core}/cli_parser.py | 0 {core => ScoutSuite/core}/console.py | 0 ScoutSuite/core/exceptions.py | 2 +- ScoutSuite/core/processingengine.py | 2 +- ScoutSuite/core/rule.py | 2 +- ScoutSuite/core/rule_definition.py | 2 +- ScoutSuite/core/ruleset.py | 2 +- ScoutSuite/core/utils.py | 2 +- ScoutSuite/output/html.py | 2 +- ScoutSuite/output/js.py | 2 +- ScoutSuite/output/utils.py | 2 +- ScoutSuite/providers/aws/configs/regions.py | 2 +- ScoutSuite/providers/aws/provider.py | 4 ++-- ScoutSuite/providers/aws/services/ec2.py | 2 +- ScoutSuite/providers/aws/services/iam.py | 2 +- ScoutSuite/providers/aws/services/rds.py | 2 +- ScoutSuite/providers/aws/services/s3.py | 2 +- ScoutSuite/providers/azure/provider.py | 2 +- ScoutSuite/providers/azure/utils.py | 2 +- ScoutSuite/providers/base/configs/base.py | 2 +- ScoutSuite/providers/base/configs/browser.py | 2 +- ScoutSuite/providers/base/configs/services.py | 2 +- ScoutSuite/providers/base/provider.py | 2 +- ScoutSuite/providers/gcp/configs/base.py | 2 +- ScoutSuite/providers/gcp/provider.py | 2 +- ScoutSuite/providers/gcp/services/cloudresourcemanager.py | 2 +- ScoutSuite/providers/gcp/services/cloudsql.py | 2 +- ScoutSuite/providers/gcp/services/cloudstorage.py | 2 +- ScoutSuite/providers/gcp/services/iam.py | 2 +- ScoutSuite/providers/gcp/utils.py | 2 +- opinel/services/cloudformation.py | 2 +- opinel/services/iam.py | 2 +- opinel/services/organizations.py | 2 +- opinel/utils/aws.py | 2 +- opinel/utils/conditions.py | 2 +- opinel/utils/credentials.py | 4 ++-- opinel/utils/fs.py | 2 +- opinel/utils/globals.py | 2 +- opinel/utils/profiles.py | 2 +- opinel/utils/threads.py | 2 +- tests/test-main.py | 2 +- tests/test-rules-processingengine.py | 2 +- tests/test-rules-ruleset.py | 2 +- tests/test-utils_cloudwatch.py | 2 +- tests/test-utils_sns.py | 2 +- 48 files changed, 48 insertions(+), 48 deletions(-) rename ScoutSuite/{ => core}/cli_parser.py (100%) rename {core => ScoutSuite/core}/console.py (100%) diff --git a/Scout.py b/Scout.py index 5edb1c55f..84babab73 100755 --- a/Scout.py +++ b/Scout.py @@ -4,7 +4,7 @@ import sys from ScoutSuite.__main__ import main as scout -from ScoutSuite.cli_parser import ScoutSuiteArgumentParser +from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser if __name__ == "__main__": parser = ScoutSuiteArgumentParser() diff --git a/ScoutSuite/__listall__.py b/ScoutSuite/__listall__.py index e4598fe0b..917027fce 100644 --- a/ScoutSuite/__listall__.py +++ b/ScoutSuite/__listall__.py @@ -7,7 +7,7 @@ try: from opinel.utils.globals import check_requirements - from core.console import configPrintException, printError, printException, printInfo + from ScoutSuite.core.console import configPrintException, printError, printException, printInfo except Exception as e: print('Error: Scout2 depends on the opinel package. Install all the requirements with the following command:') print(' $ pip install -r requirements.txt') diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 5504cb50e..6f9b102de 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -6,7 +6,7 @@ import os import webbrowser -from core.console import configPrintException, printInfo, printDebug +from ScoutSuite.core.console import configPrintException, printInfo, printDebug from opinel.utils.profiles import AWSProfiles from ScoutSuite import AWSCONFIG diff --git a/ScoutSuite/cli_parser.py b/ScoutSuite/core/cli_parser.py similarity index 100% rename from ScoutSuite/cli_parser.py rename to ScoutSuite/core/cli_parser.py diff --git a/core/console.py b/ScoutSuite/core/console.py similarity index 100% rename from core/console.py rename to ScoutSuite/core/console.py diff --git a/ScoutSuite/core/exceptions.py b/ScoutSuite/core/exceptions.py index d76976750..a6e5dd11b 100644 --- a/ScoutSuite/core/exceptions.py +++ b/ScoutSuite/core/exceptions.py @@ -3,7 +3,7 @@ Exceptions handling """ -from core.console import printDebug +from ScoutSuite.core.console import printDebug from ScoutSuite import EXCEPTIONS from ScoutSuite.output.js import JavaScriptReaderWriter diff --git a/ScoutSuite/core/processingengine.py b/ScoutSuite/core/processingengine.py index c26559891..7c320d0ed 100644 --- a/ScoutSuite/core/processingengine.py +++ b/ScoutSuite/core/processingengine.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core.console import printDebug, printError, printException +from ScoutSuite.core.console import printDebug, printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.core.utils import recurse diff --git a/ScoutSuite/core/rule.py b/ScoutSuite/core/rule.py index 6b48b4889..6492d3962 100644 --- a/ScoutSuite/core/rule.py +++ b/ScoutSuite/core/rule.py @@ -4,7 +4,7 @@ import re from opinel.utils.fs import read_ip_ranges -from core.console import printError +from ScoutSuite.core.console import printError from ScoutSuite.utils import format_service_name diff --git a/ScoutSuite/core/rule_definition.py b/ScoutSuite/core/rule_definition.py index 1e92ad6d6..a071c6d0e 100644 --- a/ScoutSuite/core/rule_definition.py +++ b/ScoutSuite/core/rule_definition.py @@ -3,7 +3,7 @@ import json import os -from core.console import printError +from ScoutSuite.core.console import printError class RuleDefinition(object): diff --git a/ScoutSuite/core/ruleset.py b/ScoutSuite/core/ruleset.py index a32c5191a..791be7e5e 100644 --- a/ScoutSuite/core/ruleset.py +++ b/ScoutSuite/core/ruleset.py @@ -4,7 +4,7 @@ import os import tempfile -from core.console import printDebug, printError, prompt_4_yes_no +from ScoutSuite.core.console import printDebug, printError, prompt_4_yes_no from ScoutSuite.core.rule import Rule from ScoutSuite.core.rule_definition import RuleDefinition diff --git a/ScoutSuite/core/utils.py b/ScoutSuite/core/utils.py index 1b25face9..92565a03f 100644 --- a/ScoutSuite/core/utils.py +++ b/ScoutSuite/core/utils.py @@ -8,7 +8,7 @@ import re from opinel.utils.conditions import pass_condition -from core.console import printError, printException +from ScoutSuite.core.console import printError, printException from ScoutSuite.core import condition_operators from ScoutSuite.providers.base.configs.browser import get_value_at diff --git a/ScoutSuite/output/html.py b/ScoutSuite/output/html.py index 9b01fd079..f1c285cec 100644 --- a/ScoutSuite/output/html.py +++ b/ScoutSuite/output/html.py @@ -7,7 +7,7 @@ import zipfile import dateutil.tz -from core.console import printInfo, printException +from ScoutSuite.core.console import printInfo, printException from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, \ GENERATOR_FILE diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index 5df552290..37b4e4488 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -6,7 +6,7 @@ import json import os -from core.console import printException, printInfo +from ScoutSuite.core.console import printException, printInfo from ScoutSuite import DEFAULT_REPORT_DIR from ScoutSuite.output.utils import get_filename, prompt_4_overwrite diff --git a/ScoutSuite/output/utils.py b/ScoutSuite/output/utils.py index 20ef42211..7027fafbd 100644 --- a/ScoutSuite/output/utils.py +++ b/ScoutSuite/output/utils.py @@ -4,7 +4,7 @@ import os import sys -from core.console import printError +from ScoutSuite.core.console import printError from six.moves import input diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 1d4943b23..b2b21bf1a 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -13,7 +13,7 @@ from queue import Queue from opinel.utils.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response -from core.console import printException, printInfo +from ScoutSuite.core.console import printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.base.configs import resource_id_map diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 5a57fa55a..01468d160 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -6,7 +6,7 @@ try: from opinel.utils.aws import get_aws_account_id, get_partition_name - from core.console import configPrintException, printInfo, printDebug + from ScoutSuite.core.console import configPrintException, printInfo, printDebug from opinel.utils.credentials import read_creds from opinel.utils.globals import check_requirements from opinel.utils.profiles import AWSProfiles @@ -16,7 +16,7 @@ print(e) sys.exit(42) -from core.console import printDebug, printError, printException, printInfo +from ScoutSuite.core.console import printDebug, printError, printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.services import AWSServicesConfig diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 62de742d9..3ef3da9c8 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -8,7 +8,7 @@ import base64 from opinel.utils.aws import get_name -from core.console import printException, printInfo +from ScoutSuite.core.console import printException, printInfo from opinel.utils.fs import load_data from opinel.utils.globals import manage_dictionary diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index a60f85d4a..0223b8eda 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -2,7 +2,7 @@ from botocore.exceptions import ClientError from opinel.utils.aws import connect_service, handle_truncated_response -from core.console import printError, printException +from ScoutSuite.core.console import printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index 55c127590..da470ae86 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from opinel.utils.aws import handle_truncated_response -from core.console import printError, printException +from ScoutSuite.core.console import printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 8456a1549..b2924d054 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -8,7 +8,7 @@ from botocore.exceptions import ClientError from opinel.services.s3 import get_s3_bucket_location from opinel.utils.aws import handle_truncated_response -from core.console import printError, printException, printInfo +from ScoutSuite.core.console import printError, printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/azure/provider.py b/ScoutSuite/providers/azure/provider.py index 5937c48a7..7875e2369 100644 --- a/ScoutSuite/providers/azure/provider.py +++ b/ScoutSuite/providers/azure/provider.py @@ -5,7 +5,7 @@ from getpass import getpass -from core.console import printError, printException +from ScoutSuite.core.console import printError, printException from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.azure.configs.services import AzureServicesConfig diff --git a/ScoutSuite/providers/azure/utils.py b/ScoutSuite/providers/azure/utils.py index 717bf5654..8e3dae2e6 100644 --- a/ScoutSuite/providers/azure/utils.py +++ b/ScoutSuite/providers/azure/utils.py @@ -2,7 +2,7 @@ import re -from core.console import printException +from ScoutSuite.core.console import printException from azure.mgmt.storage import StorageManagementClient from azure.mgmt.monitor import MonitorManagementClient diff --git a/ScoutSuite/providers/base/configs/base.py b/ScoutSuite/providers/base/configs/base.py index cdd62d216..95681f2f2 100644 --- a/ScoutSuite/providers/base/configs/base.py +++ b/ScoutSuite/providers/base/configs/base.py @@ -21,7 +21,7 @@ from ScoutSuite.providers.azure.utils import azure_connect_service from opinel.utils.aws import build_region_list -from core.console import printException, printInfo +from ScoutSuite.core.console import printException, printInfo from ScoutSuite.output.console import FetchStatusLogger from ScoutSuite.utils import format_service_name diff --git a/ScoutSuite/providers/base/configs/browser.py b/ScoutSuite/providers/base/configs/browser.py index 3dee54d71..5a5e9e171 100644 --- a/ScoutSuite/providers/base/configs/browser.py +++ b/ScoutSuite/providers/base/configs/browser.py @@ -2,7 +2,7 @@ import copy -from core.console import printError, printException +from ScoutSuite.core.console import printError, printException ######################################## diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 5a89a9ba2..d6ad4dc4e 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core.console import printError, printException, printDebug +from ScoutSuite.core.console import printError, printException, printDebug from opinel.utils.aws import get_partition_name class BaseServicesConfig(object): diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index a6e6aa78c..ea375383a 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -7,7 +7,7 @@ import sys import copy -from core.console import printException, printInfo +from ScoutSuite.core.console import printException, printInfo from ScoutSuite import __version__ as scout2_version from ScoutSuite.providers.base.configs.browser import get_object_at diff --git a/ScoutSuite/providers/gcp/configs/base.py b/ScoutSuite/providers/gcp/configs/base.py index 3db702c33..f9df42d81 100644 --- a/ScoutSuite/providers/gcp/configs/base.py +++ b/ScoutSuite/providers/gcp/configs/base.py @@ -14,7 +14,7 @@ from google.cloud import container_v1 from googleapiclient.errors import HttpError -from core.console import printException, printError +from ScoutSuite.core.console import printException, printError from ScoutSuite.providers.base.configs.base import BaseConfig from ScoutSuite.providers.gcp.utils import gcp_connect_service diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py index adf6721be..456ded1d8 100644 --- a/ScoutSuite/providers/gcp/provider.py +++ b/ScoutSuite/providers/gcp/provider.py @@ -5,7 +5,7 @@ import google.auth import googleapiclient -from core.console import printError, printException, printInfo +from ScoutSuite.core.console import printError, printException, printInfo from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.gcp.configs.services import GCPServicesConfig diff --git a/ScoutSuite/providers/gcp/services/cloudresourcemanager.py b/ScoutSuite/providers/gcp/services/cloudresourcemanager.py index c1ec32399..f44056b93 100644 --- a/ScoutSuite/providers/gcp/services/cloudresourcemanager.py +++ b/ScoutSuite/providers/gcp/services/cloudresourcemanager.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from core.console import printException +from ScoutSuite.core.console import printException class CloudResourceManager(GCPBaseConfig): diff --git a/ScoutSuite/providers/gcp/services/cloudsql.py b/ScoutSuite/providers/gcp/services/cloudsql.py index bb252e1a8..7ffc2ce27 100644 --- a/ScoutSuite/providers/gcp/services/cloudsql.py +++ b/ScoutSuite/providers/gcp/services/cloudsql.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core.console import printError +from ScoutSuite.core.console import printError from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/services/cloudstorage.py b/ScoutSuite/providers/gcp/services/cloudstorage.py index 72f260783..b558dffed 100644 --- a/ScoutSuite/providers/gcp/services/cloudstorage.py +++ b/ScoutSuite/providers/gcp/services/cloudstorage.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core.console import printError +from ScoutSuite.core.console import printError from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/services/iam.py b/ScoutSuite/providers/gcp/services/iam.py index ddce638fd..be9d91f78 100644 --- a/ScoutSuite/providers/gcp/services/iam.py +++ b/ScoutSuite/providers/gcp/services/iam.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from core.console import printError +from ScoutSuite.core.console import printError from googleapiclient import discovery diff --git a/ScoutSuite/providers/gcp/utils.py b/ScoutSuite/providers/gcp/utils.py index adc81dbcd..0c18a3d92 100644 --- a/ScoutSuite/providers/gcp/utils.py +++ b/ScoutSuite/providers/gcp/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core.console import printException +from ScoutSuite.core.console import printException from google.cloud import storage from google.cloud import logging as stackdriver_logging diff --git a/opinel/services/cloudformation.py b/opinel/services/cloudformation.py index 480a6a2e0..ea1950ad1 100644 --- a/opinel/services/cloudformation.py +++ b/opinel/services/cloudformation.py @@ -5,7 +5,7 @@ import time from opinel.utils.aws import connect_service, handle_truncated_response -from core.console import printDebug, printInfo, printError, printException +from ScoutSuite.core.console import printDebug, printInfo, printError, printException from opinel.utils.fs import read_file from opinel.utils.globals import snake_to_camel diff --git a/opinel/services/iam.py b/opinel/services/iam.py index a59df2d44..33765c995 100644 --- a/opinel/services/iam.py +++ b/opinel/services/iam.py @@ -4,7 +4,7 @@ from opinel.utils.aws import handle_truncated_response from opinel.utils.credentials import generate_password -from core.console import printInfo, printError, printException +from ScoutSuite.core.console import printInfo, printError, printException diff --git a/opinel/services/organizations.py b/opinel/services/organizations.py index 355efbfe2..4578110f8 100644 --- a/opinel/services/organizations.py +++ b/opinel/services/organizations.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from opinel.utils.aws import handle_truncated_response -from core.console import printDebug, printInfo +from ScoutSuite.core.console import printDebug, printInfo def get_organization_account_ids(api_client, exceptions = [], quiet = True): diff --git a/opinel/utils/aws.py b/opinel/utils/aws.py index 705bb61e4..9eac65958 100644 --- a/opinel/utils/aws.py +++ b/opinel/utils/aws.py @@ -5,7 +5,7 @@ from collections import Counter import time -from core.console import printInfo, printException +from ScoutSuite.core.console import printInfo, printException diff --git a/opinel/utils/conditions.py b/opinel/utils/conditions.py index a3cac7fca..9caa2adf9 100644 --- a/opinel/utils/conditions.py +++ b/opinel/utils/conditions.py @@ -6,7 +6,7 @@ import netaddr import re -from core.console import printError +from ScoutSuite.core.console import printError from iampoliciesgonewild import get_actions_from_statement, _expand_wildcard_action diff --git a/opinel/utils/credentials.py b/opinel/utils/credentials.py index 0bb0b151c..ea4ae2b81 100644 --- a/opinel/utils/credentials.py +++ b/opinel/utils/credentials.py @@ -11,8 +11,8 @@ import requests # TODO: get rid of that and make sure urllib2 validates certs ? import string -from core.console import printException, printError, printInfo -from core.console import prompt_4_mfa_code +from ScoutSuite.core.console import printException, printError, printInfo +from ScoutSuite.core.console import prompt_4_mfa_code from opinel.utils.fs import save_blob_as_json from opinel.utils.aws import connect_service diff --git a/opinel/utils/fs.py b/opinel/utils/fs.py index 7978a8ec4..9e804c6a7 100644 --- a/opinel/utils/fs.py +++ b/opinel/utils/fs.py @@ -6,7 +6,7 @@ import os import yaml -from core.console import printError, printException, prompt_4_overwrite +from ScoutSuite.core.console import printError, printException, prompt_4_overwrite from opinel.utils.conditions import pass_condition diff --git a/opinel/utils/globals.py b/opinel/utils/globals.py index 34072085c..0ba1ad6a8 100644 --- a/opinel/utils/globals.py +++ b/opinel/utils/globals.py @@ -6,7 +6,7 @@ import re from opinel import __version__ as OPINEL_VERSION -from core.console import printError +from ScoutSuite.core.console import printError ######################################## diff --git a/opinel/utils/profiles.py b/opinel/utils/profiles.py index 081337f6b..3450f1f59 100644 --- a/opinel/utils/profiles.py +++ b/opinel/utils/profiles.py @@ -4,7 +4,7 @@ import re from opinel.utils.aws import get_aws_account_id -from core.console import printDebug +from ScoutSuite.core.console import printDebug from opinel.utils.credentials import read_creds aws_dir = os.path.join(os.path.expanduser('~'), '.aws') diff --git a/opinel/utils/threads.py b/opinel/utils/threads.py index d78742a54..3eaa7187e 100644 --- a/opinel/utils/threads.py +++ b/opinel/utils/threads.py @@ -8,7 +8,7 @@ # Python3 from queue import Queue -from core.console import printException +from ScoutSuite.core.console import printException diff --git a/tests/test-main.py b/tests/test-main.py index b72a35218..209ec0988 100644 --- a/tests/test-main.py +++ b/tests/test-main.py @@ -3,7 +3,7 @@ from mock import MagicMock, patch from ScoutSuite.__main__ import main -from ScoutSuite.cli_parser import ScoutSuiteArgumentParser +from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser class TestMainClass(TestCase): diff --git a/tests/test-rules-processingengine.py b/tests/test-rules-processingengine.py index f93da05dd..e8d3eab12 100644 --- a/tests/test-rules-processingengine.py +++ b/tests/test-rules-processingengine.py @@ -4,7 +4,7 @@ import os import tempfile -from core.console import configPrintException, printError +from ScoutSuite.core.console import configPrintException, printError from ScoutSuite.core.processingengine import ProcessingEngine from ScoutSuite.core.ruleset import Ruleset diff --git a/tests/test-rules-ruleset.py b/tests/test-rules-ruleset.py index a639ef7a5..f336c3074 100644 --- a/tests/test-rules-ruleset.py +++ b/tests/test-rules-ruleset.py @@ -3,7 +3,7 @@ import os from mock import patch -from core.console import configPrintException, printDebug +from ScoutSuite.core.console import configPrintException, printDebug from ScoutSuite.core.rule import Rule from ScoutSuite.core.ruleset import Ruleset diff --git a/tests/test-utils_cloudwatch.py b/tests/test-utils_cloudwatch.py index 46d9d44f3..1bde42713 100644 --- a/tests/test-utils_cloudwatch.py +++ b/tests/test-utils_cloudwatch.py @@ -1,5 +1,5 @@ from ScoutSuite.utils_cloudwatch import * -from core.console import configPrintException +from ScoutSuite.core.console import configPrintException # diff --git a/tests/test-utils_sns.py b/tests/test-utils_sns.py index 7f3d0fce4..881027b85 100644 --- a/tests/test-utils_sns.py +++ b/tests/test-utils_sns.py @@ -1,5 +1,5 @@ from ScoutSuite.utils_sns import * -from core.console import configPrintException +from ScoutSuite.core.console import configPrintException # From 78f791015afedaa841d153f5dc49f9d17e6b6430 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Thu, 21 Feb 2019 09:19:43 -0500 Subject: [PATCH 016/154] Moved get_s3_bucket_location into file, since it only had one reference --- ScoutSuite/providers/aws/services/s3.py | 12 +++++++++++- opinel/services/s3.py | 12 ------------ 2 files changed, 11 insertions(+), 13 deletions(-) delete mode 100644 opinel/services/s3.py diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index b2924d054..9033b3373 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -6,7 +6,6 @@ import json from botocore.exceptions import ClientError -from opinel.services.s3 import get_s3_bucket_location from opinel.utils.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException, printInfo from opinel.utils.globals import manage_dictionary @@ -407,3 +406,14 @@ def get_s3_list_region(region): return 'cn-north-1' else: return region + + +def get_s3_bucket_location(s3_client, bucket_name): + """ + + :param s3_client: + :param bucket_name: + :return: + """ + location = s3_client.get_bucket_location(Bucket=bucket_name) + return location['LocationConstraint'] if location['LocationConstraint'] else 'us-east-1' diff --git a/opinel/services/s3.py b/opinel/services/s3.py deleted file mode 100644 index 2688cf2bb..000000000 --- a/opinel/services/s3.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- - - -def get_s3_bucket_location(s3_client, bucket_name): - """ - - :param s3_client: - :param bucket_name: - :return: - """ - location = s3_client.get_bucket_location(Bucket = bucket_name) - return location['LocationConstraint'] if location['LocationConstraint'] else 'us-east-1' From 9422e7258123d0eefd102d3f22fe8e0c9460289c Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 09:35:45 -0500 Subject: [PATCH 017/154] [Opinel] Moved small cleanup on console.py --- ScoutSuite/core/console.py | 49 ++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index efd4b1b21..5ec20328a 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -10,14 +10,13 @@ except NameError: pass - ######################################## # Globals ######################################## mfa_serial_format = r'^arn:aws:iam::\d+:mfa/[a-zA-Z0-9\+=,.@_-]+$' re_mfa_serial_format = re.compile(mfa_serial_format) - +re_mfa_code = re.compile(r'^\d{6}\d*$') ######################################## @@ -36,7 +35,6 @@ def configPrintException(enable): verbose_exceptions = enable - ######################################## # Print functions ######################################## @@ -46,11 +44,11 @@ def printDebug(msg): printGeneric(sys.stderr, msg) -def printError(msg, newLine = True): +def printError(msg, newLine=True): printGeneric(sys.stderr, msg, newLine) -def printException(e, debug_only = False): +def printException(e, debug_only=False): global verbose_exceptions if verbose_exceptions: printError(str(traceback.format_exc())) @@ -58,23 +56,22 @@ def printException(e, debug_only = False): printError(str(e)) -def printGeneric(out, msg, newLine = True): +def printGeneric(out, msg, newLine=True): out.write(msg) out.flush() if newLine == True: out.write('\n') -def printInfo(msg, newLine = True ): +def printInfo(msg, newLine=True): printGeneric(sys.stdout, msg, newLine) - ######################################## # Prompt functions ######################################## -def prompt(test_input = None): +def prompt(test_input=None): """ Prompt function that works for Python2 and Python3 @@ -98,7 +95,7 @@ def prompt(test_input = None): return choice -def prompt_4_mfa_code(activate = False, input = None): +def prompt_4_mfa_code(activate=False, input=None): """ Prompt for an MFA code @@ -112,19 +109,16 @@ def prompt_4_mfa_code(activate = False, input = None): prompt_string = 'Enter the next value: ' else: prompt_string = 'Enter your MFA code (or \'q\' to abort): ' - mfa_code = prompt_4_value(prompt_string, no_confirm = True, input = input) - try: - if mfa_code == 'q': - return mfa_code - int(mfa_code) - mfa_code[5] - break - except: + mfa_code = prompt_4_value(prompt_string, no_confirm=True, input=input) + if mfa_code == 'q': + return mfa_code + if not re_mfa_code.match(): printError('Error: your MFA code must only consist of digits and be at least 6 characters long.') + break return mfa_code -def prompt_4_mfa_serial(input = None): +def prompt_4_mfa_serial(input=None): """ Prompt for an MFA serial number @@ -132,10 +126,11 @@ def prompt_4_mfa_serial(input = None): :return: The MFA serial number """ - return prompt_4_value('Enter your MFA serial:', required = False, regex = re_mfa_serial_format, regex_format = mfa_serial_format, input = input) + return prompt_4_value('Enter your MFA serial:', required=False, regex=re_mfa_serial_format, + regex_format=mfa_serial_format, input=input) -def prompt_4_overwrite(filename, force_write, input = None): +def prompt_4_overwrite(filename, force_write, input=None): """ Prompt whether the file should be overwritten @@ -147,10 +142,12 @@ def prompt_4_overwrite(filename, force_write, input = None): """ if not os.path.exists(filename) or force_write: return True - return prompt_4_yes_no('File \'{}\' already exists. Do you want to overwrite it'.format(filename), input = input) + return prompt_4_yes_no('File \'{}\' already exists. Do you want to overwrite it'.format(filename), input=input) -def prompt_4_value(question, choices = None, default = None, display_choices = True, display_indices = False, authorize_list = False, is_question = False, no_confirm = False, required = True, regex = None, regex_format = '', max_laps = 5, input = None, return_index = False): +def prompt_4_value(question, choices=None, default=None, display_choices=True, display_indices=False, + authorize_list=False, is_question=False, no_confirm=False, required=True, regex=None, + regex_format='', max_laps=5, input=None, return_index=False): """ Prompt for a value . . @@ -192,7 +189,7 @@ def prompt_4_value(question, choices = None, default = None, display_choices = T if not choice or choice == '': if default: if no_confirm or prompt_4_yes_no('Use the default value (' + default + ')'): - #return default + # return default choice = default can_return = True elif not required: @@ -220,7 +217,7 @@ def prompt_4_value(question, choices = None, default = None, display_choices = T # Validate against a regex elif regex: if regex.match(choice): - #return choice + # return choice can_return = True else: printError('Error: expected format is: %s' % regex_format) @@ -233,7 +230,7 @@ def prompt_4_value(question, choices = None, default = None, display_choices = T return int(int_choice) if return_index else choice -def prompt_4_yes_no(question, input = None): +def prompt_4_yes_no(question, input=None): """ Prompt for a yes/no or y/n answer . From 6546243b28e3af4767a5201bab79274583df61aa Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 16:01:14 -0500 Subject: [PATCH 018/154] [Opinel] Moved conditions.py --- .../utils => ScoutSuite/core}/conditions.py | 116 +++++++++++++++--- ScoutSuite/core/utils.py | 77 +----------- ScoutSuite/providers/aws/provider.py | 16 +-- opinel/utils/fs.py | 4 +- 4 files changed, 103 insertions(+), 110 deletions(-) rename {opinel/utils => ScoutSuite/core}/conditions.py (59%) diff --git a/opinel/utils/conditions.py b/ScoutSuite/core/conditions.py similarity index 59% rename from opinel/utils/conditions.py rename to ScoutSuite/core/conditions.py index 9caa2adf9..782572c46 100644 --- a/opinel/utils/conditions.py +++ b/ScoutSuite/core/conditions.py @@ -6,10 +6,64 @@ import netaddr import re -from ScoutSuite.core.console import printError - from iampoliciesgonewild import get_actions_from_statement, _expand_wildcard_action +from ScoutSuite.core.console import printError, printException +from ScoutSuite.core import condition_operators + +re_get_value_at = re.compile(r'_GET_VALUE_AT_\((.*?)\)') +re_nested_get_value_at = re.compile(r'_GET_VALUE_AT_\(.*') + + +def pass_conditions(all_info, current_path, conditions, unknown_as_pass_condition=False): + """ + Check that all conditions are passed for the current path. + + :param all_info: All of the services' data + :param current_path: The value of the `path` variable defined in the finding file + :param conditions: The conditions to check as defined in the finding file + :param unknown_as_pass_condition: Consider an undetermined condition as passed + :return: + """ + + # Imported here temporarly to fix cyclic dependency + from ScoutSuite.providers.base.configs.browser import get_value_at + + if len(conditions) == 0: + return True + condition_operator = conditions.pop(0) + for condition in conditions: + if condition[0] in condition_operators: + res = pass_conditions(all_info, current_path, condition, unknown_as_pass_condition) + else: + # Conditions are formed as "path to value", "type of test", "value(s) for test" + path_to_value, test_name, test_values = condition + path_to_value = fix_path_string(all_info, current_path, path_to_value) + target_obj = get_value_at(all_info, current_path, path_to_value) + if type(test_values) != list: + dynamic_value = re_get_value_at.match(test_values) + if dynamic_value: + test_values = get_value_at(all_info, current_path, dynamic_value.groups()[0], True) + try: + res = _pass_condition(target_obj, test_name, test_values) + except Exception as e: + res = True if unknown_as_pass_condition else False + printError('Unable to process testcase \'%s\' on value \'%s\', interpreted as %s.' % ( + test_name, str(target_obj), res)) + printException(e, True) + # Quick exit and + false + if condition_operator == 'and' and not res: + return False + # Quick exit or + true + if condition_operator == 'or' and res: + return True + # Still here ? + # or -> false + # and -> true + if condition_operator == 'or': + return False + else: + return True def __prepare_age_test(a, b): @@ -31,7 +85,7 @@ def __prepare_age_test(a, b): return age, number -def pass_condition(b, test, a): +def _pass_condition(b, test, a): """ Generic test function used by Scout2 / AWS recipes . @@ -51,7 +105,7 @@ def pass_condition(b, test, a): b = str(b) result = (a == b) elif test == 'notEqual': - result = (not pass_condition(b, 'equal', a)) + result = (not _pass_condition(b, 'equal', a)) # More/Less tests elif test == 'lessThan': @@ -67,11 +121,11 @@ def pass_condition(b, test, a): elif test == 'empty': result = ((type(b) == dict and b == {}) or (type(b) == list and b == []) or (type(b) == list and b == [None])) elif test == 'notEmpty': - result = (not pass_condition(b, 'empty', 'a')) + result = (not _pass_condition(b, 'empty', 'a')) elif test == 'null': - result = ((b == None) or (type(b) == str and b == 'None')) + result = ((b is None) or (type(b) == str and b == 'None')) elif test == 'notNull': - result = (not pass_condition(b, 'null', a)) + result = (not _pass_condition(b, 'null', a)) # Boolean tests elif test == 'true': @@ -97,9 +151,9 @@ def pass_condition(b, test, a): elif test == 'containAtLeastOneOf': result = False if not type(b) == list: - b = [ b ] + b = [b] if not type(a) == list: - a = [ a ] + a = [a] for c in b: if type(c): c = str(c) @@ -109,9 +163,9 @@ def pass_condition(b, test, a): elif test == 'containAtLeastOneDifferentFrom': result = False if not type(b) == list: - b = [ b ] + b = [b] if not type(a) == list: - a = [ a ] + a = [a] for c in b: if c != None and c != '' and c not in a: result = True @@ -119,9 +173,9 @@ def pass_condition(b, test, a): elif test == 'containNoneOf': result = True if not type(b) == list: - b = [ b ] + b = [b] if not type(a) == list: - a = [ a ] + a = [a] for c in b: if c in a: result = False @@ -130,14 +184,14 @@ def pass_condition(b, test, a): # Regex tests elif test == 'match': if type(a) != list: - a = [ a ] + a = [a] b = str(b) for c in a: - if re.match(c, b) != None: + if not re.match(c, b): result = True break elif test == 'notMatch': - result = (not pass_condition(b, 'match', a)) + result = (not _pass_condition(b, 'match', a)) # Date tests elif test == 'priorToDate': @@ -156,14 +210,14 @@ def pass_condition(b, test, a): result = False grant = netaddr.IPNetwork(b) if type(a) != list: - a = [ a ] + a = [a] for c in a: known_subnet = netaddr.IPNetwork(c) if grant in known_subnet: result = True break elif test == 'notInSubnets': - result = (not pass_condition(b, 'inSubnets', a)) + result = (not _pass_condition(b, 'inSubnets', a)) # Policy statement tests elif test == 'containAction': @@ -177,13 +231,13 @@ def pass_condition(b, test, a): result = True break elif test == 'notContainAction': - result = (not pass_condition(b, 'containAction', a)) + result = (not _pass_condition(b, 'containAction', a)) elif test == 'containAtLeastOneAction': result = False if type(b) != dict: b = json.loads(b) if type(a) != list: - a = [ a ] + a = [a] actions = get_actions_from_statement(b) for c in a: if c.lower() in actions: @@ -214,3 +268,25 @@ def pass_condition(b, test, a): raise Exception return result + + +def fix_path_string(all_info, current_path, path_to_value): + # handle nested _GET_VALUE_AT_... + + # Imported here temporarly to fix cyclic dependency + from ScoutSuite.providers.base.configs.browser import get_value_at + + while True: + dynamic_path = re_get_value_at.findall(path_to_value) + if len(dynamic_path) == 0: + break + for dp in dynamic_path: + tmp = dp + while True: + nested = re_nested_get_value_at.findall(tmp) + if len(nested) == 0: + break + tmp = nested[0].replace('_GET_VALUE_AT_(', '', 1) + dv = get_value_at(all_info, current_path, tmp) + path_to_value = path_to_value.replace('_GET_VALUE_AT_(%s)' % tmp, dv) + return path_to_value diff --git a/ScoutSuite/core/utils.py b/ScoutSuite/core/utils.py index 92565a03f..f0e8de441 100644 --- a/ScoutSuite/core/utils.py +++ b/ScoutSuite/core/utils.py @@ -5,34 +5,9 @@ from six import string_types import copy -import re -from opinel.utils.conditions import pass_condition -from ScoutSuite.core.console import printError, printException - -from ScoutSuite.core import condition_operators -from ScoutSuite.providers.base.configs.browser import get_value_at - -re_get_value_at = re.compile(r'_GET_VALUE_AT_\((.*?)\)') -re_nested_get_value_at = re.compile(r'_GET_VALUE_AT_\(.*') - - -def fix_path_string(all_info, current_path, path_to_value): - # handle nested _GET_VALUE_AT_... - while True: - dynamic_path = re_get_value_at.findall(path_to_value) - if len(dynamic_path) == 0: - break - for dp in dynamic_path: - tmp = dp - while True: - nested = re_nested_get_value_at.findall(tmp) - if len(nested) == 0: - break - tmp = nested[0].replace('_GET_VALUE_AT_(', '', 1) - dv = get_value_at(all_info, current_path, tmp) - path_to_value = path_to_value.replace('_GET_VALUE_AT_(%s)' % tmp, dv) - return path_to_value +from ScoutSuite.core.console import printError +from ScoutSuite.core.conditions import pass_conditions, fix_path_string def recurse(all_info, current_info, target_path, current_path, config, add_suffix=False): @@ -106,51 +81,3 @@ def recurse(all_info, current_info, target_path, current_path, config, add_suffi printError('Entry target path: %s' % str(dbg_target_path)) raise Exception return results - - -def pass_conditions(all_info, current_path, conditions, unknown_as_pass_condition=False): - """ - Check that all conditions are passed for the current path. - - :param all_info: All of the services' data - :param current_path: The value of the `path` variable defined in the finding file - :param conditions: The conditions to check as defined in the finding file - :param unknown_as_pass_condition: Consider an undetermined condition as passed - :return: - """ - result = False - if len(conditions) == 0: - return True - condition_operator = conditions.pop(0) - for condition in conditions: - if condition[0] in condition_operators: - res = pass_conditions(all_info, current_path, condition, unknown_as_pass_condition) - else: - # Conditions are formed as "path to value", "type of test", "value(s) for test" - path_to_value, test_name, test_values = condition - path_to_value = fix_path_string(all_info, current_path, path_to_value) - target_obj = get_value_at(all_info, current_path, path_to_value) - if type(test_values) != list: - dynamic_value = re_get_value_at.match(test_values) - if dynamic_value: - test_values = get_value_at(all_info, current_path, dynamic_value.groups()[0], True) - try: - res = pass_condition(target_obj, test_name, test_values) - except Exception as e: - res = True if unknown_as_pass_condition else False - printError('Unable to process testcase \'%s\' on value \'%s\', interpreted as %s.' % ( - test_name, str(target_obj), res)) - printException(e, True) - # Quick exit and + false - if condition_operator == 'and' and not res: - return False - # Quick exit or + true - if condition_operator == 'or' and res: - return True - # Still here ? - # or -> false - # and -> true - if condition_operator == 'or': - return False - else: - return True diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 01468d160..f06280754 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -2,19 +2,9 @@ import copy import os -import sys - -try: - from opinel.utils.aws import get_aws_account_id, get_partition_name - from ScoutSuite.core.console import configPrintException, printInfo, printDebug - from opinel.utils.credentials import read_creds - from opinel.utils.globals import check_requirements - from opinel.utils.profiles import AWSProfiles -except Exception as e: - print('Error: Scout2 depends on the opinel package. Install all the requirements with the following command:') - print(' $ pip install -r requirements.txt') - print(e) - sys.exit(42) + +from opinel.utils.aws import get_aws_account_id +from opinel.utils.credentials import read_creds from ScoutSuite.core.console import printDebug, printError, printException, printInfo from opinel.utils.globals import manage_dictionary diff --git a/opinel/utils/fs.py b/opinel/utils/fs.py index 9e804c6a7..d03a949f2 100644 --- a/opinel/utils/fs.py +++ b/opinel/utils/fs.py @@ -7,7 +7,7 @@ import yaml from ScoutSuite.core.console import printError, printException, prompt_4_overwrite -from opinel.utils.conditions import pass_condition +from ScoutSuite.core.conditions import _pass_condition @@ -82,7 +82,7 @@ def read_ip_ranges(filename, local_file = True, ip_only = False, conditions = [] for condition in conditions: if type(condition) != list or len(condition) < 3: continue - condition_passed = pass_condition(d[condition[0]], condition[1], condition[2]) + condition_passed = _pass_condition(d[condition[0]], condition[1], condition[2]) if not condition_passed: break if condition_passed: From 1c8ebcdebf420b49f2d08c019923baaa7c0ca31d Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 16:06:30 -0500 Subject: [PATCH 019/154] [Opinel] Moved threads.py --- {opinel/utils => ScoutSuite/core}/threads.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) rename {opinel/utils => ScoutSuite/core}/threads.py (84%) diff --git a/opinel/utils/threads.py b/ScoutSuite/core/threads.py similarity index 84% rename from opinel/utils/threads.py rename to ScoutSuite/core/threads.py index 3eaa7187e..5bee3fa8f 100644 --- a/opinel/utils/threads.py +++ b/ScoutSuite/core/threads.py @@ -1,18 +1,12 @@ # -*- coding: utf-8 -*- from threading import Thread -try: - # Python2 - from Queue import Queue -except ImportError: - # Python3 - from queue import Queue +from six.moves.queue import Queue from ScoutSuite.core.console import printException - -def thread_work(targets, function, params = {}, num_threads = 0): +def thread_work(targets, function, params={}, num_threads=0): """ Generic multithreading helper From 9ebeb41258d288d7f7f7a1761906fddd39eb28f1 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 16:15:29 -0500 Subject: [PATCH 020/154] [Opinel] Deleted globals.py --- ScoutSuite/core/processingengine.py | 2 +- ScoutSuite/output/console.py | 2 +- ScoutSuite/providers/aws/configs/regions.py | 3 +- ScoutSuite/providers/aws/provider.py | 3 +- ScoutSuite/providers/aws/services/ec2.py | 3 +- .../providers/aws/services/elasticache.py | 5 +- ScoutSuite/providers/aws/services/elb.py | 4 +- ScoutSuite/providers/aws/services/elbv2.py | 3 +- ScoutSuite/providers/aws/services/emr.py | 2 +- ScoutSuite/providers/aws/services/iam.py | 1 - ScoutSuite/providers/aws/services/rds.py | 4 +- ScoutSuite/providers/aws/services/redshift.py | 3 +- ScoutSuite/providers/aws/services/s3.py | 2 +- ScoutSuite/providers/aws/services/sns.py | 2 +- ScoutSuite/providers/aws/services/vpc.py | 3 +- ScoutSuite/utils.py | 18 +++ opinel/services/cloudformation.py | 10 +- opinel/utils/globals.py | 105 ------------------ 18 files changed, 39 insertions(+), 136 deletions(-) delete mode 100644 opinel/utils/globals.py diff --git a/ScoutSuite/core/processingengine.py b/ScoutSuite/core/processingengine.py index 7c320d0ed..1ceb66008 100644 --- a/ScoutSuite/core/processingengine.py +++ b/ScoutSuite/core/processingengine.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.core.console import printDebug, printError, printException -from opinel.utils.globals import manage_dictionary +from ScoutSuite.utils import manage_dictionary from ScoutSuite.core.utils import recurse diff --git a/ScoutSuite/output/console.py b/ScoutSuite/output/console.py index cdc037601..06c24d239 100644 --- a/ScoutSuite/output/console.py +++ b/ScoutSuite/output/console.py @@ -4,7 +4,7 @@ import re import sys -from opinel.utils.globals import manage_dictionary +from ScoutSuite.utils import manage_dictionary from ScoutSuite.providers.base.configs.browser import get_value_at diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index b2b21bf1a..80d577080 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -14,12 +14,11 @@ from opinel.utils.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response from ScoutSuite.core.console import printException, printInfo -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.base.configs import resource_id_map from ScoutSuite.providers.base.configs.threads import thread_configs from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import format_service_name, is_throttled +from ScoutSuite.utils import format_service_name, is_throttled, manage_dictionary from ScoutSuite.providers.aws.configs.base import BaseConfig from ScoutSuite.output.console import FetchStatusLogger diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index f06280754..200f6784b 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -7,13 +7,12 @@ from opinel.utils.credentials import read_creds from ScoutSuite.core.console import printDebug, printError, printException, printInfo -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.services import AWSServicesConfig from ScoutSuite.providers.base.configs.browser import combine_paths, get_object_at, get_value_at from ScoutSuite.providers.aws.services.vpc import put_cidr_name from ScoutSuite.providers.base.provider import BaseProvider -from ScoutSuite.utils import ec2_classic +from ScoutSuite.utils import ec2_classic, manage_dictionary class AWSProvider(BaseProvider): diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 3ef3da9c8..8ac8371f0 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -10,11 +10,10 @@ from opinel.utils.aws import get_name from ScoutSuite.core.console import printException, printInfo from opinel.utils.fs import load_data -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.providers.base.configs.browser import get_attribute_at -from ScoutSuite.utils import get_keys, ec2_classic +from ScoutSuite.utils import get_keys, ec2_classic, manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/elasticache.py b/ScoutSuite/providers/aws/services/elasticache.py index a3b7c7183..ffdf6a541 100644 --- a/ScoutSuite/providers/aws/services/elasticache.py +++ b/ScoutSuite/providers/aws/services/elasticache.py @@ -1,11 +1,8 @@ # -*- coding: utf-8 -*- -from opinel.utils.globals import manage_dictionary - from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic - +from ScoutSuite.utils import ec2_classic, manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/elb.py b/ScoutSuite/providers/aws/services/elb.py index b9b1deda7..2d35580b4 100644 --- a/ScoutSuite/providers/aws/services/elb.py +++ b/ScoutSuite/providers/aws/services/elb.py @@ -2,12 +2,10 @@ """ ELB-related classes and functions """ -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic, get_keys - +from ScoutSuite.utils import ec2_classic, get_keys, manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index b45647430..df5dc10ab 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -4,11 +4,10 @@ """ from opinel.utils.aws import handle_truncated_response -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic +from ScoutSuite.utils import ec2_classic, manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/emr.py b/ScoutSuite/providers/aws/services/emr.py index dadee2ac3..c7c1f7e63 100644 --- a/ScoutSuite/providers/aws/services/emr.py +++ b/ScoutSuite/providers/aws/services/emr.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.globals import manage_dictionary +from ScoutSuite.utils import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index 0223b8eda..3ef28fd6c 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -3,7 +3,6 @@ from botocore.exceptions import ClientError from opinel.utils.aws import connect_service, handle_truncated_response from ScoutSuite.core.console import printError, printException -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig from ScoutSuite.utils import * diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index da470ae86..bb988a69e 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -2,12 +2,10 @@ from opinel.utils.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic - +from ScoutSuite.utils import ec2_classic, manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index b4d86a4e7..d8a87610c 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -4,11 +4,10 @@ """ from opinel.utils.aws import handle_truncated_response -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic +from ScoutSuite.utils import ec2_classic, manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index b2924d054..6beca7984 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -9,7 +9,7 @@ from opinel.services.s3 import get_s3_bucket_location from opinel.utils.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException, printInfo -from opinel.utils.globals import manage_dictionary +from ScoutSuite.utils import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/sns.py b/ScoutSuite/providers/aws/services/sns.py index f4fa3342e..20b066784 100644 --- a/ScoutSuite/providers/aws/services/sns.py +++ b/ScoutSuite/providers/aws/services/sns.py @@ -5,7 +5,7 @@ import json -from opinel.utils.globals import manage_dictionary +from ScoutSuite.utils import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index 776266147..2c41976aa 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -4,11 +4,10 @@ import copy from opinel.utils.aws import get_name -from opinel.utils.globals import manage_dictionary from opinel.utils.fs import load_data, read_ip_ranges from ScoutSuite.providers.base.configs.browser import get_value_at -from ScoutSuite.utils import ec2_classic, get_keys +from ScoutSuite.utils import ec2_classic, get_keys, manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig from ScoutSuite.providers.aws.configs.vpc import VPCConfig as SingleVPCConfig diff --git a/ScoutSuite/utils.py b/ScoutSuite/utils.py index 24ccc0b20..9f15da47e 100644 --- a/ScoutSuite/utils.py +++ b/ScoutSuite/utils.py @@ -91,3 +91,21 @@ def is_throttled(e): :return: True if it's a throttling exception else False """ return True if (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in [ 'Throttling', 'RequestLimitExceeded', 'ThrottlingException' ]) else False + + +def manage_dictionary(dictionary, key, init, callback=None): + """ + + :param dictionary: + :param key: + :param init: + :param callback: + + :return: + """ + if not str(key) in dictionary: + dictionary[str(key)] = init + manage_dictionary(dictionary, key, init) + if callback: + callback(dictionary[key]) + return dictionary \ No newline at end of file diff --git a/opinel/services/cloudformation.py b/opinel/services/cloudformation.py index ea1950ad1..ca4b1f724 100644 --- a/opinel/services/cloudformation.py +++ b/opinel/services/cloudformation.py @@ -7,10 +7,10 @@ from opinel.utils.aws import connect_service, handle_truncated_response from ScoutSuite.core.console import printDebug, printInfo, printError, printException from opinel.utils.fs import read_file -from opinel.utils.globals import snake_to_camel re_iam_capability = re.compile('.*?AWS::IAM.*?', re.DOTALL | re.MULTILINE) + def create_cloudformation_resource_from_template(api_client, resource_type, name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False, need_on_failure=False): """ @@ -22,7 +22,7 @@ def create_cloudformation_resource_from_template(api_client, resource_type, name :return: """ create = getattr(api_client, 'create_%s' % resource_type) - api_resource_type = snake_to_camel(resource_type) + api_resource_type = _snake_to_camel(resource_type) # Add a timestamps tags.append({'Key': 'OpinelTimestamp', 'Value': str(time.time())}) params = prepare_cloudformation_params(name, template_path, template_parameters, api_resource_type, tags) @@ -274,7 +274,7 @@ def update_cloudformation_resource_from_template(api_client, resource_type, name """ try: update = getattr(api_client, 'update_%s' % resource_type) - api_resource_type = snake_to_camel(resource_type) + api_resource_type = _snake_to_camel(resource_type) # Add a timestamps tags.append({'Key': 'OpinelTimestamp', 'Value': str(time.time())}) params = prepare_cloudformation_params(name, template_path, template_parameters, api_resource_type, tags) @@ -371,3 +371,7 @@ def cloudformation_wait(api_client, resource_type, resource_name, operation_id = printInfo('Status: %s... waiting %d seconds until next check...' % (status, increment)) timer += increment time.sleep(increment) + + +def _snake_to_camel(snake): + return "".join(val.title() for val in snake.split('_')) diff --git a/opinel/utils/globals.py b/opinel/utils/globals.py deleted file mode 100644 index 0ba1ad6a8..000000000 --- a/opinel/utils/globals.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- - -import boto3 -from distutils.version import StrictVersion -import os -import re - -from opinel import __version__ as OPINEL_VERSION -from ScoutSuite.core.console import printError - - -######################################## -# Regex -######################################## - -re_opinel = re.compile(r'^opinel>=([0-9.]+),<([0-9.]+).*') -re_boto3 = re.compile(r'^boto3>=([0-9.]+)(,<([0-9.]+).*)?') - - -######################################## -# Functions -######################################## - -def check_requirements(script_path, requirements_file = None): - """ - Check versions of opinel and boto3 - :param script_path: - :return: - """ - script_dir = os.path.dirname(script_path) - opinel_min_version = opinel_max_version = boto3_min_version = boto3_max_version = None - # Requirements file is either next to the script or in data/requirements - if not requirements_file: - requirements_file = os.path.join(script_dir, 'data/requirements.txt') - if not os.path.isfile(requirements_file): - requirements_file = os.path.join(script_dir, 'requirements.txt') - with open(requirements_file, 'rt') as f: - for requirement in f.readlines(): - opinel_requirements = re_opinel.match(requirement) - if opinel_requirements: - opinel_requirements = opinel_requirements.groups() - opinel_min_version = opinel_requirements[0] - opinel_max_version = opinel_requirements[1] - boto3_requirements = re_boto3.match(requirement) - if boto3_requirements: - boto3_requirements = boto3_requirements.groups() - boto3_min_version = boto3_requirements[0] - boto3_max_version = boto3_requirements[1] - if not check_versions(opinel_min_version, OPINEL_VERSION, opinel_max_version, 'opinel'): - return False - if not check_versions(boto3_min_version, boto3.__version__, boto3_max_version, 'boto3'): - return False - return True - - -def check_versions(min_version, installed_version, max_version, package_name, strict = False): - """ - - :param min_version: - :param installed_version: - :param max_version: - :param package_name: - - :return: - """ - if not min_version: - # If no minimum version was specified, pass - return True - if StrictVersion(installed_version) < StrictVersion(min_version): - printError('Error: the version of %s installed on this system (%s) is too old. ' - 'You need at least version %s to run this tool.' % (package_name, OPINEL_VERSION, min_version)) - return False - if max_version and StrictVersion(installed_version) >= StrictVersion(max_version): - printError('Warning: ther version of %s installed on this system (%s) is too recent; ' - 'you may experience unexpected runtime errors as versions above %s have not been tested.' % - (package_name, installed_version, max_version)) - if strict: - printError('Warning treated as error.') - return False - return True - - -def manage_dictionary(dictionary, key, init, callback = None): - """ - - :param dictionary: - :param key: - :param init: - :param callback: - - :return: - """ - if not str(key) in dictionary: - dictionary[str(key)] = init - manage_dictionary(dictionary, key, init) - if callback: - callback(dictionary[key]) - return dictionary - - -def snake_to_camel(snake): - return "".join(val.title() for val in snake.split('_')) - -def snake_to_words(snake, capitalize = False): - return " ".join(val.title() if capitalize else val for val in snake.split('_')) From 12725683202bd4d1b7de61c67f7264f62a368bde Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 16:27:32 -0500 Subject: [PATCH 021/154] [Opinel] Some compatibility improvements --- ScoutSuite/core/console.py | 5 +---- opinel/utils/fs.py | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 5ec20328a..598711d51 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -5,10 +5,7 @@ import sys import traceback -try: - input = raw_input -except NameError: - pass +from six.moves import input ######################################## # Globals diff --git a/opinel/utils/fs.py b/opinel/utils/fs.py index d03a949f2..3c0104198 100644 --- a/opinel/utils/fs.py +++ b/opinel/utils/fs.py @@ -10,7 +10,6 @@ from ScoutSuite.core.conditions import _pass_condition - class CustomJSONEncoder(json.JSONEncoder): """ JSON encoder class @@ -104,7 +103,6 @@ def read_file(file_path, mode = 'rt'): :return: Contents of the file """ - contents = '' with open(file_path, mode) as f: contents = f.read() return contents From ac1095d4f02cd1ced7097a8958d9724e5bec9158 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 11:17:15 -0500 Subject: [PATCH 022/154] Removed opinel.services --- ScoutSuite/providers/aws/services/s3.py | 5 +- opinel/services/__init__.py | 0 opinel/services/cloudformation.py | 373 ------------------------ opinel/services/cloudtrail.py | 4 - opinel/services/iam.py | 294 ------------------- opinel/services/organizations.py | 49 ---- 6 files changed, 3 insertions(+), 722 deletions(-) delete mode 100644 opinel/services/__init__.py delete mode 100644 opinel/services/cloudformation.py delete mode 100644 opinel/services/cloudtrail.py delete mode 100644 opinel/services/iam.py delete mode 100644 opinel/services/organizations.py diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 9033b3373..5b5d0f0d7 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -246,9 +246,9 @@ def get_s3_bucket_secure_transport(api_client, bucket_name, bucket_info): if 'Condition' in statement and \ 'Bool' in statement['Condition'] and \ 'aws:SecureTransport' in statement['Condition']['Bool'] and \ - ((statement['Condition']['Bool']['aws:SecureTransport'] == 'false' and \ + ((statement['Condition']['Bool']['aws:SecureTransport'] == 'false' and statement['Effect'] == 'Deny') or - (statement['Condition']['Bool']['aws:SecureTransport'] == 'true' and \ + (statement['Condition']['Bool']['aws:SecureTransport'] == 'true' and statement['Effect'] == 'Allow')): bucket_info['secure_transport_enabled'] = True return True @@ -383,6 +383,7 @@ def get_s3_bucket_keys(api_client, bucket_name, bucket, check_encryption, check_ printException(e) continue if check_acls: + # noinspection PyBroadException try: key['grantees'] = get_s3_acls(api_client, bucket_name, bucket, key_name=key['name']) except Exception as e: diff --git a/opinel/services/__init__.py b/opinel/services/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/opinel/services/cloudformation.py b/opinel/services/cloudformation.py deleted file mode 100644 index ea1950ad1..000000000 --- a/opinel/services/cloudformation.py +++ /dev/null @@ -1,373 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import re -import time - -from opinel.utils.aws import connect_service, handle_truncated_response -from ScoutSuite.core.console import printDebug, printInfo, printError, printException -from opinel.utils.fs import read_file -from opinel.utils.globals import snake_to_camel - -re_iam_capability = re.compile('.*?AWS::IAM.*?', re.DOTALL | re.MULTILINE) - -def create_cloudformation_resource_from_template(api_client, resource_type, name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False, need_on_failure=False): - """ - - :param callback: - :param name: - :param template_path: - :param template_parameters: - :param quiet: - :return: - """ - create = getattr(api_client, 'create_%s' % resource_type) - api_resource_type = snake_to_camel(resource_type) - # Add a timestamps - tags.append({'Key': 'OpinelTimestamp', 'Value': str(time.time())}) - params = prepare_cloudformation_params(name, template_path, template_parameters, api_resource_type, tags) - if not quiet: - printInfo('Creating the %s %s..' % (resource_type, name)) - response = create(**params) - resource_id_attribute = '%sId' % api_resource_type - resource_id = response[resource_id_attribute] if resource_id_attribute in response else None - operation_id = response['OperationId'] if 'OperationId' in response else None - if wait_for_completion: - cloudformation_wait(api_client, resource_type, name, operation_id) - return resource_id - - -def create_stack(api_client, stack_name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False): - """ - - :param api_client: - :param stack_name: - :param template_path: - :param template_parameters: List of parameter keys and values - :param quiet: - :return: - """ - return create_cloudformation_resource_from_template(api_client, 'stack', stack_name, template_path, template_parameters, tags, quiet, wait_for_completion, need_on_failure=True) - - -def create_or_update_stack(api_client, stack_name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False): - """ - - :param api_client: - :param stack_name: - :param template_path: - :param template_parameters: List of parameter keys and values - :param quiet: - :return: - """ - try: - stack = api_client.describe_stacks(StackName = stack_name) - printInfo('Stack already exists... ', newLine = False) - stack_id = update_stack(api_client, stack_name, template_path, template_parameters, quiet, wait_for_completion) - except Exception as e: - if hasattr(e, 'response') and type(e.response) == dict and 'Error' in e.response and e.response['Error']['Code'] == 'ValidationError': - stack_id = create_stack(api_client, stack_name, template_path, template_parameters, tags, quiet, wait_for_completion) - else: - stack_id = None - printException(e) - return stack_id - - - -def create_stack_set(api_client, stack_set_name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False): - """ - - :param api_client: - :param stack_set_name: - :param template_path: - :param template_parameters: - :param quiet: - :return: - """ - return create_cloudformation_resource_from_template(api_client, 'stack_set', stack_set_name, template_path, template_parameters, tags, quiet, wait_for_completion) - - -def create_or_update_stack_set(api_client, stack_set_name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False): - """ - - :param api_client: - :param stack_name: - :param template_path: - :param template_parameters: List of parameter keys and values - :param quiet: - :return: - """ - operation_id = stack_set_id = None - try: - stack_set = api_client.describe_stack_set(StackSetName = stack_set_name) - printInfo('Stack set already exists... ', newLine = False) - operation_id = update_stack_set(api_client, stack_set_name, template_path, template_parameters, quiet, wait_for_completion) - except Exception as e: - if hasattr(e, 'response') and type(e.response) == dict and 'Error' in e.response and e.response['Error']['Code'] == 'StackSetNotFoundException': - stack_set_id = create_stack_set(api_client, stack_set_name, template_path, template_parameters, tags, quiet, wait_for_completion) - else: - printException(e) - return (stack_set_id, operation_id) - - -def create_stack_instances(api_client, stack_set_name, account_ids, regions, quiet=False): - """ - - :param api_client: - :param stack_set_name: - :param account_ids: - :param regions: - :return: - """ - operation_preferences = {'FailureTolerancePercentage': 100, - 'MaxConcurrentPercentage': 100 - } - if not quiet: - printInfo('Creating stack instances in %d regions and %d accounts...' % (len(regions), len(account_ids))) - printDebug(' %s' % ', '.join(regions)) - response = api_client.create_stack_instances(StackSetName=stack_set_name, Accounts=account_ids, Regions=regions, OperationPreferences=operation_preferences) - if not quiet: - printInfo('Successfully started operation Id %s' % response['OperationId']) - return response['OperationId'] - - -def delete_stack_set(api_client, stack_set_name, timeout = 60 * 5): - """ - """ - printDebug('Deleting stack set %s' % stack_set_name) - # Check for instances - stack_instances = handle_truncated_response(api_client.list_stack_instances, {'StackSetName': stack_set_name}, ['Summaries'])['Summaries'] - account_ids = [] - regions = [] - if len(stack_instances) > 0: - for si in stack_instances: - if si['Account'] not in account_ids: - account_ids.append(si['Account']) - if si['Region'] not in regions: - regions.append(si['Region']) - operation_id = api_client.delete_stack_instances(StackSetName = stack_set_name, Accounts = account_ids, Regions = regions, RetainStacks = False)['OperationId'] - wait_for_operation(api_client, stack_set_name, operation_id) - api_client.delete_stack_set(StackSetName = stack_set_name) - - -def get_stackset_ready_accounts(credentials, account_ids, quiet=True): - """ - Verify which AWS accounts have been configured for CloudFormation stack set by attempting to assume the stack set execution role - - :param credentials: AWS credentials to use when calling sts:assumerole - :param org_account_ids: List of AWS accounts to check for Stackset configuration - - :return: List of account IDs in which assuming the stackset execution role worked - """ - api_client = connect_service('sts', credentials, silent=True) - configured_account_ids = [] - for account_id in account_ids: - try: - role_arn = 'arn:aws:iam::%s:role/AWSCloudFormationStackSetExecutionRole' % account_id - api_client.assume_role(RoleArn=role_arn, RoleSessionName='opinel-get_stackset_ready_accounts') - configured_account_ids.append(account_id) - except Exception as e: - pass - - if len(configured_account_ids) != len(account_ids) and not quiet: - printInfo('Only %d of these accounts have the necessary stack set execution role:' % len(configured_account_ids)) - printDebug(str(configured_account_ids)) - return configured_account_ids - - -def make_awsrecipes_stack_name(template_path): - """ - - :param template_path: - :return: - """ - return make_prefixed_stack_name('AWSRecipes', template_path) - - -def make_opinel_stack_name(template_path): - """ - - :param template_path:" - :return: - """ - return make_prefixed_stack_name('Opinel', template_path) - - -def make_prefixed_stack_name(prefix, template_path): - """ - - :param prefix: - :param template_path: - """ - parts = os.path.basename(template_path).split('-') - parts = parts if len(parts) == 1 else parts[:-1] - return ('%s-%s' % (prefix, '-'.join(parts))).split('.')[0] - - -def prepare_cloudformation_params(stack_name, template_path, template_parameters, resource_type, tags=[], need_on_failure=False): - """ - - :param api_client: - :param stack_name: - :param template_path: - :param template_parameters: List of parameter keys and values - :param quiet: - :return: - """ - printDebug('Reading CloudFormation template from %s' % template_path) - template_body = read_file(template_path) - params = {} - params['%sName' % resource_type] = stack_name - params['TemplateBody'] = template_body - if len(template_parameters): - params['Parameters'] = [] - it = iter(template_parameters) - for param in it: - printError('Param:: %s' % param) - params['Parameters'].append({'ParameterKey': param,'ParameterValue': next(it)}) - - if len(tags): - params['Tags'] = tags - if re_iam_capability.match(template_body): - params['Capabilities'] = [ - 'CAPABILITY_NAMED_IAM'] - if need_on_failure: - params['OnFailure'] = 'ROLLBACK' - return params - - -def update_stack(api_client, stack_name, template_path, template_parameters = [], quiet = False, wait_for_completion = False): - """ - - :param api_client: - :param stack_name: - :param template_path: - :param template_parameters: List of parameter keys and values - :param quiet: - :return: - """ - update_cloudformation_resource_from_template(api_client, 'stack', stack_name, template_path, template_parameters, quiet = quiet, wait_for_completion = wait_for_completion) - - -def update_stack_set(api_client, stack_set_name, template_path, template_parameters=[], quiet=False, wait_for_completion = False): - """ - - :param api_client: - :param stack_set_name: - :param template_path: - :param template_parameters: - :param quiet: - :return: - """ - return update_cloudformation_resource_from_template(api_client, 'stack_set', stack_set_name, template_path, template_parameters, [], quiet, wait_for_completion) - - -def update_cloudformation_resource_from_template(api_client, resource_type, name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False): - """ - - :param callback: - :param name: - :param template_path: - :param template_parameters: - :param quiet: - :return: - """ - try: - update = getattr(api_client, 'update_%s' % resource_type) - api_resource_type = snake_to_camel(resource_type) - # Add a timestamps - tags.append({'Key': 'OpinelTimestamp', 'Value': str(time.time())}) - params = prepare_cloudformation_params(name, template_path, template_parameters, api_resource_type, tags) - if not quiet: - printInfo('Updating the %s...' % resource_type, newLine=False) - response = update(**params) - operation_id = response['OperationId'] if resource_type == 'stack_set' else None - if wait_for_completion: - cloudformation_wait(api_client, resource_type, name, operation_id) - - except Exception as e: - if api_resource_type == 'Stack' and hasattr(e, 'response') and type(e.response == dict) and e.response['Error']['Code'] == 'ValidationError' and e.response['Error']['Message'] == 'No updates are to be performed.': - printInfo(' Already up to date.') - else: - printException(e) - printError(' Failed.') - - -def wait_for_operation(api_client, stack_set_name, operation_id, timeout = 5 * 60, increment = 5): - printDebug('Waiting for operation %s on stack set %s...' % (operation_id, stack_set_name)) - timer = 0 - status = '' - while True: - if timer >= timeout: - printError('Timed out.') - break - info = api_client.describe_stack_set_operation(StackSetName = stack_set_name, OperationId = operation_id) - status = info['StackSetOperation']['Status'] - if status not in ['RUNNING', 'STOPPING']: - break - printError('Operation status is \'%s\'... waiting %d seconds until next check...' % (status, increment)) - time.sleep(increment) - timer += increment - return 'Operation %s is %s' % (operation_id, status) - - -def wait_for_stack_set(api_client, stack_set_name, timeout = 60, increment = 5): - printDebug('Waiting for stack set %s to be ready...' % stack_set_name) - timer = 0 - while True: - if timer >= timeout: - printError('Timed out.') - break - printError('Checking the stack set\'s status...') - time.sleep(increment) - timer += increment - info = api_client.describe_stack_set(StackSetName = stack_set_name) - if info['StackSet']['Status'] == 'ACTIVE': - break - - -def still_running(callback, params, resource_type): - rc = True - response = callback(**params) - if resource_type == 'stack': - status = response['Stacks'][0]['StackStatus'] - if status.endswith('_COMPLETE') or status.endswith('_FAILED'): - rc = False - elif resource_type == 'stack_set': - status = response['StackSet']['Status'] - if status == 'ACTIVE': - rc = False - elif resource_type == 'operation': - status = response['StackSetOperation']['Status'] - if status != 'RUNNING': - rc = False - return (rc, status) - - -def cloudformation_wait(api_client, resource_type, resource_name, operation_id = None, timeout = 5 * 60, increment = 5): - if resource_type == 'stack': - callback = api_client.describe_stacks - params = {'StackName': resource_name} - elif resource_type == 'stack_set': - params = {'StackSetName': resource_name} - if operation_id: - callback = api_client.describe_stack_set_operation - params['OperationId'] = operation_id - resource_type = 'operation' - else: - callback = api_client.describe_stack_set - else: - printError('Unknown resource type: %s' % resource_type) - return - timer = 0 - while True: - if timer >= timeout: - printError('Timed out.') - break - rc, status = still_running(callback, params, resource_type) - if rc == False: - printInfo('Status: %s' % status) - break - printInfo('Status: %s... waiting %d seconds until next check...' % (status, increment)) - timer += increment - time.sleep(increment) diff --git a/opinel/services/cloudtrail.py b/opinel/services/cloudtrail.py deleted file mode 100644 index 646802d96..000000000 --- a/opinel/services/cloudtrail.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- - -def get_trails(api_client): - return api_client.describe_trails()['trailList'] diff --git a/opinel/services/iam.py b/opinel/services/iam.py deleted file mode 100644 index 33765c995..000000000 --- a/opinel/services/iam.py +++ /dev/null @@ -1,294 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from opinel.utils.aws import handle_truncated_response -from opinel.utils.credentials import generate_password -from ScoutSuite.core.console import printInfo, printError, printException - - - -def add_user_to_group(iam_client, user, group, quiet = False): - """ - Add an IAM user to an IAM group - - :param iam_client: - :param group: - :param user: - :param user_info: - :param dry_run: - :return: - """ - if not quiet: - printInfo('Adding user to group %s...' % group) - iam_client.add_user_to_group(GroupName = group, UserName = user) - - -def create_groups(iam_client, groups): - """ - Create a number of IAM group, silently handling exceptions when entity already exists - . - :param iam_client: AWS API client for IAM - :param groups: Name of IAM groups to be created. - - :return: None - """ - groups_data = [] - if type(groups) != list: - groups = [ groups ] - for group in groups: - errors = [] - try: - printInfo('Creating group %s...' % group) - iam_client.create_group(GroupName = group) - except Exception as e: - if e.response['Error']['Code'] != 'EntityAlreadyExists': - printException(e) - errors.append('iam:creategroup') - groups_data.append({'groupname': group, 'errors': errors}) - return groups_data - - -def create_user(iam_client, user, groups = [], with_password= False, with_mfa = False, with_access_key = False, require_password_reset = True): - """ - - :param iam_client: AWS API client for IAM - :param user: Name of the user to create - :param groups: Name of the IAM groups to add the user to - :param with_password: Boolean indicating whether creation of a password should be done - :param with_mfa: Boolean indicating whether creation of an MFA device should be done - :param with_access_key: Boolean indicating whether creation of an API access key should be done - :param require_password_reset: Boolean indicating whether users should reset their password after first login - :return: - """ - user_data = {'username': user, 'errors': []} - printInfo('Creating user %s...' % user) - try: - iam_client.create_user(UserName = user) - except Exception as e: - user_data['errors'].append('iam:createuser') - return user_data - # Add user to groups - if type(groups) != list: - groups = [ groups ] - for group in groups: - try: - add_user_to_group(iam_client, user, group) - except Exception as e: - printException(e) - user_data['errors'].append('iam:addusertogroup - %s' % group) - # Generate password - if with_password: - try: - printInfo('Creating a login profile...') - user_data['password'] = generate_password() - iam_client.create_login_profile(UserName = user, Password = user_data['password'] , PasswordResetRequired = require_password_reset) - except Exception as e: - printException(e) - user_data['errors'].append('iam:createloginprofile') - # Enable MFA - if False and with_mfa: - printInfo('Enabling MFA...') - serial = '' - mfa_code1 = '' - mfa_code2 = '' - # Create an MFA device, Display the QR Code, and activate the MFA device - try: - mfa_serial = False # enable_mfa(iam_client, user, '%s/qrcode.png' % user) - except Exception as e: - return 42 - # Request access key - if with_access_key: - try: - printInfo('Creating an API access key...') - access_key = iam_client.create_access_key(UserName=user)['AccessKey'] - user_data['AccessKeyId'] = access_key['AccessKeyId'] - user_data['SecretAccessKey'] = access_key['SecretAccessKey'] - except Exception as e: - printException(e) - user_data['errors'].append('iam:createaccesskey') - return user_data - - -def delete_user(iam_client, user, mfa_serial = None, keep_user = False, terminated_groups = []): - """ - Delete IAM user - - :param iam_client: - :param user: - :param mfa_serial: - :param keep_user: - :param terminated_groups: - :return: - """ - errors = [] - printInfo('Deleting user %s...' % user) - # Delete access keys - try: - aws_keys = get_access_keys(iam_client, user) - for aws_key in aws_keys: - try: - printInfo('Deleting access key ID %s... ' % aws_key['AccessKeyId'], False) - iam_client.delete_access_key(AccessKeyId = aws_key['AccessKeyId'], UserName = user) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - except Exception as e: - printException(e) - printError('Failed to get access keys for user %s.' % user) - # Deactivate and delete MFA devices - try: - mfa_devices = iam_client.list_mfa_devices(UserName = user)['MFADevices'] - for mfa_device in mfa_devices: - serial = mfa_device['SerialNumber'] - try: - printInfo('Deactivating MFA device %s... ' % serial, False) - iam_client.deactivate_mfa_device(SerialNumber = serial, UserName = user) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - delete_virtual_mfa_device(iam_client, serial) - if mfa_serial: - delete_virtual_mfa_device(iam_client, mfa_serial) - except Exception as e: - printException(e) - printError('Faile to fetch/delete MFA device serial number for user %s.' % user) - errors.append(e.response['Error']['Code']) - # Remove IAM user from groups - try: - groups = iam_client.list_groups_for_user(UserName = user)['Groups'] - for group in groups: - try: - printInfo('Removing from group %s... ' % group['GroupName'], False) - iam_client.remove_user_from_group(GroupName = group['GroupName'], UserName = user) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - except Exception as e: - printException(e) - printError('Failed to fetch IAM groups for user %s.' % user) - errors.append(e.response['Error']['Code']) - # Delete login profile - login_profile = [] - try: - login_profile = iam_client.get_login_profile(UserName = user)['LoginProfile'] - except Exception as e: - pass - try: - if len(login_profile): - printInfo('Deleting login profile... ', False) - iam_client.delete_login_profile(UserName = user) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - # Delete inline policies - try: - printInfo('Deleting inline policies... ', False) - policies = iam_client.list_user_policies(UserName = user) - for policy in policies['PolicyNames']: - iam_client.delete_user_policy(UserName = user, PolicyName = policy) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - # Detach managed policies - try: - printInfo('Detaching managed policies... ', False) - policies = iam_client.list_attached_user_policies(UserName = user) - for policy in policies['AttachedPolicies']: - iam_client.detach_user_policy(UserName = user, PolicyArn = policy['PolicyArn']) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - # Delete IAM user - try: - if not keep_user: - iam_client.delete_user(UserName = user) - printInfo('User %s deleted.' % user) - else: - for group in terminated_groups: - add_user_to_group(iam_client, group, user) - except Exception as e: - printException(e) - printError('Failed to delete user.') - errors.append(e.response['Error']['Code']) - pass - return errors - - -def delete_virtual_mfa_device(iam_client, mfa_serial): - """ - Delete a vritual MFA device given its serial number - - :param iam_client: - :param mfa_serial: - :return: - """ - try: - printInfo('Deleting MFA device %s...' % mfa_serial) - iam_client.delete_virtual_mfa_device(SerialNumber = mfa_serial) - except Exception as e: - printException(e) - printError('Failed to delete MFA device %s' % mfa_serial) - pass - -def get_access_keys(iam_client, user_name): - """ - - :param iam_client: - :param user_name: - :return: - """ - keys = handle_truncated_response(iam_client.list_access_keys, {'UserName': user_name}, ['AccessKeyMetadata'])['AccessKeyMetadata'] - return keys - - -def init_group_category_regex(category_groups, category_regex_args): - """ - Initialize and compile regular expression for category groups - - :param category_regex_args: List of string regex - - :return: List of compiled regex - """ - category_regex = [] - authorized_empty_regex = 1 - if len(category_regex_args) and len(category_groups) != len(category_regex_args): - printError('Error: you must provide as many regex as category groups.') - return None - for regex in category_regex_args: - if len(regex) < 1: - if authorized_empty_regex > 0: - category_regex.append(None) - authorized_empty_regex -= 1 - else: - printError('Error: you cannot have more than one empty regex to automatically assign groups to users.') - return None - else: - category_regex.append(re.compile(regex)) - return category_regex - - - -def show_access_keys(iam_client, user_name): - """ - - :param iam_client: - :param user_name: - :return: - """ - keys = get_access_keys(iam_client, user_name) - printInfo('User \'%s\' currently has %s access keys:' % (user_name, len(keys))) - for key in keys: - printInfo('\t%s (%s)' % (key['AccessKeyId'], key['Status'])) diff --git a/opinel/services/organizations.py b/opinel/services/organizations.py deleted file mode 100644 index 4578110f8..000000000 --- a/opinel/services/organizations.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -from opinel.utils.aws import handle_truncated_response -from ScoutSuite.core.console import printDebug, printInfo - - -def get_organization_account_ids(api_client, exceptions = [], quiet = True): - - # List all accounts in the organization - org_accounts = get_organization_accounts(api_client, exceptions, quiet) - return [ account['Id'] for account in org_accounts ] - - -def get_organization_accounts(api_client, exceptions = [], quiet = True): - - # List all accounts in the organization - org_accounts = handle_truncated_response(api_client.list_accounts, {}, ['Accounts'])['Accounts'] - if not quiet: - printInfo('Found %d accounts in the organization.' % len(org_accounts)) - for account in org_accounts: - printDebug(str(account)) - if len(exceptions): - filtered_accounts = [] - for account in org_accounts: - if account['Id'] not in exceptions: - filtered_accounts.append(account) - org_accounts = filtered_accounts - return org_accounts - - -def get_organizational_units(api_client): - ous = [] - roots = api_client.list_roots()['Roots'] - return get_children_organizational_units(api_client, roots) - - -def get_children_organizational_units(api_client, parents): - ous = [] - for parent in parents: - children = handle_truncated_response(api_client.list_organizational_units_for_parent, {'ParentId': parent['Id']}, ['OrganizationalUnits'])['OrganizationalUnits'] - if len(children): - ous += get_children_organizational_units(api_client, children) - else: - ous.append(parent) - return ous - -def list_accounts_for_parent(api_client, parent): - return handle_truncated_response(api_client.list_accounts_for_parent, {'ParentId': parent['Id']}, ['Accounts'])['Accounts'] - From 335b518fbefd79fa251a87269c8924c48e90f72f Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 13:13:06 -0500 Subject: [PATCH 023/154] Moved aws.py and fixed warnings in it --- .../utils => ScoutSuite/providers/aws}/aws.py | 34 ++++++++++--------- ScoutSuite/providers/aws/configs/base.py | 2 +- ScoutSuite/providers/aws/configs/regions.py | 2 +- ScoutSuite/providers/aws/provider.py | 2 +- ScoutSuite/providers/aws/services/ec2.py | 2 +- ScoutSuite/providers/aws/services/efs.py | 2 +- ScoutSuite/providers/aws/services/elbv2.py | 2 +- ScoutSuite/providers/aws/services/iam.py | 2 +- ScoutSuite/providers/aws/services/rds.py | 2 +- ScoutSuite/providers/aws/services/redshift.py | 2 +- ScoutSuite/providers/aws/services/route53.py | 2 +- ScoutSuite/providers/aws/services/s3.py | 2 +- ScoutSuite/providers/aws/services/vpc.py | 2 +- ScoutSuite/providers/base/configs/base.py | 4 +-- ScoutSuite/providers/base/configs/services.py | 2 +- opinel/utils/credentials.py | 2 +- opinel/utils/profiles.py | 2 +- requirements.txt | 1 + 18 files changed, 36 insertions(+), 33 deletions(-) rename {opinel/utils => ScoutSuite/providers/aws}/aws.py (76%) diff --git a/opinel/utils/aws.py b/ScoutSuite/providers/aws/aws.py similarity index 76% rename from opinel/utils/aws.py rename to ScoutSuite/providers/aws/aws.py index 9eac65958..52bbc702a 100644 --- a/opinel/utils/aws.py +++ b/ScoutSuite/providers/aws/aws.py @@ -8,8 +8,7 @@ from ScoutSuite.core.console import printInfo, printException - -def build_region_list(service, chosen_regions = [], partition_name = 'aws'): +def build_region_list(service, chosen_regions=None, partition_name='aws'): """ Build the list of target region names @@ -19,16 +18,18 @@ def build_region_list(service, chosen_regions = [], partition_name = 'aws'): :return: """ - service = 'ec2containerservice' if service == 'ecs' else service # Of course things aren't that easy... + if chosen_regions is None: + chosen_regions = [] + service = 'ec2containerservice' if service == 'ecs' else service # Get list of regions from botocore - regions = Session().get_available_regions(service, partition_name = partition_name) + regions = Session().get_available_regions(service, partition_name=partition_name) if len(chosen_regions): return list((Counter(regions) & Counter(chosen_regions)).elements()) else: return regions -def connect_service(service, credentials, region_name = None, config = None, silent = False): +def connect_service(service, credentials, region_name=None, config=None, silent=False): """ Instantiates an AWS API client @@ -42,12 +43,10 @@ def connect_service(service, credentials, region_name = None, config = None, sil """ api_client = None try: - client_params = {} - client_params['service_name'] = service.lower() - session_params = {} - session_params['aws_access_key_id'] = credentials['AccessKeyId'] - session_params['aws_secret_access_key'] = credentials['SecretAccessKey'] - session_params['aws_session_token'] = credentials['SessionToken'] + client_params = {'service_name': service.lower()} + session_params = {'aws_access_key_id': credentials['AccessKeyId'], + 'aws_secret_access_key': credentials['SecretAccessKey'], + 'aws_session_token': credentials['SessionToken']} if region_name: client_params['region_name'] = region_name session_params['region_name'] = region_name @@ -55,10 +54,10 @@ def connect_service(service, credentials, region_name = None, config = None, sil client_params['config'] = config aws_session = boto3.session.Session(**session_params) if not silent: - infoMessage = 'Connecting to AWS %s' % service + info_message = 'Connecting to AWS %s' % service if region_name: - infoMessage = infoMessage + ' in %s' % region_name - printInfo('%s...' % infoMessage) + info_message = info_message + ' in %s' % region_name + printInfo('%s...' % info_message) api_client = aws_session.client(**client_params) except Exception as e: printException(e) @@ -86,7 +85,7 @@ def get_name(src, dst, default_attribute): def get_caller_identity(credentials): - api_client = connect_service('sts', credentials, silent = True) + api_client = connect_service('sts', credentials, silent=True) return api_client.get_caller_identity() @@ -132,6 +131,7 @@ def handle_truncated_response(callback, params, entities): if not marker_found: break except Exception as e: + # noinspection PyTypeChecker if is_throttled(e): time.sleep(1) else: @@ -146,4 +146,6 @@ def is_throttled(e): :param e: Exception raised :return: True if it's a throttling exception else False """ - return True if (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in [ 'Throttling', 'RequestLimitExceeded', 'ThrottlingException', 'TooManyRequestsException' ]) else False + return True if (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in + ['Throttling', 'RequestLimitExceeded', 'ThrottlingException', 'TooManyRequestsException']) \ + else False diff --git a/ScoutSuite/providers/aws/configs/base.py b/ScoutSuite/providers/aws/configs/base.py index 0884cacdf..fcd6f4e4a 100644 --- a/ScoutSuite/providers/aws/configs/base.py +++ b/ScoutSuite/providers/aws/configs/base.py @@ -8,7 +8,7 @@ from ScoutSuite.providers.base.configs.base import BaseConfig -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response class AWSBaseConfig(BaseConfig): diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 80d577080..8e918de9d 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -12,7 +12,7 @@ except ImportError: from queue import Queue -from opinel.utils.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response +from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response from ScoutSuite.core.console import printException, printInfo from ScoutSuite.providers.base.configs import resource_id_map diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 200f6784b..55c5d239a 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -3,7 +3,7 @@ import copy import os -from opinel.utils.aws import get_aws_account_id +from ScoutSuite.providers.aws.aws import get_aws_account_id from opinel.utils.credentials import read_creds from ScoutSuite.core.console import printDebug, printError, printException, printInfo diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 8ac8371f0..ce52dfe23 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -7,7 +7,7 @@ import netaddr import base64 -from opinel.utils.aws import get_name +from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.core.console import printException, printInfo from opinel.utils.fs import load_data diff --git a/ScoutSuite/providers/aws/services/efs.py b/ScoutSuite/providers/aws/services/efs.py index 31ee09dd2..58198baf1 100644 --- a/ScoutSuite/providers/aws/services/efs.py +++ b/ScoutSuite/providers/aws/services/efs.py @@ -2,7 +2,7 @@ """ EFS-related classes and functions """ -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index df5dc10ab..b67c1c5f5 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -3,7 +3,7 @@ ELBv2-related classes and functions """ -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index 3ef28fd6c..dd67370a9 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from botocore.exceptions import ClientError -from opinel.utils.aws import connect_service, handle_truncated_response +from ScoutSuite.providers.aws.aws import connect_service, handle_truncated_response from ScoutSuite.core.console import printError, printException from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index bb988a69e..eeff7459c 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index d8a87610c..772d40edc 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -3,7 +3,7 @@ Redshift-related classes and functions """ -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig diff --git a/ScoutSuite/providers/aws/services/route53.py b/ScoutSuite/providers/aws/services/route53.py index c7d658f44..08d502f1c 100644 --- a/ScoutSuite/providers/aws/services/route53.py +++ b/ScoutSuite/providers/aws/services/route53.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index f8c7330bb..da9b1d0e2 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -6,7 +6,7 @@ import json from botocore.exceptions import ClientError -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException, printInfo from ScoutSuite.utils import manage_dictionary diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index 2c41976aa..e6cf953e9 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -3,7 +3,7 @@ import netaddr import copy -from opinel.utils.aws import get_name +from ScoutSuite.providers.aws.aws import get_name from opinel.utils.fs import load_data, read_ip_ranges from ScoutSuite.providers.base.configs.browser import get_value_at diff --git a/ScoutSuite/providers/base/configs/base.py b/ScoutSuite/providers/base/configs/base.py index 95681f2f2..99dd70a3b 100644 --- a/ScoutSuite/providers/base/configs/base.py +++ b/ScoutSuite/providers/base/configs/base.py @@ -16,11 +16,11 @@ from ScoutSuite.providers.base.configs.threads import thread_configs # TODO do this better without name conflict -from opinel.utils.aws import connect_service +from ScoutSuite.providers.aws.aws import connect_service from ScoutSuite.providers.gcp.utils import gcp_connect_service from ScoutSuite.providers.azure.utils import azure_connect_service -from opinel.utils.aws import build_region_list +from ScoutSuite.providers.aws.aws import build_region_list from ScoutSuite.core.console import printException, printInfo from ScoutSuite.output.console import FetchStatusLogger diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index d6ad4dc4e..5d19b7317 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.core.console import printError, printException, printDebug -from opinel.utils.aws import get_partition_name +from ScoutSuite.providers.aws.aws import get_partition_name class BaseServicesConfig(object): diff --git a/opinel/utils/credentials.py b/opinel/utils/credentials.py index ea4ae2b81..018caf7e5 100644 --- a/opinel/utils/credentials.py +++ b/opinel/utils/credentials.py @@ -14,7 +14,7 @@ from ScoutSuite.core.console import printException, printError, printInfo from ScoutSuite.core.console import prompt_4_mfa_code from opinel.utils.fs import save_blob_as_json -from opinel.utils.aws import connect_service +from ScoutSuite.providers.aws.aws import connect_service ######################################## diff --git a/opinel/utils/profiles.py b/opinel/utils/profiles.py index 3450f1f59..f4714ce67 100644 --- a/opinel/utils/profiles.py +++ b/opinel/utils/profiles.py @@ -3,7 +3,7 @@ import os import re -from opinel.utils.aws import get_aws_account_id +from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.core.console import printDebug from opinel.utils.credentials import read_creds diff --git a/requirements.txt b/requirements.txt index e4ff9ba7f..4fd46b27c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ netaddr>=0.7.11 # AWS Provider boto3>=1.9.60 +botocore>=1.12.60 # Opinel requests>=2.4.0,<3.0.0 From 225f0c53f924fe7c1ef782145e2e608ce8c7da0e Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 14:11:02 -0500 Subject: [PATCH 024/154] Solved PEP8 issues in regions.py --- ScoutSuite/providers/aws/configs/regions.py | 40 +++++++++++---------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 8e918de9d..c5650cb3e 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -12,7 +12,8 @@ except ImportError: from queue import Queue -from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response +from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, \ + handle_truncated_response from ScoutSuite.core.console import printException, printInfo from ScoutSuite.providers.base.configs import resource_id_map @@ -72,7 +73,7 @@ def __init__(self, service_metadata=None, thread_config=4): continue params = resource_metadata['params'] if 'params' in resource_metadata else {} ignore_exceptions = True if 'no_exceptions' in resource_metadata and \ - resource_metadata['no_exceptions'] == True else False + resource_metadata['no_exceptions'] else False if not only_first_region: self.targets['other_regions'] += ((resource, resource_metadata['response'], @@ -91,14 +92,13 @@ def init_region_config(self, region): :param region: Name of the region """ - self.regions[region] = self.region_config_class(region_name = region, resource_types = self.resource_types) + self.regions[region] = self.region_config_class(region_name=region, resource_types=self.resource_types) - def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets = None): + def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): """ Fetch all the configuration supported by Scout2 for a given service :param credentials: F - :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. @@ -125,7 +125,8 @@ def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower() # Init regions - regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class + # TODO: move this code within this class + regions = build_region_list(api_service, regions, partition_name) self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) # Threading to fetch & parse resources (queue consumer) @@ -156,7 +157,8 @@ def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets for j in range(self.thread_config['list']): qr.put(None) - def _init_threading(self, function, params=None, num_threads=10): + @staticmethod + def _init_threading(function, params=None, num_threads=10): """ Initialize queue and threads @@ -166,7 +168,8 @@ def _init_threading(self, function, params=None, num_threads=10): :return: """ params = {} if params is None else params - q = Queue(maxsize=0) # TODO: find something appropriate + # TODO: find something appropriate + q = Queue(maxsize=0) for i in range(num_threads): worker = Thread(target=function, args=(q, params)) worker.setDaemon(True) @@ -181,10 +184,10 @@ def _fetch_region(self, q, params): region, targets = q.get() or (None, None) if region: self.init_region_config(region) - api_client = connect_service(params['api_service'], params['credentials'], region, silent = True) + api_client = connect_service(params['api_service'], params['credentials'], region, silent=True) api_clients[region] = api_client # TODO : something here for single_region stuff - self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) # params['targets']) + self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) self.fetchstatuslogger.counts['regions']['fetched'] += 1 except Exception as e: printException(e) @@ -247,12 +250,14 @@ def tweak_params(self, params, credentials): ######################################## +# noinspection PyBroadException class RegionConfig(BaseConfig): """ Base class for ... """ - def __init__(self, region_name, resource_types=None): + def __init__(self, region_name, resource_types=None, **kwargs): + super().__init__(**kwargs) resource_types = {} if resource_types is None else resource_types self.region = region_name self.name = region_name @@ -265,7 +270,7 @@ def __init__(self, region_name, resource_types=None): self.vpc_resource_types = resource_types['vpc'] def fetch_all(self, api_client, fetchstatuslogger, q, targets): - ''' + """ Make all API calls as defined in metadata.json :param api_client: @@ -273,9 +278,9 @@ def fetch_all(self, api_client, fetchstatuslogger, q, targets): :param q: :param targets: :return: - ''' + """ self.fetchstatuslogger = fetchstatuslogger - if targets != None: + if targets is not None: # Ensure targets is a tuple if type(targets) != list and type(targets) != tuple: targets = tuple(targets,) @@ -285,7 +290,7 @@ def fetch_all(self, api_client, fetchstatuslogger, q, targets): self._fetch_targets(api_client, q, target) def _fetch_targets(self, api_client, q, target): - ''' + """ Make an API call defined in metadata.json. Parse the returned object as implemented in the "parse_[object name]" method. @@ -293,7 +298,7 @@ def _fetch_targets(self, api_client, q, target): :param q: :param target: :return: - ''' + """ # Handle & format the target type target_type, response_attribute, list_method_name, list_params, ignore_list_error = target list_method = getattr(api_client, list_method_name) @@ -318,7 +323,7 @@ def _fetch_targets(self, api_client, q, target): # Add to the queue q.put((callback, region, target)) - def store_target(self, global_params, region, target): + def store_target(self, target): target_type = target.pop('scout2_target_type') if 'VpcId' in target: vpc_id = target.pop('VpcId') @@ -330,4 +335,3 @@ def store_target(self, global_params, region, target): target_id = target[resource_id_map[target_type]] get_name(target, target, resource_id_map[target_type]) target_dict[target_id] = target - From 5b7dfee8fc7a56c2757bfd17dc4f1aa6f438bc34 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 14:11:02 -0500 Subject: [PATCH 025/154] Revert "Solved PEP8 issues in regions.py" This reverts commit 225f0c53f924fe7c1ef782145e2e608ce8c7da0e. --- ScoutSuite/providers/aws/configs/regions.py | 40 ++++++++++----------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index c5650cb3e..8e918de9d 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -12,8 +12,7 @@ except ImportError: from queue import Queue -from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, \ - handle_truncated_response +from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response from ScoutSuite.core.console import printException, printInfo from ScoutSuite.providers.base.configs import resource_id_map @@ -73,7 +72,7 @@ def __init__(self, service_metadata=None, thread_config=4): continue params = resource_metadata['params'] if 'params' in resource_metadata else {} ignore_exceptions = True if 'no_exceptions' in resource_metadata and \ - resource_metadata['no_exceptions'] else False + resource_metadata['no_exceptions'] == True else False if not only_first_region: self.targets['other_regions'] += ((resource, resource_metadata['response'], @@ -92,13 +91,14 @@ def init_region_config(self, region): :param region: Name of the region """ - self.regions[region] = self.region_config_class(region_name=region, resource_types=self.resource_types) + self.regions[region] = self.region_config_class(region_name = region, resource_types = self.resource_types) - def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets = None): """ Fetch all the configuration supported by Scout2 for a given service :param credentials: F + :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. @@ -125,8 +125,7 @@ def fetch_all(self, credentials, regions=None, partition_name='aws', targets=Non api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower() # Init regions - # TODO: move this code within this class - regions = build_region_list(api_service, regions, partition_name) + regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) # Threading to fetch & parse resources (queue consumer) @@ -157,8 +156,7 @@ def fetch_all(self, credentials, regions=None, partition_name='aws', targets=Non for j in range(self.thread_config['list']): qr.put(None) - @staticmethod - def _init_threading(function, params=None, num_threads=10): + def _init_threading(self, function, params=None, num_threads=10): """ Initialize queue and threads @@ -168,8 +166,7 @@ def _init_threading(function, params=None, num_threads=10): :return: """ params = {} if params is None else params - # TODO: find something appropriate - q = Queue(maxsize=0) + q = Queue(maxsize=0) # TODO: find something appropriate for i in range(num_threads): worker = Thread(target=function, args=(q, params)) worker.setDaemon(True) @@ -184,10 +181,10 @@ def _fetch_region(self, q, params): region, targets = q.get() or (None, None) if region: self.init_region_config(region) - api_client = connect_service(params['api_service'], params['credentials'], region, silent=True) + api_client = connect_service(params['api_service'], params['credentials'], region, silent = True) api_clients[region] = api_client # TODO : something here for single_region stuff - self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) + self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) # params['targets']) self.fetchstatuslogger.counts['regions']['fetched'] += 1 except Exception as e: printException(e) @@ -250,14 +247,12 @@ def tweak_params(self, params, credentials): ######################################## -# noinspection PyBroadException class RegionConfig(BaseConfig): """ Base class for ... """ - def __init__(self, region_name, resource_types=None, **kwargs): - super().__init__(**kwargs) + def __init__(self, region_name, resource_types=None): resource_types = {} if resource_types is None else resource_types self.region = region_name self.name = region_name @@ -270,7 +265,7 @@ def __init__(self, region_name, resource_types=None, **kwargs): self.vpc_resource_types = resource_types['vpc'] def fetch_all(self, api_client, fetchstatuslogger, q, targets): - """ + ''' Make all API calls as defined in metadata.json :param api_client: @@ -278,9 +273,9 @@ def fetch_all(self, api_client, fetchstatuslogger, q, targets): :param q: :param targets: :return: - """ + ''' self.fetchstatuslogger = fetchstatuslogger - if targets is not None: + if targets != None: # Ensure targets is a tuple if type(targets) != list and type(targets) != tuple: targets = tuple(targets,) @@ -290,7 +285,7 @@ def fetch_all(self, api_client, fetchstatuslogger, q, targets): self._fetch_targets(api_client, q, target) def _fetch_targets(self, api_client, q, target): - """ + ''' Make an API call defined in metadata.json. Parse the returned object as implemented in the "parse_[object name]" method. @@ -298,7 +293,7 @@ def _fetch_targets(self, api_client, q, target): :param q: :param target: :return: - """ + ''' # Handle & format the target type target_type, response_attribute, list_method_name, list_params, ignore_list_error = target list_method = getattr(api_client, list_method_name) @@ -323,7 +318,7 @@ def _fetch_targets(self, api_client, q, target): # Add to the queue q.put((callback, region, target)) - def store_target(self, target): + def store_target(self, global_params, region, target): target_type = target.pop('scout2_target_type') if 'VpcId' in target: vpc_id = target.pop('VpcId') @@ -335,3 +330,4 @@ def store_target(self, target): target_id = target[resource_id_map[target_type]] get_name(target, target, resource_id_map[target_type]) target_dict[target_id] = target + From 6cd5877c4e848dd599084f7cd0b5f5e597212416 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 14:27:39 -0500 Subject: [PATCH 026/154] PEP8 fixes in regions.py --- ScoutSuite/providers/aws/configs/regions.py | 36 ++++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 8e918de9d..fc9293c0b 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -6,13 +6,15 @@ import copy import re from threading import Thread + # Python2 vs Python3 try: from Queue import Queue except ImportError: from queue import Queue -from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response +from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, \ + handle_truncated_response from ScoutSuite.core.console import printException, printInfo from ScoutSuite.providers.base.configs import resource_id_map @@ -31,6 +33,7 @@ first_cap_re = re.compile('(.)([A-Z][a-z]+)') all_caps_re = re.compile('([a-z0-9])([A-Z])') + ######################################## # RegionalServiceConfig ######################################## @@ -50,7 +53,7 @@ def __init__(self, service_metadata=None, thread_config=4): self.regions = {} self.thread_config = thread_configs[thread_config] self.service = re.sub(r'Config$', "", type(self).__name__).lower() - + # Booleans that define if threads should keep running self.run_q_threads = True self.run_qr_threads = True @@ -91,14 +94,13 @@ def init_region_config(self, region): :param region: Name of the region """ - self.regions[region] = self.region_config_class(region_name = region, resource_types = self.resource_types) + self.regions[region] = self.region_config_class(region_name=region, resource_types=self.resource_types) - def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets = None): + def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): """ Fetch all the configuration supported by Scout2 for a given service :param credentials: F - :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. @@ -125,7 +127,7 @@ def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower() # Init regions - regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class + regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) # Threading to fetch & parse resources (queue consumer) @@ -166,7 +168,7 @@ def _init_threading(self, function, params=None, num_threads=10): :return: """ params = {} if params is None else params - q = Queue(maxsize=0) # TODO: find something appropriate + q = Queue(maxsize=0) # TODO: find something appropriate for i in range(num_threads): worker = Thread(target=function, args=(q, params)) worker.setDaemon(True) @@ -181,10 +183,11 @@ def _fetch_region(self, q, params): region, targets = q.get() or (None, None) if region: self.init_region_config(region) - api_client = connect_service(params['api_service'], params['credentials'], region, silent = True) + api_client = connect_service(params['api_service'], params['credentials'], region, silent=True) api_clients[region] = api_client # TODO : something here for single_region stuff - self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) # params['targets']) + self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], + targets) # params['targets']) self.fetchstatuslogger.counts['regions']['fetched'] += 1 except Exception as e: printException(e) @@ -242,11 +245,13 @@ def tweak_params(self, params, credentials): params = get_aws_account_id(credentials) return params + ######################################## # RegionConfig ######################################## +# noinspection PyBroadException class RegionConfig(BaseConfig): """ Base class for ... @@ -265,7 +270,7 @@ def __init__(self, region_name, resource_types=None): self.vpc_resource_types = resource_types['vpc'] def fetch_all(self, api_client, fetchstatuslogger, q, targets): - ''' + """ Make all API calls as defined in metadata.json :param api_client: @@ -273,19 +278,19 @@ def fetch_all(self, api_client, fetchstatuslogger, q, targets): :param q: :param targets: :return: - ''' + """ self.fetchstatuslogger = fetchstatuslogger - if targets != None: + if targets is not None: # Ensure targets is a tuple if type(targets) != list and type(targets) != tuple: - targets = tuple(targets,) + targets = tuple(targets, ) elif type(targets) != tuple: targets = tuple(targets) for target in targets: self._fetch_targets(api_client, q, target) def _fetch_targets(self, api_client, q, target): - ''' + """ Make an API call defined in metadata.json. Parse the returned object as implemented in the "parse_[object name]" method. @@ -293,7 +298,7 @@ def _fetch_targets(self, api_client, q, target): :param q: :param target: :return: - ''' + """ # Handle & format the target type target_type, response_attribute, list_method_name, list_params, ignore_list_error = target list_method = getattr(api_client, list_method_name) @@ -330,4 +335,3 @@ def store_target(self, global_params, region, target): target_id = target[resource_id_map[target_type]] get_name(target, target, resource_id_map[target_type]) target_dict[target_id] = target - From caf8888d9a318c38cda155657220caad69d69fcc Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 14:33:24 -0500 Subject: [PATCH 027/154] Reformatted a few AWS files --- ScoutSuite/providers/aws/configs/vpc.py | 3 +- ScoutSuite/providers/aws/provider.py | 26 +++++++++++------ .../providers/aws/services/awslambda.py | 4 +-- .../providers/aws/services/cloudformation.py | 6 ++-- .../providers/aws/services/cloudtrail.py | 17 +++++------ .../providers/aws/services/cloudwatch.py | 4 +-- .../providers/aws/services/directconnect.py | 4 +-- ScoutSuite/providers/aws/services/ec2.py | 14 +++++---- ScoutSuite/providers/aws/services/efs.py | 13 +++++---- .../providers/aws/services/elasticache.py | 8 ++--- ScoutSuite/providers/aws/services/elb.py | 10 ++++--- ScoutSuite/providers/aws/services/elbv2.py | 8 +++-- ScoutSuite/providers/aws/services/iam.py | 2 +- ScoutSuite/providers/aws/services/rds.py | 29 ++++++++++--------- ScoutSuite/providers/aws/services/redshift.py | 10 +++---- ScoutSuite/providers/aws/services/route53.py | 23 ++++++--------- ScoutSuite/providers/aws/services/s3.py | 1 + ScoutSuite/providers/aws/services/ses.py | 12 ++++---- ScoutSuite/providers/aws/services/sns.py | 4 +-- ScoutSuite/providers/aws/services/sqs.py | 8 ++--- ScoutSuite/providers/aws/services/vpc.py | 5 ++++ 21 files changed, 108 insertions(+), 103 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/vpc.py b/ScoutSuite/providers/aws/configs/vpc.py index 0d9edaaef..ccc0aa288 100644 --- a/ScoutSuite/providers/aws/configs/vpc.py +++ b/ScoutSuite/providers/aws/configs/vpc.py @@ -3,12 +3,13 @@ Base class for vpc-specific services """ + class VPCConfig(object): """ Configuration for a single VPC in a single service """ - def __init__(self, vpc_resource_types, name = None): + def __init__(self, vpc_resource_types, name=None): self.name = name for resource_type in vpc_resource_types: setattr(self, resource_type, {}) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 55c5d239a..c15cf2647 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -20,7 +20,8 @@ class AWSProvider(BaseProvider): Implements provider for AWS """ - def __init__(self, profile=None, report_dir=None, timestamp=None, services=None, skipped_services=None, thread_config=4, **kwargs): + def __init__(self, profile=None, report_dir=None, timestamp=None, services=None, skipped_services=None, + thread_config=4, **kwargs): services = [] if services is None else services skipped_services = [] if skipped_services is None else skipped_services @@ -112,11 +113,13 @@ def _check_ec2_zone_distribution(self): def _add_last_snapshot_date_to_ec2_volumes(self): for region in self.services['ec2']['regions'].values(): for volumeId, volume in region.get('volumes').items(): - completed_snapshots = [s for s in region['snapshots'].values() if s['VolumeId'] == volumeId and s['State'] == 'completed'] + completed_snapshots = [s for s in region['snapshots'].values() if + s['VolumeId'] == volumeId and s['State'] == 'completed'] sorted_snapshots = sorted(completed_snapshots, key=lambda s: s['StartTime'], reverse=True) volume['LastSnapshotDate'] = sorted_snapshots[0]['StartTime'] if len(sorted_snapshots) > 0 else None - def add_security_group_name_to_ec2_grants_callback(self, current_config, path, current_path, ec2_grant, callback_args): + def add_security_group_name_to_ec2_grants_callback(self, current_config, path, current_path, ec2_grant, + callback_args): sg_id = ec2_grant['GroupId'] if sg_id in current_path: target = current_path[:(current_path.index(sg_id) + 1)] @@ -239,8 +242,9 @@ def _match_iam_policies_and_buckets(self): parts = full_path.split('/') bucket_name = parts[0].split(':')[-1] self._update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, - iam_info['permissions']['Action'][action][iam_entity][ - 'Allow'][allowed_iam_entity]['Resource'][full_path]) + iam_info['permissions']['Action'][action][iam_entity][ + 'Allow'][allowed_iam_entity]['Resource'][ + full_path]) # For notresource statements, we must fetch the policy document to determine which buckets are not protected if 'NotResource' in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ allowed_iam_entity]: @@ -352,7 +356,8 @@ def _match_instances_and_roles(self): role_instances[instance_profile_id] iam_config['roles'][role_id]['instances_count'] += len(role_instances[instance_profile_id]) - def match_roles_and_cloudformation_stacks_callback(self, current_config, path, current_path, stack_id, callback_args): + def match_roles_and_cloudformation_stacks_callback(self, current_config, path, current_path, stack_id, + callback_args): if 'RoleARN' not in current_config: return role_arn = current_config.pop('RoleARN') @@ -376,7 +381,8 @@ def _get_role_info(self, attribute_name, attribute_value): def process_vpc_peering_connections_callback(self, current_config, path, current_path, pc_id, callback_args): # Create a list of peering connection IDs in each VPC - info = 'AccepterVpcInfo' if current_config['AccepterVpcInfo']['OwnerId'] == self.aws_account_id else 'RequesterVpcInfo' + info = 'AccepterVpcInfo' if current_config['AccepterVpcInfo'][ + 'OwnerId'] == self.aws_account_id else 'RequesterVpcInfo' region = current_path[current_path.index('regions') + 1] vpc_id = current_config[info]['VpcId'] if vpc_id not in self.services['vpc']['regions'][region]['vpcs']: @@ -399,7 +405,8 @@ def process_vpc_peering_connections_callback(self, current_config, path, current else: current_config['peer_info']['name'] = current_config['peer_info']['OwnerId'] - def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id, callback_args): + def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id, + callback_args): service = current_path[1] original_resource_path = combine_paths(copy.deepcopy(current_path), [resource_id]) resource = get_object_at(self, original_resource_path) @@ -574,7 +581,8 @@ def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_l attached_vpc['subnets'][subnet_id]['flow_logs'].append(flow_log_id) elif attached_resource.startswith('subnet-'): subnet_path = combine_paths(current_path[0:4], - ['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets', attached_resource]) + ['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets', + attached_resource]) subnet = get_object_at(self, subnet_path) manage_dictionary(subnet, 'flow_logs', []) if flow_log_id not in subnet['flow_logs']: diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index bb567c5ae..7eb9ff54b 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -6,7 +6,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # LambdaRegionConfig ######################################## @@ -18,7 +17,6 @@ def parse_function(self, global_params, region, function): self.functions[function['name']] = function - ######################################## # LambdaConfig ######################################## @@ -30,5 +28,5 @@ class LambdaConfig(RegionalServiceConfig): region_config_class = LambdaRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(LambdaConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/cloudformation.py b/ScoutSuite/providers/aws/services/cloudformation.py index bc770b890..2b683c937 100644 --- a/ScoutSuite/providers/aws/services/cloudformation.py +++ b/ScoutSuite/providers/aws/services/cloudformation.py @@ -5,7 +5,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # CloudFormationRegionConfig ######################################## @@ -25,7 +24,7 @@ def parse_stack(self, global_params, region, stack): """ stack['id'] = stack.pop('StackId') stack['name'] = stack.pop('StackName') - + stack_description = api_clients[region].describe_stacks(StackName=stack['name']) stack['termination_protection'] = stack_description['Stacks'][0]['EnableTerminationProtection'] @@ -35,7 +34,6 @@ def parse_stack(self, global_params, region, stack): self.stacks[stack['name']] = stack - ######################################## # CloudFormationConfig ######################################## @@ -47,5 +45,5 @@ class CloudFormationConfig(RegionalServiceConfig): region_config_class = CloudFormationRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(CloudFormationConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/cloudtrail.py b/ScoutSuite/providers/aws/services/cloudtrail.py index 0738c2662..0f085e2e6 100644 --- a/ScoutSuite/providers/aws/services/cloudtrail.py +++ b/ScoutSuite/providers/aws/services/cloudtrail.py @@ -7,7 +7,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # CloudTrailRegionConfig ######################################## @@ -45,22 +44,22 @@ def parse_trail(self, global_params, region, trail): if key not in trail_config: trail_config[key] = False trail_details = api_client.get_trail_status(Name=trail['TrailARN']) - for key in ['KMSKeyId', 'IsLogging', 'LatestDeliveryTime', 'LatestDeliveryError', 'StartLoggingTime', 'StopLoggingTime', 'LatestNotificationTime', 'LatestNotificationError', 'LatestCloudWatchLogsDeliveryError', 'LatestCloudWatchLogsDeliveryTime']: + for key in ['KMSKeyId', 'IsLogging', 'LatestDeliveryTime', 'LatestDeliveryError', 'StartLoggingTime', + 'StopLoggingTime', 'LatestNotificationTime', 'LatestNotificationError', + 'LatestCloudWatchLogsDeliveryError', 'LatestCloudWatchLogsDeliveryTime']: trail_config[key] = trail_details[key] if key in trail_details else None - if trail_details: # using trail ARN instead of name as with Organizations the trail would be located in another account trail_config['wildcard_data_logging'] = self.data_logging_status(trail_config['TrailARN'], trail_details, api_client) - + for es in api_client.get_event_selectors(TrailName=trail_config['TrailARN'])['EventSelectors']: trail_config['DataEventsEnabled'] = len(es['DataResources']) > 0 trail_config['ManagementEventsEnabled'] = es['IncludeManagementEvents'] - - self.trails[trail_id] = trail_config + self.trails[trail_id] = trail_config def data_logging_status(self, trail_name, trail_details, api_client): for es in api_client.get_event_selectors(TrailName=trail_name)['EventSelectors']: @@ -90,11 +89,10 @@ class CloudTrailConfig(RegionalServiceConfig): region_config_class = CloudTrailRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(CloudTrailConfig, self).__init__(service_metadata, thread_config) - ######################################## # Post Processing ######################################## @@ -108,7 +106,8 @@ def cloudtrail_postprocessing(aws_config): cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['flagged_items'] = 0 # Global services logging disabled if 'cloudtrail-no-global-services-logging' in cloudtrail_config['violations']: - if len(cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items']) != cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['checked_items']: + if len(cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items']) != \ + cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['checked_items']: cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items'] = [] cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['flagged_items'] = 0 # CloudTrail not enabled at all... diff --git a/ScoutSuite/providers/aws/services/cloudwatch.py b/ScoutSuite/providers/aws/services/cloudwatch.py index e5ccaf98a..95ce45dc6 100644 --- a/ScoutSuite/providers/aws/services/cloudwatch.py +++ b/ScoutSuite/providers/aws/services/cloudwatch.py @@ -6,7 +6,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # CloudWatchRegionConfig ######################################## @@ -33,7 +32,6 @@ def parse_alarm(self, global_params, region, alarm): self.alarms[alarm_id] = alarm - ######################################## # CloudWatchConfig ######################################## @@ -45,5 +43,5 @@ class CloudWatchConfig(RegionalServiceConfig): region_config_class = CloudWatchRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(CloudWatchConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/directconnect.py b/ScoutSuite/providers/aws/services/directconnect.py index f848930d7..761e8d4c0 100644 --- a/ScoutSuite/providers/aws/services/directconnect.py +++ b/ScoutSuite/providers/aws/services/directconnect.py @@ -3,7 +3,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # DirectConnectRegionConfig ######################################## @@ -26,7 +25,6 @@ def parse_connection(self, global_params, region, connection): self.connections[connection['id']] = connection - ######################################## # DirectConnectConfig ######################################## @@ -38,5 +36,5 @@ class DirectConnectConfig(RegionalServiceConfig): region_config_class = DirectConnectRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(DirectConnectConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index ce52dfe23..662c00066 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -16,7 +16,6 @@ from ScoutSuite.utils import get_keys, ec2_classic, manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # Globals ######################################## @@ -71,7 +70,8 @@ def parse_instance(self, global_params, region, reservation): self.vpcs[vpc_id].instances[i['InstanceId']] = instance def _get_user_data(self, region, instance_id): - user_data_response = api_clients[region].describe_instance_attribute(Attribute='userData', InstanceId=instance_id) + user_data_response = api_clients[region].describe_instance_attribute(Attribute='userData', + InstanceId=instance_id) if 'Value' not in user_data_response['UserData'].keys(): return None @@ -92,7 +92,7 @@ def parse_image(self, global_params, region, image): image['id'] = id image['name'] = name - self.images[id] = image + self.images[id] = image def parse_security_group(self, global_params, region, group): """ @@ -172,8 +172,9 @@ def parse_snapshot(self, global_params, region, snapshot): self.snapshots[snapshot['id']] = snapshot # Get snapshot attribute snapshot['createVolumePermission'] = \ - api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', SnapshotId=snapshot['id'])[ - 'CreateVolumePermissions'] + api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', + SnapshotId=snapshot['id'])[ + 'CreateVolumePermissions'] snapshot['public'] = self._is_public(snapshot) def _is_public(self, snapshot): @@ -225,6 +226,7 @@ def analyze_ec2_config(ec2_info, aws_account_id, force_write): printInfo('Error') printException(e) + def add_security_group_name_to_ec2_grants_callback(ec2_config, current_config, path, current_path, ec2_grant, callback_args): """ @@ -290,7 +292,7 @@ def link_elastic_ips_callback2(ec2_config, current_config, path, current_path, i current_config['PublicIpAddress'] = callback_args['elastic_ip'] elif current_config['PublicIpAddress'] != callback_args['elastic_ip']: printInfo('Warning: public IP address exists (%s) for an instance associated with an elastic IP (%s)' % ( - current_config['PublicIpAddress'], callback_args['elastic_ip'])) + current_config['PublicIpAddress'], callback_args['elastic_ip'])) # This can happen... fix it diff --git a/ScoutSuite/providers/aws/services/efs.py b/ScoutSuite/providers/aws/services/efs.py index 58198baf1..ab7e3dd85 100644 --- a/ScoutSuite/providers/aws/services/efs.py +++ b/ScoutSuite/providers/aws/services/efs.py @@ -7,7 +7,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # EFSRegionConfig ######################################## @@ -31,19 +30,21 @@ def parse_file_system(self, global_params, region, file_system): else: file_system['name'] = None # Get tags - file_system['tags'] = handle_truncated_response(api_clients[region].describe_tags, {'FileSystemId': fs_id}, ['Tags'])['Tags'] + file_system['tags'] = \ + handle_truncated_response(api_clients[region].describe_tags, {'FileSystemId': fs_id}, ['Tags'])['Tags'] # Get mount targets - mount_targets = handle_truncated_response(api_clients[region].describe_mount_targets, {'FileSystemId': fs_id}, ['MountTargets'])['MountTargets'] + mount_targets = handle_truncated_response(api_clients[region].describe_mount_targets, {'FileSystemId': fs_id}, + ['MountTargets'])['MountTargets'] file_system['mount_targets'] = {} for mt in mount_targets: mt_id = mt['MountTargetId'] file_system['mount_targets'][mt_id] = mt # Get security groups - file_system['mount_targets'][mt_id]['security_groups'] = api_clients[region].describe_mount_target_security_groups(MountTargetId = mt_id)['SecurityGroups'] + file_system['mount_targets'][mt_id]['security_groups'] = \ + api_clients[region].describe_mount_target_security_groups(MountTargetId=mt_id)['SecurityGroups'] self.file_systems[fs_id] = file_system - ######################################## # EFSConfig ######################################## @@ -55,5 +56,5 @@ class EFSConfig(RegionalServiceConfig): region_config_class = EFSRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(EFSConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/elasticache.py b/ScoutSuite/providers/aws/services/elasticache.py index ffdf6a541..9af741cc0 100644 --- a/ScoutSuite/providers/aws/services/elasticache.py +++ b/ScoutSuite/providers/aws/services/elasticache.py @@ -26,7 +26,9 @@ def parse_cluster(self, global_params, region, cluster): cluster['name'] = cluster_name # Must fetch info about the subnet group to retrieve the VPC ID... if 'CacheSubnetGroupName' in cluster: - subnet_group = api_clients[region].describe_cache_subnet_groups(CacheSubnetGroupName = cluster['CacheSubnetGroupName'])['CacheSubnetGroups'][0] + subnet_group = \ + api_clients[region].describe_cache_subnet_groups(CacheSubnetGroupName=cluster['CacheSubnetGroupName'])[ + 'CacheSubnetGroups'][0] vpc_id = subnet_group['VpcId'] else: vpc_id = ec2_classic @@ -36,7 +38,6 @@ def parse_cluster(self, global_params, region, cluster): if subnet_group: self.vpcs[vpc_id].subnet_groups[subnet_group['CacheSubnetGroupName']] = subnet_group - def parse_security_group(self, global_params, region, security_group): """ Parse a single ElastiCache security group @@ -50,7 +51,6 @@ def parse_security_group(self, global_params, region, security_group): self.security_groups[security_group['name']] = security_group - ######################################## # ElastiCacheConfig ######################################## @@ -62,5 +62,5 @@ class ElastiCacheConfig(RegionalServiceConfig): region_config_class = ElastiCacheRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(ElastiCacheConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/elb.py b/ScoutSuite/providers/aws/services/elb.py index 2d35580b4..5e32137be 100644 --- a/ScoutSuite/providers/aws/services/elb.py +++ b/ScoutSuite/providers/aws/services/elb.py @@ -46,7 +46,9 @@ def parse_elb(self, global_params, region, lb): elb['listeners'][l['Listener']['LoadBalancerPort']] = listener # Fetch LB policies here. This is not ideal, but the alternative is to download all policies and clean up after... if len(policy_names): - policies = api_clients[region].describe_load_balancer_policies(LoadBalancerName = elb['name'], PolicyNames = policy_names)['PolicyDescriptions'] + policies = \ + api_clients[region].describe_load_balancer_policies(LoadBalancerName=elb['name'], PolicyNames=policy_names)[ + 'PolicyDescriptions'] for policy in policies: policy['name'] = policy.pop('PolicyName') policy_id = self.get_non_provider_id(policy['name']) @@ -55,11 +57,11 @@ def parse_elb(self, global_params, region, lb): for i in lb['Instances']: elb['instances'].append(i['InstanceId']) # Get attributes - elb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerName=elb['name'])['LoadBalancerAttributes'] + elb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerName=elb['name'])[ + 'LoadBalancerAttributes'] self.vpcs[vpc_id].elbs[self.get_non_provider_id(elb['name'])] = elb - ######################################## # ELBConfig ######################################## @@ -71,5 +73,5 @@ class ELBConfig(RegionalServiceConfig): region_config_class = ELBRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(ELBConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index b67c1c5f5..f0166d781 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -43,14 +43,16 @@ def parse_lb(self, global_params, region, lb): pass lb['listeners'] = {} # Get listeners - listeners = handle_truncated_response(api_clients[region].describe_listeners, {'LoadBalancerArn': lb['arn']}, ['Listeners'])['Listeners'] + listeners = handle_truncated_response(api_clients[region].describe_listeners, {'LoadBalancerArn': lb['arn']}, + ['Listeners'])['Listeners'] for listener in listeners: listener.pop('ListenerArn') listener.pop('LoadBalancerArn') port = listener.pop('Port') lb['listeners'][port] = listener # Get attributes - lb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerArn = lb['arn'])['Attributes'] + lb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerArn=lb['arn'])[ + 'Attributes'] self.vpcs[vpc_id].lbs[self.get_non_provider_id(lb['name'])] = lb def parse_ssl_policie(self, global_params, region, policy): @@ -68,5 +70,5 @@ class ELBv2Config(RegionalServiceConfig): """ region_config_class = ELBv2RegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(ELBv2Config, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index dd67370a9..c0c4fe16c 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -425,4 +425,4 @@ def __parse_resource(self, effect, action_string, action, resource_string, resou self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ resource][policy_type], policy_name, {}) self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ - resource][policy_type][policy_name]['condition'] = condition \ No newline at end of file + resource][policy_type][policy_name]['condition'] = condition diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index eeff7459c..70425fe68 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -25,25 +25,25 @@ def parse_instance(self, global_params, region, dbi): :param region: Name of the AWS region :param instance: Instance """ - vpc_id = dbi['DBSubnetGroup']['VpcId'] if 'DBSubnetGroup' in dbi and 'VpcId' in dbi['DBSubnetGroup'] and dbi['DBSubnetGroup']['VpcId'] else ec2_classic + vpc_id = dbi['DBSubnetGroup']['VpcId'] if 'DBSubnetGroup' in dbi and 'VpcId' in dbi['DBSubnetGroup'] and \ + dbi['DBSubnetGroup']['VpcId'] else ec2_classic instance = {} instance['name'] = dbi.pop('DBInstanceIdentifier') for key in ['InstanceCreateTime', 'Engine', 'DBInstanceStatus', 'AutoMinorVersionUpgrade', 'DBInstanceClass', 'MultiAZ', 'Endpoint', 'BackupRetentionPeriod', 'PubliclyAccessible', 'StorageEncrypted', 'VpcSecurityGroups', 'DBSecurityGroups', 'DBParameterGroups', 'EnhancedMonitoringResourceArn', 'StorageEncrypted']: - # parameter_groups , security_groups, vpc_security_groups + # parameter_groups , security_groups, vpc_security_groups instance[key] = dbi[key] if key in dbi else None # If part of a cluster, multi AZ information is only available via cluster information if 'DBClusterIdentifier' in dbi: api_client = api_clients[region] - cluster = api_client.describe_db_clusters(DBClusterIdentifier = dbi['DBClusterIdentifier'])['DBClusters'][0] + cluster = api_client.describe_db_clusters(DBClusterIdentifier=dbi['DBClusterIdentifier'])['DBClusters'][0] instance['MultiAZ'] = cluster['MultiAZ'] # Save manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) self.vpcs[vpc_id].instances[instance['name']] = instance - def parse_snapshot(self, global_params, region, dbs): """ @@ -64,19 +64,21 @@ def parse_snapshot(self, global_params, region, dbs): for attribute in attributes: snapshot[attribute] = dbs[attribute] if attribute in dbs else None api_client = api_clients[region] - attributes = api_client.describe_db_snapshot_attributes(DBSnapshotIdentifier = snapshot_id)['DBSnapshotAttributesResult'] + attributes = api_client.describe_db_snapshot_attributes(DBSnapshotIdentifier=snapshot_id)[ + 'DBSnapshotAttributesResult'] snapshot['attributes'] = attributes['DBSnapshotAttributes'] if 'DBSnapshotAttributes' in attributes else {} # Save manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) self.vpcs[vpc_id].snapshots[snapshot_id] = snapshot - def parse_parameter_group(self, global_params, region, parameter_group): parameter_group['arn'] = parameter_group.pop('DBParameterGroupArn') parameter_group['name'] = parameter_group.pop('DBParameterGroupName') api_client = api_clients[region] try: - parameters = handle_truncated_response(api_client.describe_db_parameters, {'DBParameterGroupName': parameter_group['name']}, ['Parameters'])['Parameters'] + parameters = handle_truncated_response(api_client.describe_db_parameters, + {'DBParameterGroupName': parameter_group['name']}, ['Parameters'])[ + 'Parameters'] manage_dictionary(parameter_group, 'parameters', {}) for parameter in parameters: if not parameter['IsModifiable']: @@ -91,7 +93,6 @@ def parse_parameter_group(self, global_params, region, parameter_group): parameter_group_id = self.get_non_provider_id(parameter_group['name']) (self).parameter_groups[parameter_group_id] = parameter_group - def parse_security_group(self, global_params, region, security_group): """ Parse a single Redsfhit security group @@ -100,17 +101,16 @@ def parse_security_group(self, global_params, region, security_group): :param region: Name of the AWS region :param security)_group: Security group """ - #vpc_id = security_group.pop('VpcId') if 'VpcId' in security_group else ec2_classic - #manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) + # vpc_id = security_group.pop('VpcId') if 'VpcId' in security_group else ec2_classic + # manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) security_group['arn'] = security_group.pop('DBSecurityGroupArn') security_group['name'] = security_group.pop('DBSecurityGroupName') # Save - #manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - #self.vpcs[vpc_id].security_groups[security_group['name']] = security_group + # manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) + # self.vpcs[vpc_id].security_groups[security_group['name']] = security_group self.security_groups[security_group['name']] = security_group - ######################################## # RDSConfig ######################################## @@ -122,7 +122,7 @@ class RDSConfig(RegionalServiceConfig): region_config_class = RDSRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(RDSConfig, self).__init__(service_metadata, thread_config) @@ -136,6 +136,7 @@ def get_security_groups_info(rds_client, region_info): for group in groups: region_info['vpcs'][ec2_classic]['security_groups'][group['DBSecurityGroupName']] = parse_security_group(group) + def parse_security_group(group): security_group = {} security_group['name'] = group['DBSecurityGroupName'] diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index 772d40edc..505ea1848 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -33,7 +33,6 @@ def parse_cluster(self, global_params, region, cluster): cluster['name'] = name self.vpcs[vpc_id].clusters[name] = cluster - def parse_parameter_group(self, global_params, region, parameter_group): """ Parse a single Redshift parameter group and fetch all of its parameters @@ -43,11 +42,12 @@ def parse_parameter_group(self, global_params, region, parameter_group): :param parameter_group: Parameter group """ pg_name = parameter_group.pop('ParameterGroupName') - pg_id = self.get_non_provider_id(pg_name) # Name could be used as only letters digits or hyphens + pg_id = self.get_non_provider_id(pg_name) # Name could be used as only letters digits or hyphens parameter_group['name'] = pg_name parameter_group['parameters'] = {} api_client = api_clients[region] - parameters = handle_truncated_response(api_client.describe_cluster_parameters, {'ParameterGroupName': pg_name}, ['Parameters'])['Parameters'] + parameters = handle_truncated_response(api_client.describe_cluster_parameters, {'ParameterGroupName': pg_name}, + ['Parameters'])['Parameters'] for parameter in parameters: param = {} param['value'] = parameter['ParameterValue'] @@ -55,7 +55,6 @@ def parse_parameter_group(self, global_params, region, parameter_group): parameter_group['parameters'][parameter['ParameterName']] = param (self).parameter_groups[pg_id] = parameter_group - def parse_security_group(self, global_params, region, security_group): """ Parse a single Redsfhit security group @@ -69,7 +68,6 @@ def parse_security_group(self, global_params, region, security_group): self.security_groups['name'] = security_group - ######################################## # RedshiftConfig ######################################## @@ -81,5 +79,5 @@ class RedshiftConfig(RegionalServiceConfig): region_config_class = RedshiftRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(RedshiftConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/route53.py b/ScoutSuite/providers/aws/services/route53.py index 08d502f1c..b7290af1e 100644 --- a/ScoutSuite/providers/aws/services/route53.py +++ b/ScoutSuite/providers/aws/services/route53.py @@ -5,7 +5,6 @@ from ScoutSuite.providers.aws.configs.base import AWSBaseConfig - class Route53DomainsConfig(AWSBaseConfig): """ Object that holds the Route53Domains configuration @@ -20,8 +19,6 @@ def __init__(self, target_config): self.domains_count = 0 super(Route53DomainsConfig, self).__init__(target_config) - - ######################################## ##### Domains ######################################## @@ -31,14 +28,13 @@ def parse_domains(self, domain, params): """ domain_id = self.get_non_provider_id(domain['DomainName']) domain['name'] = domain.pop('DomainName') - #TODO: Get Dnssec info when available - #api_client = params['api_client'] - #details = api_client.get_domain_detail(DomainName = domain['name']) - #get_keys(details, domain, ['Dnssec']) + # TODO: Get Dnssec info when available + # api_client = params['api_client'] + # details = api_client.get_domain_detail(DomainName = domain['name']) + # get_keys(details, domain, ['Dnssec']) self.domains[domain_id] = domain - class Route53Config(AWSBaseConfig): """ Object that holds the Route53 configuration @@ -53,8 +49,6 @@ def __init__(self, target_config): self.hosted_zones_count = 0 super(Route53Config, self).__init__(target_config) - - ######################################## ##### hosted_zoness ######################################## @@ -66,9 +60,10 @@ def parse_hosted_zones(self, hosted_zone, params): hosted_zone_id = hosted_zone.pop('Id') hosted_zone['name'] = hosted_zone.pop('Name') api_client = params['api_client'] - record_sets = handle_truncated_response(api_client.list_resource_record_sets, {'HostedZoneId': hosted_zone_id}, ['ResourceRecordSets']) + record_sets = handle_truncated_response(api_client.list_resource_record_sets, {'HostedZoneId': hosted_zone_id}, + ['ResourceRecordSets']) hosted_zone.update(record_sets) - #print(str(record_sets)) - #record_sets = api_client.list_resource_record_sets() - #hosted_zone['RecordSets'] = record_sets['Resourc'] + # print(str(record_sets)) + # record_sets = api_client.list_resource_record_sets() + # hosted_zone['RecordSets'] = record_sets['Resourc'] self.hosted_zones[hosted_zone_id] = hosted_zone diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index da9b1d0e2..e7befa31e 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -272,6 +272,7 @@ def get_s3_bucket_versioning(api_client, bucket_name, bucket_info): bucket_info['version_mfa_delete_enabled'] = None return False + def _status_to_bool(value): """ Converts a string to True if it is equal to 'Enabled' or to False otherwise. """ return value == 'Enabled' diff --git a/ScoutSuite/providers/aws/services/ses.py b/ScoutSuite/providers/aws/services/ses.py index eff667512..237139cba 100644 --- a/ScoutSuite/providers/aws/services/ses.py +++ b/ScoutSuite/providers/aws/services/ses.py @@ -5,7 +5,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # SESRegionConfig ######################################## @@ -23,18 +22,19 @@ def parse_identitie(self, global_params, region, identity_name): :param region: Name of the AWS region """ identity = {'name': identity_name, 'policies': {}} - policy_names = api_clients[region].list_identity_policies(Identity = identity_name)['PolicyNames'] + policy_names = api_clients[region].list_identity_policies(Identity=identity_name)['PolicyNames'] if len(policy_names): - policies = api_clients[region].get_identity_policies(Identity = identity_name, PolicyNames = policy_names)['Policies'] + policies = api_clients[region].get_identity_policies(Identity=identity_name, PolicyNames=policy_names)[ + 'Policies'] for policy_name in policies: identity['policies'][policy_name] = json.loads(policies[policy_name]) - dkim = api_clients[region].get_identity_dkim_attributes(Identities = [ identity_name ])['DkimAttributes'][identity_name] + dkim = api_clients[region].get_identity_dkim_attributes(Identities=[identity_name])['DkimAttributes'][ + identity_name] identity['DkimEnabled'] = dkim['DkimEnabled'] identity['DkimVerificationStatus'] = dkim['DkimVerificationStatus'] self.identities[self.get_non_provider_id(identity_name)] = identity - ######################################## # SESConfig ######################################## @@ -46,5 +46,5 @@ class SESConfig(RegionalServiceConfig): region_config_class = SESRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(SESConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/sns.py b/ScoutSuite/providers/aws/services/sns.py index 20b066784..511a5282e 100644 --- a/ScoutSuite/providers/aws/services/sns.py +++ b/ScoutSuite/providers/aws/services/sns.py @@ -36,7 +36,6 @@ def parse_subscription(self, params, region, subscription): topic['subscriptions']['protocol'][protocol].append(subscription) topic['subscriptions_count'] += 1 - def parse_topic(self, params, region, topic): """ Parse a single topic and fetch additional attributes @@ -59,7 +58,6 @@ def parse_topic(self, params, region, topic): self.topics[topic['name']] = topic - ######################################## # SNSConfig ######################################## @@ -71,5 +69,5 @@ class SNSConfig(RegionalServiceConfig): region_config_class = SNSRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(SNSConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/sqs.py b/ScoutSuite/providers/aws/services/sqs.py index eab64bc6d..d4d512d8a 100644 --- a/ScoutSuite/providers/aws/services/sqs.py +++ b/ScoutSuite/providers/aws/services/sqs.py @@ -5,7 +5,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # SQSRegionConfig ######################################## @@ -24,7 +23,9 @@ def parse_queue(self, global_params, region, queue_url): :param queue_url: URL of the AWS queue """ queue = {'QueueUrl': queue_url} - attributes = api_clients[region].get_queue_attributes(QueueUrl = queue_url, AttributeNames = ['CreatedTimestamp', 'Policy', 'QueueArn', 'KmsMasterKeyId'])['Attributes'] + attributes = api_clients[region].get_queue_attributes(QueueUrl=queue_url, + AttributeNames=['CreatedTimestamp', 'Policy', 'QueueArn', + 'KmsMasterKeyId'])['Attributes'] queue['arn'] = attributes.pop('QueueArn') queue['kms_master_key_id'] = attributes.pop('KmsMasterKeyId', None) queue['CreatedTimestamp'] = attributes.pop('CreatedTimestamp', None) @@ -38,7 +39,6 @@ def parse_queue(self, global_params, region, queue_url): self.queues[queue['name']] = queue - ######################################## # SQSConfig ######################################## @@ -50,5 +50,5 @@ class SQSConfig(RegionalServiceConfig): region_config_class = SQSRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(SQSConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index e6cf953e9..461b8bee4 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -17,6 +17,7 @@ protocols_dict = load_data('protocols.json', 'protocols') + ######################################## # VPCRegionConfig ######################################## @@ -159,6 +160,7 @@ def __init__(self, service_metadata, thread_config): known_cidrs = {'0.0.0.0/0': 'All'} + def put_cidr_name(aws_config, current_config, path, current_path, resource_id, callback_args): """ Add a display name for all known CIDRs @@ -186,6 +188,7 @@ def put_cidr_name(aws_config, current_config, path, current_path, resource_id, c aws_ip_ranges = {} # read_ip_ranges(aws_ip_ranges_filename, False) + def get_cidr_name(cidr, ip_ranges_files, ip_ranges_name_key): """ Read display name for CIDRs from ip-ranges files @@ -208,6 +211,7 @@ def get_cidr_name(cidr, ip_ranges_files, ip_ranges_name_key): return 'Unknown CIDR in %s %s' % (ip_range['service'], ip_range['region']) return 'Unknown CIDR' + def propagate_vpc_names(aws_config, current_config, path, current_path, resource_id, callback_args): """ Propagate VPC names in VPC-related services (info only fetched during EC2 calls) @@ -229,6 +233,7 @@ def propagate_vpc_names(aws_config, current_config, path, current_path, resource target_path = '.'.join(target_path) current_config['name'] = get_value_at(aws_config, target_path, target_path) + def get_subnet_flow_logs_list(current_config, subnet): """ Return the flow logs that cover a given subnet From 75924804f88f478541475f7181019e57ee26ae8c Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 08:34:08 -0500 Subject: [PATCH 028/154] [Opinel] Removed CLI --- opinel/utils/cli_parser.py | 179 ------------------------------------- 1 file changed, 179 deletions(-) delete mode 100644 opinel/utils/cli_parser.py diff --git a/opinel/utils/cli_parser.py b/opinel/utils/cli_parser.py deleted file mode 100644 index dbb8d6ce2..000000000 --- a/opinel/utils/cli_parser.py +++ /dev/null @@ -1,179 +0,0 @@ -# -*- coding: utf-8 -*- - -import argparse -import json -import os -import sys -import tempfile - -opinel_arg_dir = os.path.join(os.path.expanduser('~'), '.aws/opinel') - -class OpinelArgumentParser(object): - """ - """ - - def __init__(self, tool_name = ''): - self.parser = argparse.ArgumentParser() - self.default_args = read_default_args(tool_name) - - def add_argument(self, argument_name, help = None, dest = None, nargs = None, default = None, action = None, choices = None): - - # Built-in, common arguments - if argument_name == 'debug': - self.parser.add_argument('--debug', - dest='debug', - default=False, - action='store_true', - help='Print the stack trace when exception occurs' if not help else help) - elif argument_name == 'dry-run': - self.parser.add_argument('--dry-run', - dest='dry_run', - default=False, - action='store_true', - help='Executes read-only actions (check status, describe*, get*, list*...)' if not help else help) - elif argument_name == 'profile': - default = os.environ.get('AWS_PROFILE', 'default') - default_origin = " (from AWS_PROFILE)." if 'AWS_PROFILE' in os.environ else "." - self.parser.add_argument('--profile', - dest='profile', - default=[default], - nargs='+', - help='Name of the profile. Defaults to %(default)s' + default_origin if not help else help) - elif argument_name == 'regions': - self.parser.add_argument('--regions', - dest='regions', - default=[], - nargs='+', - help='Name of regions to run the tool in, defaults to all' if not help else help) - elif argument_name == 'partition-name': - self.parser.add_argument('--partition-name', - dest='partition_name', - default='aws', - help='Switch out of the public AWS partition (e.g. US gov or China)') - elif argument_name == 'vpc': - self.parser.add_argument('--vpc', - dest='vpc', - default=[], - nargs='+', - help='Name of VPC to run the tool in, defaults to all' if not help else help) - elif argument_name == 'force': - self.parser.add_argument('--force', - dest='force_write', - default=False, - action='store_true', - help='Overwrite existing files' if not help else help) - elif argument_name == 'ip-ranges': - self.parser.add_argument('--ip-ranges', - dest='ip_ranges', - default=[], - nargs='+', - help='Config file(s) that contain your known IP ranges.' if not help else help) - elif argument_name == 'ip-ranges-name-key': - self.parser.add_argument('--ip-ranges-name-key', - dest='ip_ranges_name_key', - default='name', - help='Name of the key containing the display name of a known CIDR.' if not help else help) - elif argument_name == 'mfa-serial': - self.parser.add_argument('--mfa-serial', - dest='mfa_serial', - default=None, - help='ARN of the user\'s MFA device' if not help else help) - elif argument_name == 'mfa-code': - self.parser.add_argument('--mfa-code', - dest='mfa_code', - default=None, - help='Six-digit code displayed on the MFA device.' if not help else help) - elif argument_name == 'csv-credentials': - self.parser.add_argument('--csv-credentials', - dest='csv_credentials', - default=None, - help='Path to a CSV file containing the access key ID and secret key' if not help else help) - elif argument_name == 'user-name': - self.parser.add_argument('--user-name', - dest='user_name', - default=[], - nargs='+', - help='Name of the user.' if not help else help) - elif argument_name == 'bucket-name': - self.parser.add_argument('--bucket-name', - dest='bucket_name', - default=[None], - help='Name of the s3 bucket.' if not help else help) - elif argument_name == 'group-name': - self.parser.add_argument('--group-name', - dest='group_name', - default=[], - nargs='+', - help='Name of the IAM group.' if not help else help) - - # Default - elif help != None and default != None and (nargs != None or action != None): - dest = argument_name.replace('-', '_') if not dest else dest - if nargs: - if not choices: - self.parser.add_argument('--%s' % argument_name, - dest = dest, - default = self.default_args[dest] if dest in self.default_args else default, - nargs = nargs, - help = help) - else: - self.parser.add_argument('--%s' % argument_name, - dest = dest, - default = self.default_args[dest] if dest in self.default_args else default, - nargs = nargs, - choices = choices, - help = help) - elif action: - self.parser.add_argument('--%s' % argument_name, - dest = dest, - default = self.default_args[dest] if dest in self.default_args else default, - action = action, - help = help) - - # Error - else: - raise Exception('Invalid parameter name %s' % argument_name) - - - def parse_args(self): - args = self.parser.parse_args() - return args - - -def read_default_args(tool_name): - """ - Read default argument values for a given tool - - :param tool_name: Name of the script to read the default arguments for - :return: Dictionary of default arguments (shared + tool-specific) - """ - global opinel_arg_dir - - profile_name = 'default' - # h4ck to have an early read of the profile name - for i, arg in enumerate(sys.argv): - if arg == '--profile' and len(sys.argv) >= i + 1: - profile_name = sys.argv[i + 1] - #if not os.path.isdir(opinel_arg_dir): - # os.makedirs(opinel_arg_dir) - if not os.path.isdir(opinel_arg_dir): - try: - os.makedirs(opinel_arg_dir) - except: - # Within AWS Lambda, home directories are not writable. This attempts to detect that... - # ...and uses the /tmp folder, which *is* writable in AWS Lambda - opinel_arg_dir = os.path.join(tempfile.gettempdir(), '.aws/opinel') - if not os.path.isdir(opinel_arg_dir): - os.makedirs(opinel_arg_dir) - opinel_arg_file = os.path.join(opinel_arg_dir, '%s.json' % profile_name) - default_args = {} - if os.path.isfile(opinel_arg_file): - with open(opinel_arg_file, 'rt') as f: - all_args = json.load(f) - for target in all_args: - if tool_name.endswith(target): - default_args.update(all_args[target]) - for k in all_args['shared']: - if k not in default_args: - default_args[k] = all_args['shared'][k] - return default_args From 45662148d94357f462d1d777415224fe1329cdad Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 08:36:44 -0500 Subject: [PATCH 029/154] [Opinel] Removed data/requirements.txt --- opinel/data/requirements.txt | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 opinel/data/requirements.txt diff --git a/opinel/data/requirements.txt b/opinel/data/requirements.txt deleted file mode 100644 index 8ba8bd8a8..000000000 --- a/opinel/data/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -boto3>=1.4.4 -requests>=2.4.0,<3.0.0 -netaddr>=0.7.11 -iampoliciesgonewild>=1.0.6.2 -pyyaml>=3.12 From ad661d250f5c9e981fc5f6e1b6250067f4337126 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 08:58:50 -0500 Subject: [PATCH 030/154] [Opinel] Moved console to ScoutSuite --- ScoutSuite/__listall__.py | 2 +- ScoutSuite/__main__.py | 2 +- ScoutSuite/core/exceptions.py | 2 +- ScoutSuite/core/processingengine.py | 2 +- ScoutSuite/core/rule.py | 3 +-- ScoutSuite/core/rule_definition.py | 2 +- ScoutSuite/core/ruleset.py | 2 +- ScoutSuite/core/utils.py | 2 +- ScoutSuite/output/html.py | 4 ++-- ScoutSuite/output/js.py | 2 +- ScoutSuite/output/utils.py | 2 +- ScoutSuite/providers/aws/configs/regions.py | 2 +- ScoutSuite/providers/aws/configs/services.py | 3 --- ScoutSuite/providers/aws/provider.py | 4 ++-- ScoutSuite/providers/aws/services/ec2.py | 3 +-- ScoutSuite/providers/aws/services/iam.py | 2 +- ScoutSuite/providers/aws/services/rds.py | 2 +- ScoutSuite/providers/aws/services/s3.py | 2 +- ScoutSuite/providers/azure/provider.py | 2 +- ScoutSuite/providers/azure/utils.py | 2 +- ScoutSuite/providers/base/configs/base.py | 2 +- ScoutSuite/providers/base/configs/browser.py | 2 +- ScoutSuite/providers/base/configs/services.py | 4 ++-- ScoutSuite/providers/base/provider.py | 3 +-- ScoutSuite/providers/gcp/configs/base.py | 2 +- ScoutSuite/providers/gcp/configs/services.py | 3 --- ScoutSuite/providers/gcp/provider.py | 2 +- ScoutSuite/providers/gcp/services/cloudresourcemanager.py | 2 +- ScoutSuite/providers/gcp/services/cloudsql.py | 3 +-- ScoutSuite/providers/gcp/services/cloudstorage.py | 2 +- ScoutSuite/providers/gcp/services/computeengine.py | 3 --- ScoutSuite/providers/gcp/services/iam.py | 3 +-- ScoutSuite/providers/gcp/services/stackdriverlogging.py | 2 -- ScoutSuite/providers/gcp/services/stackdrivermonitoring.py | 2 -- ScoutSuite/providers/gcp/utils.py | 2 +- {opinel/utils => core}/console.py | 3 ++- opinel/services/cloudformation.py | 5 ++--- opinel/services/iam.py | 2 +- opinel/services/organizations.py | 2 +- opinel/utils/aws.py | 2 +- opinel/utils/conditions.py | 2 +- opinel/utils/credentials.py | 4 ++-- opinel/utils/fs.py | 2 +- opinel/utils/globals.py | 2 +- opinel/utils/profiles.py | 3 +-- opinel/utils/threads.py | 2 +- tests/test-rules-processingengine.py | 2 +- tests/test-rules-ruleset.py | 2 +- tests/test-scoutsuite.py | 1 - tests/test-utils_cloudwatch.py | 2 +- tests/test-utils_sns.py | 2 +- 51 files changed, 51 insertions(+), 71 deletions(-) rename {opinel/utils => core}/console.py (99%) diff --git a/ScoutSuite/__listall__.py b/ScoutSuite/__listall__.py index f5a6c7ad0..e4598fe0b 100644 --- a/ScoutSuite/__listall__.py +++ b/ScoutSuite/__listall__.py @@ -7,7 +7,7 @@ try: from opinel.utils.globals import check_requirements - from opinel.utils.console import configPrintException, printError, printException, printInfo + from core.console import configPrintException, printError, printException, printInfo except Exception as e: print('Error: Scout2 depends on the opinel package. Install all the requirements with the following command:') print(' $ pip install -r requirements.txt') diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 1acc5d39f..5504cb50e 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -6,7 +6,7 @@ import os import webbrowser -from opinel.utils.console import configPrintException, printInfo, printDebug +from core.console import configPrintException, printInfo, printDebug from opinel.utils.profiles import AWSProfiles from ScoutSuite import AWSCONFIG diff --git a/ScoutSuite/core/exceptions.py b/ScoutSuite/core/exceptions.py index 79035b5f7..d76976750 100644 --- a/ScoutSuite/core/exceptions.py +++ b/ScoutSuite/core/exceptions.py @@ -3,7 +3,7 @@ Exceptions handling """ -from opinel.utils.console import printDebug +from core.console import printDebug from ScoutSuite import EXCEPTIONS from ScoutSuite.output.js import JavaScriptReaderWriter diff --git a/ScoutSuite/core/processingengine.py b/ScoutSuite/core/processingengine.py index a4bea6769..c26559891 100644 --- a/ScoutSuite/core/processingengine.py +++ b/ScoutSuite/core/processingengine.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printDebug, printError, printException +from core.console import printDebug, printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.core.utils import recurse diff --git a/ScoutSuite/core/rule.py b/ScoutSuite/core/rule.py index 2fff065e1..6b48b4889 100644 --- a/ScoutSuite/core/rule.py +++ b/ScoutSuite/core/rule.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- import json -import os import re from opinel.utils.fs import read_ip_ranges -from opinel.utils.console import printDebug, printError, printException +from core.console import printError from ScoutSuite.utils import format_service_name diff --git a/ScoutSuite/core/rule_definition.py b/ScoutSuite/core/rule_definition.py index d0a6a1657..1e92ad6d6 100644 --- a/ScoutSuite/core/rule_definition.py +++ b/ScoutSuite/core/rule_definition.py @@ -3,7 +3,7 @@ import json import os -from opinel.utils.console import printDebug, printError, printException +from core.console import printError class RuleDefinition(object): diff --git a/ScoutSuite/core/ruleset.py b/ScoutSuite/core/ruleset.py index 7b14eb552..a32c5191a 100644 --- a/ScoutSuite/core/ruleset.py +++ b/ScoutSuite/core/ruleset.py @@ -4,7 +4,7 @@ import os import tempfile -from opinel.utils.console import printDebug, printError, prompt_4_yes_no +from core.console import printDebug, printError, prompt_4_yes_no from ScoutSuite.core.rule import Rule from ScoutSuite.core.rule_definition import RuleDefinition diff --git a/ScoutSuite/core/utils.py b/ScoutSuite/core/utils.py index af7229ccf..1b25face9 100644 --- a/ScoutSuite/core/utils.py +++ b/ScoutSuite/core/utils.py @@ -8,7 +8,7 @@ import re from opinel.utils.conditions import pass_condition -from opinel.utils.console import printError, printException +from core.console import printError, printException from ScoutSuite.core import condition_operators from ScoutSuite.providers.base.configs.browser import get_value_at diff --git a/ScoutSuite/output/html.py b/ScoutSuite/output/html.py index 5a7321a35..9b01fd079 100644 --- a/ScoutSuite/output/html.py +++ b/ScoutSuite/output/html.py @@ -7,10 +7,10 @@ import zipfile import dateutil.tz -from opinel.utils.console import printInfo, printException +from core.console import printInfo, printException from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, \ - GENERATOR_FILE, REPORT_TITLE + GENERATOR_FILE from ScoutSuite.output.js import JavaScriptReaderWriter from ScoutSuite.output.utils import get_filename, prompt_4_overwrite diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index 7292384d4..5df552290 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -6,7 +6,7 @@ import json import os -from opinel.utils.console import printException, printInfo +from core.console import printException, printInfo from ScoutSuite import DEFAULT_REPORT_DIR from ScoutSuite.output.utils import get_filename, prompt_4_overwrite diff --git a/ScoutSuite/output/utils.py b/ScoutSuite/output/utils.py index 748ec5e4e..20ef42211 100644 --- a/ScoutSuite/output/utils.py +++ b/ScoutSuite/output/utils.py @@ -4,7 +4,7 @@ import os import sys -from opinel.utils.console import printError +from core.console import printError from six.moves import input diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 08aa99815..1d4943b23 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -13,7 +13,7 @@ from queue import Queue from opinel.utils.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response -from opinel.utils.console import printException, printInfo +from core.console import printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.base.configs import resource_id_map diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 23290e87c..4705bd665 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError, printException, printInfo, printDebug - from ScoutSuite.providers.aws.services.awslambda import LambdaConfig from ScoutSuite.providers.aws.services.cloudformation import CloudFormationConfig from ScoutSuite.providers.aws.services.cloudtrail import CloudTrailConfig @@ -23,7 +21,6 @@ from ScoutSuite.providers.aws.services.sqs import SQSConfig from ScoutSuite.providers.aws.services.vpc import VPCConfig from ScoutSuite.providers.base.configs.services import BaseServicesConfig -from ScoutSuite.utils import format_service_name try: from ScoutSuite.providers.aws.services.config_private import ConfigConfig diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 4054370a4..5a57fa55a 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -6,7 +6,7 @@ try: from opinel.utils.aws import get_aws_account_id, get_partition_name - from opinel.utils.console import configPrintException, printInfo, printDebug + from core.console import configPrintException, printInfo, printDebug from opinel.utils.credentials import read_creds from opinel.utils.globals import check_requirements from opinel.utils.profiles import AWSProfiles @@ -16,7 +16,7 @@ print(e) sys.exit(42) -from opinel.utils.console import printDebug, printError, printException, printInfo +from core.console import printDebug, printError, printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.services import AWSServicesConfig diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 915589599..62de742d9 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -8,13 +8,12 @@ import base64 from opinel.utils.aws import get_name -from opinel.utils.console import printException, printInfo +from core.console import printException, printInfo from opinel.utils.fs import load_data from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.providers.base.configs.browser import get_attribute_at -from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.utils import get_keys, ec2_classic from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index aa6c5d4db..a60f85d4a 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -2,7 +2,7 @@ from botocore.exceptions import ClientError from opinel.utils.aws import connect_service, handle_truncated_response -from opinel.utils.console import printError, printException +from core.console import printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index 2f8a190a4..55c127590 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from opinel.utils.aws import handle_truncated_response -from opinel.utils.console import printError, printException +from core.console import printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index d3a0be048..8456a1549 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -8,7 +8,7 @@ from botocore.exceptions import ClientError from opinel.services.s3 import get_s3_bucket_location from opinel.utils.aws import handle_truncated_response -from opinel.utils.console import printError, printException, printInfo +from core.console import printError, printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/azure/provider.py b/ScoutSuite/providers/azure/provider.py index 41a249740..5937c48a7 100644 --- a/ScoutSuite/providers/azure/provider.py +++ b/ScoutSuite/providers/azure/provider.py @@ -5,7 +5,7 @@ from getpass import getpass -from opinel.utils.console import printError, printException +from core.console import printError, printException from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.azure.configs.services import AzureServicesConfig diff --git a/ScoutSuite/providers/azure/utils.py b/ScoutSuite/providers/azure/utils.py index 7096b0b26..717bf5654 100644 --- a/ScoutSuite/providers/azure/utils.py +++ b/ScoutSuite/providers/azure/utils.py @@ -2,7 +2,7 @@ import re -from opinel.utils.console import printException, printInfo +from core.console import printException from azure.mgmt.storage import StorageManagementClient from azure.mgmt.monitor import MonitorManagementClient diff --git a/ScoutSuite/providers/base/configs/base.py b/ScoutSuite/providers/base/configs/base.py index 554ea4ce9..cdd62d216 100644 --- a/ScoutSuite/providers/base/configs/base.py +++ b/ScoutSuite/providers/base/configs/base.py @@ -21,7 +21,7 @@ from ScoutSuite.providers.azure.utils import azure_connect_service from opinel.utils.aws import build_region_list -from opinel.utils.console import printException, printInfo +from core.console import printException, printInfo from ScoutSuite.output.console import FetchStatusLogger from ScoutSuite.utils import format_service_name diff --git a/ScoutSuite/providers/base/configs/browser.py b/ScoutSuite/providers/base/configs/browser.py index 59aade272..3dee54d71 100644 --- a/ScoutSuite/providers/base/configs/browser.py +++ b/ScoutSuite/providers/base/configs/browser.py @@ -2,7 +2,7 @@ import copy -from opinel.utils.console import printError, printException +from core.console import printError, printException ######################################## diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 6fcf89400..5a89a9ba2 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError, printException, printDebug -from opinel.utils.aws import get_aws_account_id, get_partition_name +from core.console import printError, printException, printDebug +from opinel.utils.aws import get_partition_name class BaseServicesConfig(object): diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index 80cba7087..a6e6aa78c 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -7,8 +7,7 @@ import sys import copy -from opinel.utils.console import printException, printInfo -from opinel.utils.globals import manage_dictionary +from core.console import printException, printInfo from ScoutSuite import __version__ as scout2_version from ScoutSuite.providers.base.configs.browser import get_object_at diff --git a/ScoutSuite/providers/gcp/configs/base.py b/ScoutSuite/providers/gcp/configs/base.py index 756d95601..3db702c33 100644 --- a/ScoutSuite/providers/gcp/configs/base.py +++ b/ScoutSuite/providers/gcp/configs/base.py @@ -14,7 +14,7 @@ from google.cloud import container_v1 from googleapiclient.errors import HttpError -from opinel.utils.console import printException, printError +from core.console import printException, printError from ScoutSuite.providers.base.configs.base import BaseConfig from ScoutSuite.providers.gcp.utils import gcp_connect_service diff --git a/ScoutSuite/providers/gcp/configs/services.py b/ScoutSuite/providers/gcp/configs/services.py index 514b6265b..0693601a4 100644 --- a/ScoutSuite/providers/gcp/configs/services.py +++ b/ScoutSuite/providers/gcp/configs/services.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError, printException, printDebug - from ScoutSuite.providers.base.configs.services import BaseServicesConfig from ScoutSuite.providers.gcp.services.cloudstorage import CloudStorageConfig from ScoutSuite.providers.gcp.services.cloudsql import CloudSQLConfig from ScoutSuite.providers.gcp.services.iam import IAMConfig from ScoutSuite.providers.gcp.services.stackdriverlogging import StackdriverLoggingConfig -from ScoutSuite.providers.gcp.services.stackdrivermonitoring import StackdriverMonitoringConfig from ScoutSuite.providers.gcp.services.computeengine import ComputeEngineConfig from ScoutSuite.providers.gcp.services.cloudresourcemanager import CloudResourceManager diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py index ede8e9c3c..adf6721be 100644 --- a/ScoutSuite/providers/gcp/provider.py +++ b/ScoutSuite/providers/gcp/provider.py @@ -5,7 +5,7 @@ import google.auth import googleapiclient -from opinel.utils.console import printError, printException, printInfo +from core.console import printError, printException, printInfo from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.gcp.configs.services import GCPServicesConfig diff --git a/ScoutSuite/providers/gcp/services/cloudresourcemanager.py b/ScoutSuite/providers/gcp/services/cloudresourcemanager.py index 385a505d3..c1ec32399 100644 --- a/ScoutSuite/providers/gcp/services/cloudresourcemanager.py +++ b/ScoutSuite/providers/gcp/services/cloudresourcemanager.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from opinel.utils.console import printError, printException, printInfo +from core.console import printException class CloudResourceManager(GCPBaseConfig): diff --git a/ScoutSuite/providers/gcp/services/cloudsql.py b/ScoutSuite/providers/gcp/services/cloudsql.py index de24abd6c..bb252e1a8 100644 --- a/ScoutSuite/providers/gcp/services/cloudsql.py +++ b/ScoutSuite/providers/gcp/services/cloudsql.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError +from core.console import printError -import operator from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/services/cloudstorage.py b/ScoutSuite/providers/gcp/services/cloudstorage.py index b7c1cacfd..72f260783 100644 --- a/ScoutSuite/providers/gcp/services/cloudstorage.py +++ b/ScoutSuite/providers/gcp/services/cloudstorage.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError +from core.console import printError from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/services/computeengine.py b/ScoutSuite/providers/gcp/services/computeengine.py index 72c82b28a..4a03d8677 100644 --- a/ScoutSuite/providers/gcp/services/computeengine.py +++ b/ScoutSuite/providers/gcp/services/computeengine.py @@ -2,9 +2,6 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from googleapiclient.errors import HttpError -import json -from opinel.utils.console import printException, printError class ComputeEngineConfig(GCPBaseConfig): targets = ( diff --git a/ScoutSuite/providers/gcp/services/iam.py b/ScoutSuite/providers/gcp/services/iam.py index 8a58a5f9d..ddce638fd 100644 --- a/ScoutSuite/providers/gcp/services/iam.py +++ b/ScoutSuite/providers/gcp/services/iam.py @@ -2,10 +2,9 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from opinel.utils.console import printError, printException, printInfo +from core.console import printError from googleapiclient import discovery -from ScoutSuite.providers.gcp.utils import gcp_connect_service class IAMConfig(GCPBaseConfig): diff --git a/ScoutSuite/providers/gcp/services/stackdriverlogging.py b/ScoutSuite/providers/gcp/services/stackdriverlogging.py index 5bb5878ec..53998933b 100644 --- a/ScoutSuite/providers/gcp/services/stackdriverlogging.py +++ b/ScoutSuite/providers/gcp/services/stackdriverlogging.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError - from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/services/stackdrivermonitoring.py b/ScoutSuite/providers/gcp/services/stackdrivermonitoring.py index d8d0e94ed..ddb003b55 100644 --- a/ScoutSuite/providers/gcp/services/stackdrivermonitoring.py +++ b/ScoutSuite/providers/gcp/services/stackdrivermonitoring.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printError - from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/utils.py b/ScoutSuite/providers/gcp/utils.py index c2ea17de3..adc81dbcd 100644 --- a/ScoutSuite/providers/gcp/utils.py +++ b/ScoutSuite/providers/gcp/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.console import printException +from core.console import printException from google.cloud import storage from google.cloud import logging as stackdriver_logging diff --git a/opinel/utils/console.py b/core/console.py similarity index 99% rename from opinel/utils/console.py rename to core/console.py index 6b4a71357..efd4b1b21 100644 --- a/opinel/utils/console.py +++ b/core/console.py @@ -7,7 +7,8 @@ try: input = raw_input -except NameError: pass +except NameError: + pass ######################################## diff --git a/opinel/services/cloudformation.py b/opinel/services/cloudformation.py index 0478e681c..480a6a2e0 100644 --- a/opinel/services/cloudformation.py +++ b/opinel/services/cloudformation.py @@ -1,14 +1,13 @@ # -*- coding: utf-8 -*- -import json import os import re import time from opinel.utils.aws import connect_service, handle_truncated_response -from opinel.utils.console import printDebug, printInfo, printError, printException, prompt_4_yes_no +from core.console import printDebug, printInfo, printError, printException from opinel.utils.fs import read_file -from opinel.utils.globals import snake_to_camel, snake_to_words +from opinel.utils.globals import snake_to_camel re_iam_capability = re.compile('.*?AWS::IAM.*?', re.DOTALL | re.MULTILINE) diff --git a/opinel/services/iam.py b/opinel/services/iam.py index 41c8a4347..a59df2d44 100644 --- a/opinel/services/iam.py +++ b/opinel/services/iam.py @@ -4,7 +4,7 @@ from opinel.utils.aws import handle_truncated_response from opinel.utils.credentials import generate_password -from opinel.utils.console import printInfo, printError, printException +from core.console import printInfo, printError, printException diff --git a/opinel/services/organizations.py b/opinel/services/organizations.py index fee03f458..355efbfe2 100644 --- a/opinel/services/organizations.py +++ b/opinel/services/organizations.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from opinel.utils.aws import handle_truncated_response -from opinel.utils.console import printDebug, printInfo +from core.console import printDebug, printInfo def get_organization_account_ids(api_client, exceptions = [], quiet = True): diff --git a/opinel/utils/aws.py b/opinel/utils/aws.py index 6872d129c..705bb61e4 100644 --- a/opinel/utils/aws.py +++ b/opinel/utils/aws.py @@ -5,7 +5,7 @@ from collections import Counter import time -from opinel.utils.console import printInfo, printException +from core.console import printInfo, printException diff --git a/opinel/utils/conditions.py b/opinel/utils/conditions.py index 6e1982205..a3cac7fca 100644 --- a/opinel/utils/conditions.py +++ b/opinel/utils/conditions.py @@ -6,7 +6,7 @@ import netaddr import re -from opinel.utils.console import printError +from core.console import printError from iampoliciesgonewild import get_actions_from_statement, _expand_wildcard_action diff --git a/opinel/utils/credentials.py b/opinel/utils/credentials.py index 126b0a6b7..0bb0b151c 100644 --- a/opinel/utils/credentials.py +++ b/opinel/utils/credentials.py @@ -11,8 +11,8 @@ import requests # TODO: get rid of that and make sure urllib2 validates certs ? import string -from opinel.utils.console import printException, printError, printInfo -from opinel.utils.console import prompt_4_mfa_code +from core.console import printException, printError, printInfo +from core.console import prompt_4_mfa_code from opinel.utils.fs import save_blob_as_json from opinel.utils.aws import connect_service diff --git a/opinel/utils/fs.py b/opinel/utils/fs.py index 8c2ee07d6..7978a8ec4 100644 --- a/opinel/utils/fs.py +++ b/opinel/utils/fs.py @@ -6,7 +6,7 @@ import os import yaml -from opinel.utils.console import printError, printException, prompt_4_overwrite +from core.console import printError, printException, prompt_4_overwrite from opinel.utils.conditions import pass_condition diff --git a/opinel/utils/globals.py b/opinel/utils/globals.py index 5164e7cd4..34072085c 100644 --- a/opinel/utils/globals.py +++ b/opinel/utils/globals.py @@ -6,7 +6,7 @@ import re from opinel import __version__ as OPINEL_VERSION -from opinel.utils.console import printError +from core.console import printError ######################################## diff --git a/opinel/utils/profiles.py b/opinel/utils/profiles.py index 4b398eb58..081337f6b 100644 --- a/opinel/utils/profiles.py +++ b/opinel/utils/profiles.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- -import fileinput import os import re from opinel.utils.aws import get_aws_account_id -from opinel.utils.console import printDebug +from core.console import printDebug from opinel.utils.credentials import read_creds aws_dir = os.path.join(os.path.expanduser('~'), '.aws') diff --git a/opinel/utils/threads.py b/opinel/utils/threads.py index 9309b715a..d78742a54 100644 --- a/opinel/utils/threads.py +++ b/opinel/utils/threads.py @@ -8,7 +8,7 @@ # Python3 from queue import Queue -from opinel.utils.console import printException +from core.console import printException diff --git a/tests/test-rules-processingengine.py b/tests/test-rules-processingengine.py index f95d66d76..f93da05dd 100644 --- a/tests/test-rules-processingengine.py +++ b/tests/test-rules-processingengine.py @@ -4,7 +4,7 @@ import os import tempfile -from opinel.utils.console import configPrintException, printError +from core.console import configPrintException, printError from ScoutSuite.core.processingengine import ProcessingEngine from ScoutSuite.core.ruleset import Ruleset diff --git a/tests/test-rules-ruleset.py b/tests/test-rules-ruleset.py index fe5593d5f..a639ef7a5 100644 --- a/tests/test-rules-ruleset.py +++ b/tests/test-rules-ruleset.py @@ -3,7 +3,7 @@ import os from mock import patch -from opinel.utils.console import configPrintException, printDebug +from core.console import configPrintException, printDebug from ScoutSuite.core.rule import Rule from ScoutSuite.core.ruleset import Ruleset diff --git a/tests/test-scoutsuite.py b/tests/test-scoutsuite.py index 362bad187..1af03246d 100644 --- a/tests/test-scoutsuite.py +++ b/tests/test-scoutsuite.py @@ -3,7 +3,6 @@ import mock from nose.plugins.attrib import attr -from opinel.utils.console import configPrintException from opinel.utils.credentials import read_creds_from_environment_variables from ScoutSuite.__main__ import * diff --git a/tests/test-utils_cloudwatch.py b/tests/test-utils_cloudwatch.py index 1341d8a46..46d9d44f3 100644 --- a/tests/test-utils_cloudwatch.py +++ b/tests/test-utils_cloudwatch.py @@ -1,5 +1,5 @@ from ScoutSuite.utils_cloudwatch import * -from opinel.utils.console import configPrintException +from core.console import configPrintException # diff --git a/tests/test-utils_sns.py b/tests/test-utils_sns.py index 34e747a1e..7f3d0fce4 100644 --- a/tests/test-utils_sns.py +++ b/tests/test-utils_sns.py @@ -1,5 +1,5 @@ from ScoutSuite.utils_sns import * -from opinel.utils.console import configPrintException +from core.console import configPrintException # From 52ae39d4a7bc80d9e442c3be0c93aca63992a891 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 09:01:59 -0500 Subject: [PATCH 031/154] [Opinel] Moved cli_parser to core --- Scout.py | 2 +- ScoutSuite/__listall__.py | 2 +- ScoutSuite/__main__.py | 2 +- ScoutSuite/{ => core}/cli_parser.py | 0 {core => ScoutSuite/core}/console.py | 0 ScoutSuite/core/exceptions.py | 2 +- ScoutSuite/core/processingengine.py | 2 +- ScoutSuite/core/rule.py | 2 +- ScoutSuite/core/rule_definition.py | 2 +- ScoutSuite/core/ruleset.py | 2 +- ScoutSuite/core/utils.py | 2 +- ScoutSuite/output/html.py | 2 +- ScoutSuite/output/js.py | 2 +- ScoutSuite/output/utils.py | 2 +- ScoutSuite/providers/aws/configs/regions.py | 2 +- ScoutSuite/providers/aws/provider.py | 4 ++-- ScoutSuite/providers/aws/services/ec2.py | 2 +- ScoutSuite/providers/aws/services/iam.py | 2 +- ScoutSuite/providers/aws/services/rds.py | 2 +- ScoutSuite/providers/aws/services/s3.py | 2 +- ScoutSuite/providers/azure/provider.py | 2 +- ScoutSuite/providers/azure/utils.py | 2 +- ScoutSuite/providers/base/configs/base.py | 2 +- ScoutSuite/providers/base/configs/browser.py | 2 +- ScoutSuite/providers/base/configs/services.py | 2 +- ScoutSuite/providers/base/provider.py | 2 +- ScoutSuite/providers/gcp/configs/base.py | 2 +- ScoutSuite/providers/gcp/provider.py | 2 +- ScoutSuite/providers/gcp/services/cloudresourcemanager.py | 2 +- ScoutSuite/providers/gcp/services/cloudsql.py | 2 +- ScoutSuite/providers/gcp/services/cloudstorage.py | 2 +- ScoutSuite/providers/gcp/services/iam.py | 2 +- ScoutSuite/providers/gcp/utils.py | 2 +- opinel/services/cloudformation.py | 2 +- opinel/services/iam.py | 2 +- opinel/services/organizations.py | 2 +- opinel/utils/aws.py | 2 +- opinel/utils/conditions.py | 2 +- opinel/utils/credentials.py | 4 ++-- opinel/utils/fs.py | 2 +- opinel/utils/globals.py | 2 +- opinel/utils/profiles.py | 2 +- opinel/utils/threads.py | 2 +- tests/test-main.py | 2 +- tests/test-rules-processingengine.py | 2 +- tests/test-rules-ruleset.py | 2 +- tests/test-utils_cloudwatch.py | 2 +- tests/test-utils_sns.py | 2 +- 48 files changed, 48 insertions(+), 48 deletions(-) rename ScoutSuite/{ => core}/cli_parser.py (100%) rename {core => ScoutSuite/core}/console.py (100%) diff --git a/Scout.py b/Scout.py index 5edb1c55f..84babab73 100755 --- a/Scout.py +++ b/Scout.py @@ -4,7 +4,7 @@ import sys from ScoutSuite.__main__ import main as scout -from ScoutSuite.cli_parser import ScoutSuiteArgumentParser +from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser if __name__ == "__main__": parser = ScoutSuiteArgumentParser() diff --git a/ScoutSuite/__listall__.py b/ScoutSuite/__listall__.py index e4598fe0b..917027fce 100644 --- a/ScoutSuite/__listall__.py +++ b/ScoutSuite/__listall__.py @@ -7,7 +7,7 @@ try: from opinel.utils.globals import check_requirements - from core.console import configPrintException, printError, printException, printInfo + from ScoutSuite.core.console import configPrintException, printError, printException, printInfo except Exception as e: print('Error: Scout2 depends on the opinel package. Install all the requirements with the following command:') print(' $ pip install -r requirements.txt') diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 5504cb50e..6f9b102de 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -6,7 +6,7 @@ import os import webbrowser -from core.console import configPrintException, printInfo, printDebug +from ScoutSuite.core.console import configPrintException, printInfo, printDebug from opinel.utils.profiles import AWSProfiles from ScoutSuite import AWSCONFIG diff --git a/ScoutSuite/cli_parser.py b/ScoutSuite/core/cli_parser.py similarity index 100% rename from ScoutSuite/cli_parser.py rename to ScoutSuite/core/cli_parser.py diff --git a/core/console.py b/ScoutSuite/core/console.py similarity index 100% rename from core/console.py rename to ScoutSuite/core/console.py diff --git a/ScoutSuite/core/exceptions.py b/ScoutSuite/core/exceptions.py index d76976750..a6e5dd11b 100644 --- a/ScoutSuite/core/exceptions.py +++ b/ScoutSuite/core/exceptions.py @@ -3,7 +3,7 @@ Exceptions handling """ -from core.console import printDebug +from ScoutSuite.core.console import printDebug from ScoutSuite import EXCEPTIONS from ScoutSuite.output.js import JavaScriptReaderWriter diff --git a/ScoutSuite/core/processingengine.py b/ScoutSuite/core/processingengine.py index c26559891..7c320d0ed 100644 --- a/ScoutSuite/core/processingengine.py +++ b/ScoutSuite/core/processingengine.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core.console import printDebug, printError, printException +from ScoutSuite.core.console import printDebug, printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.core.utils import recurse diff --git a/ScoutSuite/core/rule.py b/ScoutSuite/core/rule.py index 6b48b4889..6492d3962 100644 --- a/ScoutSuite/core/rule.py +++ b/ScoutSuite/core/rule.py @@ -4,7 +4,7 @@ import re from opinel.utils.fs import read_ip_ranges -from core.console import printError +from ScoutSuite.core.console import printError from ScoutSuite.utils import format_service_name diff --git a/ScoutSuite/core/rule_definition.py b/ScoutSuite/core/rule_definition.py index 1e92ad6d6..a071c6d0e 100644 --- a/ScoutSuite/core/rule_definition.py +++ b/ScoutSuite/core/rule_definition.py @@ -3,7 +3,7 @@ import json import os -from core.console import printError +from ScoutSuite.core.console import printError class RuleDefinition(object): diff --git a/ScoutSuite/core/ruleset.py b/ScoutSuite/core/ruleset.py index a32c5191a..791be7e5e 100644 --- a/ScoutSuite/core/ruleset.py +++ b/ScoutSuite/core/ruleset.py @@ -4,7 +4,7 @@ import os import tempfile -from core.console import printDebug, printError, prompt_4_yes_no +from ScoutSuite.core.console import printDebug, printError, prompt_4_yes_no from ScoutSuite.core.rule import Rule from ScoutSuite.core.rule_definition import RuleDefinition diff --git a/ScoutSuite/core/utils.py b/ScoutSuite/core/utils.py index 1b25face9..92565a03f 100644 --- a/ScoutSuite/core/utils.py +++ b/ScoutSuite/core/utils.py @@ -8,7 +8,7 @@ import re from opinel.utils.conditions import pass_condition -from core.console import printError, printException +from ScoutSuite.core.console import printError, printException from ScoutSuite.core import condition_operators from ScoutSuite.providers.base.configs.browser import get_value_at diff --git a/ScoutSuite/output/html.py b/ScoutSuite/output/html.py index 9b01fd079..f1c285cec 100644 --- a/ScoutSuite/output/html.py +++ b/ScoutSuite/output/html.py @@ -7,7 +7,7 @@ import zipfile import dateutil.tz -from core.console import printInfo, printException +from ScoutSuite.core.console import printInfo, printException from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, \ GENERATOR_FILE diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index 5df552290..37b4e4488 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -6,7 +6,7 @@ import json import os -from core.console import printException, printInfo +from ScoutSuite.core.console import printException, printInfo from ScoutSuite import DEFAULT_REPORT_DIR from ScoutSuite.output.utils import get_filename, prompt_4_overwrite diff --git a/ScoutSuite/output/utils.py b/ScoutSuite/output/utils.py index 20ef42211..7027fafbd 100644 --- a/ScoutSuite/output/utils.py +++ b/ScoutSuite/output/utils.py @@ -4,7 +4,7 @@ import os import sys -from core.console import printError +from ScoutSuite.core.console import printError from six.moves import input diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 1d4943b23..b2b21bf1a 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -13,7 +13,7 @@ from queue import Queue from opinel.utils.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response -from core.console import printException, printInfo +from ScoutSuite.core.console import printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.base.configs import resource_id_map diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 5a57fa55a..01468d160 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -6,7 +6,7 @@ try: from opinel.utils.aws import get_aws_account_id, get_partition_name - from core.console import configPrintException, printInfo, printDebug + from ScoutSuite.core.console import configPrintException, printInfo, printDebug from opinel.utils.credentials import read_creds from opinel.utils.globals import check_requirements from opinel.utils.profiles import AWSProfiles @@ -16,7 +16,7 @@ print(e) sys.exit(42) -from core.console import printDebug, printError, printException, printInfo +from ScoutSuite.core.console import printDebug, printError, printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.services import AWSServicesConfig diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 62de742d9..3ef3da9c8 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -8,7 +8,7 @@ import base64 from opinel.utils.aws import get_name -from core.console import printException, printInfo +from ScoutSuite.core.console import printException, printInfo from opinel.utils.fs import load_data from opinel.utils.globals import manage_dictionary diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index a60f85d4a..0223b8eda 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -2,7 +2,7 @@ from botocore.exceptions import ClientError from opinel.utils.aws import connect_service, handle_truncated_response -from core.console import printError, printException +from ScoutSuite.core.console import printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index 55c127590..da470ae86 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from opinel.utils.aws import handle_truncated_response -from core.console import printError, printException +from ScoutSuite.core.console import printError, printException from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 8456a1549..b2924d054 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -8,7 +8,7 @@ from botocore.exceptions import ClientError from opinel.services.s3 import get_s3_bucket_location from opinel.utils.aws import handle_truncated_response -from core.console import printError, printException, printInfo +from ScoutSuite.core.console import printError, printException, printInfo from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/azure/provider.py b/ScoutSuite/providers/azure/provider.py index 5937c48a7..7875e2369 100644 --- a/ScoutSuite/providers/azure/provider.py +++ b/ScoutSuite/providers/azure/provider.py @@ -5,7 +5,7 @@ from getpass import getpass -from core.console import printError, printException +from ScoutSuite.core.console import printError, printException from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.azure.configs.services import AzureServicesConfig diff --git a/ScoutSuite/providers/azure/utils.py b/ScoutSuite/providers/azure/utils.py index 717bf5654..8e3dae2e6 100644 --- a/ScoutSuite/providers/azure/utils.py +++ b/ScoutSuite/providers/azure/utils.py @@ -2,7 +2,7 @@ import re -from core.console import printException +from ScoutSuite.core.console import printException from azure.mgmt.storage import StorageManagementClient from azure.mgmt.monitor import MonitorManagementClient diff --git a/ScoutSuite/providers/base/configs/base.py b/ScoutSuite/providers/base/configs/base.py index cdd62d216..95681f2f2 100644 --- a/ScoutSuite/providers/base/configs/base.py +++ b/ScoutSuite/providers/base/configs/base.py @@ -21,7 +21,7 @@ from ScoutSuite.providers.azure.utils import azure_connect_service from opinel.utils.aws import build_region_list -from core.console import printException, printInfo +from ScoutSuite.core.console import printException, printInfo from ScoutSuite.output.console import FetchStatusLogger from ScoutSuite.utils import format_service_name diff --git a/ScoutSuite/providers/base/configs/browser.py b/ScoutSuite/providers/base/configs/browser.py index 3dee54d71..5a5e9e171 100644 --- a/ScoutSuite/providers/base/configs/browser.py +++ b/ScoutSuite/providers/base/configs/browser.py @@ -2,7 +2,7 @@ import copy -from core.console import printError, printException +from ScoutSuite.core.console import printError, printException ######################################## diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 5a89a9ba2..d6ad4dc4e 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core.console import printError, printException, printDebug +from ScoutSuite.core.console import printError, printException, printDebug from opinel.utils.aws import get_partition_name class BaseServicesConfig(object): diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index a6e6aa78c..ea375383a 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -7,7 +7,7 @@ import sys import copy -from core.console import printException, printInfo +from ScoutSuite.core.console import printException, printInfo from ScoutSuite import __version__ as scout2_version from ScoutSuite.providers.base.configs.browser import get_object_at diff --git a/ScoutSuite/providers/gcp/configs/base.py b/ScoutSuite/providers/gcp/configs/base.py index 3db702c33..f9df42d81 100644 --- a/ScoutSuite/providers/gcp/configs/base.py +++ b/ScoutSuite/providers/gcp/configs/base.py @@ -14,7 +14,7 @@ from google.cloud import container_v1 from googleapiclient.errors import HttpError -from core.console import printException, printError +from ScoutSuite.core.console import printException, printError from ScoutSuite.providers.base.configs.base import BaseConfig from ScoutSuite.providers.gcp.utils import gcp_connect_service diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py index adf6721be..456ded1d8 100644 --- a/ScoutSuite/providers/gcp/provider.py +++ b/ScoutSuite/providers/gcp/provider.py @@ -5,7 +5,7 @@ import google.auth import googleapiclient -from core.console import printError, printException, printInfo +from ScoutSuite.core.console import printError, printException, printInfo from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.gcp.configs.services import GCPServicesConfig diff --git a/ScoutSuite/providers/gcp/services/cloudresourcemanager.py b/ScoutSuite/providers/gcp/services/cloudresourcemanager.py index c1ec32399..f44056b93 100644 --- a/ScoutSuite/providers/gcp/services/cloudresourcemanager.py +++ b/ScoutSuite/providers/gcp/services/cloudresourcemanager.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from core.console import printException +from ScoutSuite.core.console import printException class CloudResourceManager(GCPBaseConfig): diff --git a/ScoutSuite/providers/gcp/services/cloudsql.py b/ScoutSuite/providers/gcp/services/cloudsql.py index bb252e1a8..7ffc2ce27 100644 --- a/ScoutSuite/providers/gcp/services/cloudsql.py +++ b/ScoutSuite/providers/gcp/services/cloudsql.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core.console import printError +from ScoutSuite.core.console import printError from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/services/cloudstorage.py b/ScoutSuite/providers/gcp/services/cloudstorage.py index 72f260783..b558dffed 100644 --- a/ScoutSuite/providers/gcp/services/cloudstorage.py +++ b/ScoutSuite/providers/gcp/services/cloudstorage.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core.console import printError +from ScoutSuite.core.console import printError from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig diff --git a/ScoutSuite/providers/gcp/services/iam.py b/ScoutSuite/providers/gcp/services/iam.py index ddce638fd..be9d91f78 100644 --- a/ScoutSuite/providers/gcp/services/iam.py +++ b/ScoutSuite/providers/gcp/services/iam.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from core.console import printError +from ScoutSuite.core.console import printError from googleapiclient import discovery diff --git a/ScoutSuite/providers/gcp/utils.py b/ScoutSuite/providers/gcp/utils.py index adc81dbcd..0c18a3d92 100644 --- a/ScoutSuite/providers/gcp/utils.py +++ b/ScoutSuite/providers/gcp/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core.console import printException +from ScoutSuite.core.console import printException from google.cloud import storage from google.cloud import logging as stackdriver_logging diff --git a/opinel/services/cloudformation.py b/opinel/services/cloudformation.py index 480a6a2e0..ea1950ad1 100644 --- a/opinel/services/cloudformation.py +++ b/opinel/services/cloudformation.py @@ -5,7 +5,7 @@ import time from opinel.utils.aws import connect_service, handle_truncated_response -from core.console import printDebug, printInfo, printError, printException +from ScoutSuite.core.console import printDebug, printInfo, printError, printException from opinel.utils.fs import read_file from opinel.utils.globals import snake_to_camel diff --git a/opinel/services/iam.py b/opinel/services/iam.py index a59df2d44..33765c995 100644 --- a/opinel/services/iam.py +++ b/opinel/services/iam.py @@ -4,7 +4,7 @@ from opinel.utils.aws import handle_truncated_response from opinel.utils.credentials import generate_password -from core.console import printInfo, printError, printException +from ScoutSuite.core.console import printInfo, printError, printException diff --git a/opinel/services/organizations.py b/opinel/services/organizations.py index 355efbfe2..4578110f8 100644 --- a/opinel/services/organizations.py +++ b/opinel/services/organizations.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from opinel.utils.aws import handle_truncated_response -from core.console import printDebug, printInfo +from ScoutSuite.core.console import printDebug, printInfo def get_organization_account_ids(api_client, exceptions = [], quiet = True): diff --git a/opinel/utils/aws.py b/opinel/utils/aws.py index 705bb61e4..9eac65958 100644 --- a/opinel/utils/aws.py +++ b/opinel/utils/aws.py @@ -5,7 +5,7 @@ from collections import Counter import time -from core.console import printInfo, printException +from ScoutSuite.core.console import printInfo, printException diff --git a/opinel/utils/conditions.py b/opinel/utils/conditions.py index a3cac7fca..9caa2adf9 100644 --- a/opinel/utils/conditions.py +++ b/opinel/utils/conditions.py @@ -6,7 +6,7 @@ import netaddr import re -from core.console import printError +from ScoutSuite.core.console import printError from iampoliciesgonewild import get_actions_from_statement, _expand_wildcard_action diff --git a/opinel/utils/credentials.py b/opinel/utils/credentials.py index 0bb0b151c..ea4ae2b81 100644 --- a/opinel/utils/credentials.py +++ b/opinel/utils/credentials.py @@ -11,8 +11,8 @@ import requests # TODO: get rid of that and make sure urllib2 validates certs ? import string -from core.console import printException, printError, printInfo -from core.console import prompt_4_mfa_code +from ScoutSuite.core.console import printException, printError, printInfo +from ScoutSuite.core.console import prompt_4_mfa_code from opinel.utils.fs import save_blob_as_json from opinel.utils.aws import connect_service diff --git a/opinel/utils/fs.py b/opinel/utils/fs.py index 7978a8ec4..9e804c6a7 100644 --- a/opinel/utils/fs.py +++ b/opinel/utils/fs.py @@ -6,7 +6,7 @@ import os import yaml -from core.console import printError, printException, prompt_4_overwrite +from ScoutSuite.core.console import printError, printException, prompt_4_overwrite from opinel.utils.conditions import pass_condition diff --git a/opinel/utils/globals.py b/opinel/utils/globals.py index 34072085c..0ba1ad6a8 100644 --- a/opinel/utils/globals.py +++ b/opinel/utils/globals.py @@ -6,7 +6,7 @@ import re from opinel import __version__ as OPINEL_VERSION -from core.console import printError +from ScoutSuite.core.console import printError ######################################## diff --git a/opinel/utils/profiles.py b/opinel/utils/profiles.py index 081337f6b..3450f1f59 100644 --- a/opinel/utils/profiles.py +++ b/opinel/utils/profiles.py @@ -4,7 +4,7 @@ import re from opinel.utils.aws import get_aws_account_id -from core.console import printDebug +from ScoutSuite.core.console import printDebug from opinel.utils.credentials import read_creds aws_dir = os.path.join(os.path.expanduser('~'), '.aws') diff --git a/opinel/utils/threads.py b/opinel/utils/threads.py index d78742a54..3eaa7187e 100644 --- a/opinel/utils/threads.py +++ b/opinel/utils/threads.py @@ -8,7 +8,7 @@ # Python3 from queue import Queue -from core.console import printException +from ScoutSuite.core.console import printException diff --git a/tests/test-main.py b/tests/test-main.py index b72a35218..209ec0988 100644 --- a/tests/test-main.py +++ b/tests/test-main.py @@ -3,7 +3,7 @@ from mock import MagicMock, patch from ScoutSuite.__main__ import main -from ScoutSuite.cli_parser import ScoutSuiteArgumentParser +from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser class TestMainClass(TestCase): diff --git a/tests/test-rules-processingengine.py b/tests/test-rules-processingengine.py index f93da05dd..e8d3eab12 100644 --- a/tests/test-rules-processingengine.py +++ b/tests/test-rules-processingengine.py @@ -4,7 +4,7 @@ import os import tempfile -from core.console import configPrintException, printError +from ScoutSuite.core.console import configPrintException, printError from ScoutSuite.core.processingengine import ProcessingEngine from ScoutSuite.core.ruleset import Ruleset diff --git a/tests/test-rules-ruleset.py b/tests/test-rules-ruleset.py index a639ef7a5..f336c3074 100644 --- a/tests/test-rules-ruleset.py +++ b/tests/test-rules-ruleset.py @@ -3,7 +3,7 @@ import os from mock import patch -from core.console import configPrintException, printDebug +from ScoutSuite.core.console import configPrintException, printDebug from ScoutSuite.core.rule import Rule from ScoutSuite.core.ruleset import Ruleset diff --git a/tests/test-utils_cloudwatch.py b/tests/test-utils_cloudwatch.py index 46d9d44f3..1bde42713 100644 --- a/tests/test-utils_cloudwatch.py +++ b/tests/test-utils_cloudwatch.py @@ -1,5 +1,5 @@ from ScoutSuite.utils_cloudwatch import * -from core.console import configPrintException +from ScoutSuite.core.console import configPrintException # diff --git a/tests/test-utils_sns.py b/tests/test-utils_sns.py index 7f3d0fce4..881027b85 100644 --- a/tests/test-utils_sns.py +++ b/tests/test-utils_sns.py @@ -1,5 +1,5 @@ from ScoutSuite.utils_sns import * -from core.console import configPrintException +from ScoutSuite.core.console import configPrintException # From f3128aefad894993505829de372a9552d8390155 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Thu, 21 Feb 2019 09:19:43 -0500 Subject: [PATCH 032/154] Moved get_s3_bucket_location into file, since it only had one reference --- ScoutSuite/providers/aws/services/s3.py | 12 +++++++++++- opinel/services/s3.py | 12 ------------ 2 files changed, 11 insertions(+), 13 deletions(-) delete mode 100644 opinel/services/s3.py diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index b2924d054..9033b3373 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -6,7 +6,6 @@ import json from botocore.exceptions import ClientError -from opinel.services.s3 import get_s3_bucket_location from opinel.utils.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException, printInfo from opinel.utils.globals import manage_dictionary @@ -407,3 +406,14 @@ def get_s3_list_region(region): return 'cn-north-1' else: return region + + +def get_s3_bucket_location(s3_client, bucket_name): + """ + + :param s3_client: + :param bucket_name: + :return: + """ + location = s3_client.get_bucket_location(Bucket=bucket_name) + return location['LocationConstraint'] if location['LocationConstraint'] else 'us-east-1' diff --git a/opinel/services/s3.py b/opinel/services/s3.py deleted file mode 100644 index 2688cf2bb..000000000 --- a/opinel/services/s3.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- - - -def get_s3_bucket_location(s3_client, bucket_name): - """ - - :param s3_client: - :param bucket_name: - :return: - """ - location = s3_client.get_bucket_location(Bucket = bucket_name) - return location['LocationConstraint'] if location['LocationConstraint'] else 'us-east-1' From 7717ab5f959964a69a9ed6ba1d63662fbd0cac96 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 09:35:45 -0500 Subject: [PATCH 033/154] [Opinel] Moved small cleanup on console.py --- ScoutSuite/core/console.py | 49 ++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index efd4b1b21..5ec20328a 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -10,14 +10,13 @@ except NameError: pass - ######################################## # Globals ######################################## mfa_serial_format = r'^arn:aws:iam::\d+:mfa/[a-zA-Z0-9\+=,.@_-]+$' re_mfa_serial_format = re.compile(mfa_serial_format) - +re_mfa_code = re.compile(r'^\d{6}\d*$') ######################################## @@ -36,7 +35,6 @@ def configPrintException(enable): verbose_exceptions = enable - ######################################## # Print functions ######################################## @@ -46,11 +44,11 @@ def printDebug(msg): printGeneric(sys.stderr, msg) -def printError(msg, newLine = True): +def printError(msg, newLine=True): printGeneric(sys.stderr, msg, newLine) -def printException(e, debug_only = False): +def printException(e, debug_only=False): global verbose_exceptions if verbose_exceptions: printError(str(traceback.format_exc())) @@ -58,23 +56,22 @@ def printException(e, debug_only = False): printError(str(e)) -def printGeneric(out, msg, newLine = True): +def printGeneric(out, msg, newLine=True): out.write(msg) out.flush() if newLine == True: out.write('\n') -def printInfo(msg, newLine = True ): +def printInfo(msg, newLine=True): printGeneric(sys.stdout, msg, newLine) - ######################################## # Prompt functions ######################################## -def prompt(test_input = None): +def prompt(test_input=None): """ Prompt function that works for Python2 and Python3 @@ -98,7 +95,7 @@ def prompt(test_input = None): return choice -def prompt_4_mfa_code(activate = False, input = None): +def prompt_4_mfa_code(activate=False, input=None): """ Prompt for an MFA code @@ -112,19 +109,16 @@ def prompt_4_mfa_code(activate = False, input = None): prompt_string = 'Enter the next value: ' else: prompt_string = 'Enter your MFA code (or \'q\' to abort): ' - mfa_code = prompt_4_value(prompt_string, no_confirm = True, input = input) - try: - if mfa_code == 'q': - return mfa_code - int(mfa_code) - mfa_code[5] - break - except: + mfa_code = prompt_4_value(prompt_string, no_confirm=True, input=input) + if mfa_code == 'q': + return mfa_code + if not re_mfa_code.match(): printError('Error: your MFA code must only consist of digits and be at least 6 characters long.') + break return mfa_code -def prompt_4_mfa_serial(input = None): +def prompt_4_mfa_serial(input=None): """ Prompt for an MFA serial number @@ -132,10 +126,11 @@ def prompt_4_mfa_serial(input = None): :return: The MFA serial number """ - return prompt_4_value('Enter your MFA serial:', required = False, regex = re_mfa_serial_format, regex_format = mfa_serial_format, input = input) + return prompt_4_value('Enter your MFA serial:', required=False, regex=re_mfa_serial_format, + regex_format=mfa_serial_format, input=input) -def prompt_4_overwrite(filename, force_write, input = None): +def prompt_4_overwrite(filename, force_write, input=None): """ Prompt whether the file should be overwritten @@ -147,10 +142,12 @@ def prompt_4_overwrite(filename, force_write, input = None): """ if not os.path.exists(filename) or force_write: return True - return prompt_4_yes_no('File \'{}\' already exists. Do you want to overwrite it'.format(filename), input = input) + return prompt_4_yes_no('File \'{}\' already exists. Do you want to overwrite it'.format(filename), input=input) -def prompt_4_value(question, choices = None, default = None, display_choices = True, display_indices = False, authorize_list = False, is_question = False, no_confirm = False, required = True, regex = None, regex_format = '', max_laps = 5, input = None, return_index = False): +def prompt_4_value(question, choices=None, default=None, display_choices=True, display_indices=False, + authorize_list=False, is_question=False, no_confirm=False, required=True, regex=None, + regex_format='', max_laps=5, input=None, return_index=False): """ Prompt for a value . . @@ -192,7 +189,7 @@ def prompt_4_value(question, choices = None, default = None, display_choices = T if not choice or choice == '': if default: if no_confirm or prompt_4_yes_no('Use the default value (' + default + ')'): - #return default + # return default choice = default can_return = True elif not required: @@ -220,7 +217,7 @@ def prompt_4_value(question, choices = None, default = None, display_choices = T # Validate against a regex elif regex: if regex.match(choice): - #return choice + # return choice can_return = True else: printError('Error: expected format is: %s' % regex_format) @@ -233,7 +230,7 @@ def prompt_4_value(question, choices = None, default = None, display_choices = T return int(int_choice) if return_index else choice -def prompt_4_yes_no(question, input = None): +def prompt_4_yes_no(question, input=None): """ Prompt for a yes/no or y/n answer . From 2fd8a991e7653300f8026fde2afba214b91318f0 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 16:01:14 -0500 Subject: [PATCH 034/154] [Opinel] Moved conditions.py --- .../utils => ScoutSuite/core}/conditions.py | 116 +++++++++++++++--- ScoutSuite/core/utils.py | 77 +----------- ScoutSuite/providers/aws/provider.py | 16 +-- opinel/utils/fs.py | 4 +- 4 files changed, 103 insertions(+), 110 deletions(-) rename {opinel/utils => ScoutSuite/core}/conditions.py (59%) diff --git a/opinel/utils/conditions.py b/ScoutSuite/core/conditions.py similarity index 59% rename from opinel/utils/conditions.py rename to ScoutSuite/core/conditions.py index 9caa2adf9..782572c46 100644 --- a/opinel/utils/conditions.py +++ b/ScoutSuite/core/conditions.py @@ -6,10 +6,64 @@ import netaddr import re -from ScoutSuite.core.console import printError - from iampoliciesgonewild import get_actions_from_statement, _expand_wildcard_action +from ScoutSuite.core.console import printError, printException +from ScoutSuite.core import condition_operators + +re_get_value_at = re.compile(r'_GET_VALUE_AT_\((.*?)\)') +re_nested_get_value_at = re.compile(r'_GET_VALUE_AT_\(.*') + + +def pass_conditions(all_info, current_path, conditions, unknown_as_pass_condition=False): + """ + Check that all conditions are passed for the current path. + + :param all_info: All of the services' data + :param current_path: The value of the `path` variable defined in the finding file + :param conditions: The conditions to check as defined in the finding file + :param unknown_as_pass_condition: Consider an undetermined condition as passed + :return: + """ + + # Imported here temporarly to fix cyclic dependency + from ScoutSuite.providers.base.configs.browser import get_value_at + + if len(conditions) == 0: + return True + condition_operator = conditions.pop(0) + for condition in conditions: + if condition[0] in condition_operators: + res = pass_conditions(all_info, current_path, condition, unknown_as_pass_condition) + else: + # Conditions are formed as "path to value", "type of test", "value(s) for test" + path_to_value, test_name, test_values = condition + path_to_value = fix_path_string(all_info, current_path, path_to_value) + target_obj = get_value_at(all_info, current_path, path_to_value) + if type(test_values) != list: + dynamic_value = re_get_value_at.match(test_values) + if dynamic_value: + test_values = get_value_at(all_info, current_path, dynamic_value.groups()[0], True) + try: + res = _pass_condition(target_obj, test_name, test_values) + except Exception as e: + res = True if unknown_as_pass_condition else False + printError('Unable to process testcase \'%s\' on value \'%s\', interpreted as %s.' % ( + test_name, str(target_obj), res)) + printException(e, True) + # Quick exit and + false + if condition_operator == 'and' and not res: + return False + # Quick exit or + true + if condition_operator == 'or' and res: + return True + # Still here ? + # or -> false + # and -> true + if condition_operator == 'or': + return False + else: + return True def __prepare_age_test(a, b): @@ -31,7 +85,7 @@ def __prepare_age_test(a, b): return age, number -def pass_condition(b, test, a): +def _pass_condition(b, test, a): """ Generic test function used by Scout2 / AWS recipes . @@ -51,7 +105,7 @@ def pass_condition(b, test, a): b = str(b) result = (a == b) elif test == 'notEqual': - result = (not pass_condition(b, 'equal', a)) + result = (not _pass_condition(b, 'equal', a)) # More/Less tests elif test == 'lessThan': @@ -67,11 +121,11 @@ def pass_condition(b, test, a): elif test == 'empty': result = ((type(b) == dict and b == {}) or (type(b) == list and b == []) or (type(b) == list and b == [None])) elif test == 'notEmpty': - result = (not pass_condition(b, 'empty', 'a')) + result = (not _pass_condition(b, 'empty', 'a')) elif test == 'null': - result = ((b == None) or (type(b) == str and b == 'None')) + result = ((b is None) or (type(b) == str and b == 'None')) elif test == 'notNull': - result = (not pass_condition(b, 'null', a)) + result = (not _pass_condition(b, 'null', a)) # Boolean tests elif test == 'true': @@ -97,9 +151,9 @@ def pass_condition(b, test, a): elif test == 'containAtLeastOneOf': result = False if not type(b) == list: - b = [ b ] + b = [b] if not type(a) == list: - a = [ a ] + a = [a] for c in b: if type(c): c = str(c) @@ -109,9 +163,9 @@ def pass_condition(b, test, a): elif test == 'containAtLeastOneDifferentFrom': result = False if not type(b) == list: - b = [ b ] + b = [b] if not type(a) == list: - a = [ a ] + a = [a] for c in b: if c != None and c != '' and c not in a: result = True @@ -119,9 +173,9 @@ def pass_condition(b, test, a): elif test == 'containNoneOf': result = True if not type(b) == list: - b = [ b ] + b = [b] if not type(a) == list: - a = [ a ] + a = [a] for c in b: if c in a: result = False @@ -130,14 +184,14 @@ def pass_condition(b, test, a): # Regex tests elif test == 'match': if type(a) != list: - a = [ a ] + a = [a] b = str(b) for c in a: - if re.match(c, b) != None: + if not re.match(c, b): result = True break elif test == 'notMatch': - result = (not pass_condition(b, 'match', a)) + result = (not _pass_condition(b, 'match', a)) # Date tests elif test == 'priorToDate': @@ -156,14 +210,14 @@ def pass_condition(b, test, a): result = False grant = netaddr.IPNetwork(b) if type(a) != list: - a = [ a ] + a = [a] for c in a: known_subnet = netaddr.IPNetwork(c) if grant in known_subnet: result = True break elif test == 'notInSubnets': - result = (not pass_condition(b, 'inSubnets', a)) + result = (not _pass_condition(b, 'inSubnets', a)) # Policy statement tests elif test == 'containAction': @@ -177,13 +231,13 @@ def pass_condition(b, test, a): result = True break elif test == 'notContainAction': - result = (not pass_condition(b, 'containAction', a)) + result = (not _pass_condition(b, 'containAction', a)) elif test == 'containAtLeastOneAction': result = False if type(b) != dict: b = json.loads(b) if type(a) != list: - a = [ a ] + a = [a] actions = get_actions_from_statement(b) for c in a: if c.lower() in actions: @@ -214,3 +268,25 @@ def pass_condition(b, test, a): raise Exception return result + + +def fix_path_string(all_info, current_path, path_to_value): + # handle nested _GET_VALUE_AT_... + + # Imported here temporarly to fix cyclic dependency + from ScoutSuite.providers.base.configs.browser import get_value_at + + while True: + dynamic_path = re_get_value_at.findall(path_to_value) + if len(dynamic_path) == 0: + break + for dp in dynamic_path: + tmp = dp + while True: + nested = re_nested_get_value_at.findall(tmp) + if len(nested) == 0: + break + tmp = nested[0].replace('_GET_VALUE_AT_(', '', 1) + dv = get_value_at(all_info, current_path, tmp) + path_to_value = path_to_value.replace('_GET_VALUE_AT_(%s)' % tmp, dv) + return path_to_value diff --git a/ScoutSuite/core/utils.py b/ScoutSuite/core/utils.py index 92565a03f..f0e8de441 100644 --- a/ScoutSuite/core/utils.py +++ b/ScoutSuite/core/utils.py @@ -5,34 +5,9 @@ from six import string_types import copy -import re -from opinel.utils.conditions import pass_condition -from ScoutSuite.core.console import printError, printException - -from ScoutSuite.core import condition_operators -from ScoutSuite.providers.base.configs.browser import get_value_at - -re_get_value_at = re.compile(r'_GET_VALUE_AT_\((.*?)\)') -re_nested_get_value_at = re.compile(r'_GET_VALUE_AT_\(.*') - - -def fix_path_string(all_info, current_path, path_to_value): - # handle nested _GET_VALUE_AT_... - while True: - dynamic_path = re_get_value_at.findall(path_to_value) - if len(dynamic_path) == 0: - break - for dp in dynamic_path: - tmp = dp - while True: - nested = re_nested_get_value_at.findall(tmp) - if len(nested) == 0: - break - tmp = nested[0].replace('_GET_VALUE_AT_(', '', 1) - dv = get_value_at(all_info, current_path, tmp) - path_to_value = path_to_value.replace('_GET_VALUE_AT_(%s)' % tmp, dv) - return path_to_value +from ScoutSuite.core.console import printError +from ScoutSuite.core.conditions import pass_conditions, fix_path_string def recurse(all_info, current_info, target_path, current_path, config, add_suffix=False): @@ -106,51 +81,3 @@ def recurse(all_info, current_info, target_path, current_path, config, add_suffi printError('Entry target path: %s' % str(dbg_target_path)) raise Exception return results - - -def pass_conditions(all_info, current_path, conditions, unknown_as_pass_condition=False): - """ - Check that all conditions are passed for the current path. - - :param all_info: All of the services' data - :param current_path: The value of the `path` variable defined in the finding file - :param conditions: The conditions to check as defined in the finding file - :param unknown_as_pass_condition: Consider an undetermined condition as passed - :return: - """ - result = False - if len(conditions) == 0: - return True - condition_operator = conditions.pop(0) - for condition in conditions: - if condition[0] in condition_operators: - res = pass_conditions(all_info, current_path, condition, unknown_as_pass_condition) - else: - # Conditions are formed as "path to value", "type of test", "value(s) for test" - path_to_value, test_name, test_values = condition - path_to_value = fix_path_string(all_info, current_path, path_to_value) - target_obj = get_value_at(all_info, current_path, path_to_value) - if type(test_values) != list: - dynamic_value = re_get_value_at.match(test_values) - if dynamic_value: - test_values = get_value_at(all_info, current_path, dynamic_value.groups()[0], True) - try: - res = pass_condition(target_obj, test_name, test_values) - except Exception as e: - res = True if unknown_as_pass_condition else False - printError('Unable to process testcase \'%s\' on value \'%s\', interpreted as %s.' % ( - test_name, str(target_obj), res)) - printException(e, True) - # Quick exit and + false - if condition_operator == 'and' and not res: - return False - # Quick exit or + true - if condition_operator == 'or' and res: - return True - # Still here ? - # or -> false - # and -> true - if condition_operator == 'or': - return False - else: - return True diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 01468d160..f06280754 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -2,19 +2,9 @@ import copy import os -import sys - -try: - from opinel.utils.aws import get_aws_account_id, get_partition_name - from ScoutSuite.core.console import configPrintException, printInfo, printDebug - from opinel.utils.credentials import read_creds - from opinel.utils.globals import check_requirements - from opinel.utils.profiles import AWSProfiles -except Exception as e: - print('Error: Scout2 depends on the opinel package. Install all the requirements with the following command:') - print(' $ pip install -r requirements.txt') - print(e) - sys.exit(42) + +from opinel.utils.aws import get_aws_account_id +from opinel.utils.credentials import read_creds from ScoutSuite.core.console import printDebug, printError, printException, printInfo from opinel.utils.globals import manage_dictionary diff --git a/opinel/utils/fs.py b/opinel/utils/fs.py index 9e804c6a7..d03a949f2 100644 --- a/opinel/utils/fs.py +++ b/opinel/utils/fs.py @@ -7,7 +7,7 @@ import yaml from ScoutSuite.core.console import printError, printException, prompt_4_overwrite -from opinel.utils.conditions import pass_condition +from ScoutSuite.core.conditions import _pass_condition @@ -82,7 +82,7 @@ def read_ip_ranges(filename, local_file = True, ip_only = False, conditions = [] for condition in conditions: if type(condition) != list or len(condition) < 3: continue - condition_passed = pass_condition(d[condition[0]], condition[1], condition[2]) + condition_passed = _pass_condition(d[condition[0]], condition[1], condition[2]) if not condition_passed: break if condition_passed: From 6aadea4a09043256abd01c0e3ece820f10443e3a Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 16:06:30 -0500 Subject: [PATCH 035/154] [Opinel] Moved threads.py --- {opinel/utils => ScoutSuite/core}/threads.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) rename {opinel/utils => ScoutSuite/core}/threads.py (84%) diff --git a/opinel/utils/threads.py b/ScoutSuite/core/threads.py similarity index 84% rename from opinel/utils/threads.py rename to ScoutSuite/core/threads.py index 3eaa7187e..5bee3fa8f 100644 --- a/opinel/utils/threads.py +++ b/ScoutSuite/core/threads.py @@ -1,18 +1,12 @@ # -*- coding: utf-8 -*- from threading import Thread -try: - # Python2 - from Queue import Queue -except ImportError: - # Python3 - from queue import Queue +from six.moves.queue import Queue from ScoutSuite.core.console import printException - -def thread_work(targets, function, params = {}, num_threads = 0): +def thread_work(targets, function, params={}, num_threads=0): """ Generic multithreading helper From feea688273127751205f64b7a80480522bf1b08d Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 16:15:29 -0500 Subject: [PATCH 036/154] [Opinel] Deleted globals.py --- ScoutSuite/core/processingengine.py | 2 +- ScoutSuite/output/console.py | 2 +- ScoutSuite/providers/aws/configs/regions.py | 3 +- ScoutSuite/providers/aws/provider.py | 3 +- ScoutSuite/providers/aws/services/ec2.py | 3 +- .../providers/aws/services/elasticache.py | 5 +- ScoutSuite/providers/aws/services/elb.py | 4 +- ScoutSuite/providers/aws/services/elbv2.py | 3 +- ScoutSuite/providers/aws/services/emr.py | 2 +- ScoutSuite/providers/aws/services/iam.py | 1 - ScoutSuite/providers/aws/services/rds.py | 4 +- ScoutSuite/providers/aws/services/redshift.py | 3 +- ScoutSuite/providers/aws/services/s3.py | 2 +- ScoutSuite/providers/aws/services/sns.py | 2 +- ScoutSuite/providers/aws/services/vpc.py | 3 +- ScoutSuite/utils.py | 18 +++ opinel/services/cloudformation.py | 10 +- opinel/utils/globals.py | 105 ------------------ 18 files changed, 39 insertions(+), 136 deletions(-) delete mode 100644 opinel/utils/globals.py diff --git a/ScoutSuite/core/processingengine.py b/ScoutSuite/core/processingengine.py index 7c320d0ed..1ceb66008 100644 --- a/ScoutSuite/core/processingengine.py +++ b/ScoutSuite/core/processingengine.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.core.console import printDebug, printError, printException -from opinel.utils.globals import manage_dictionary +from ScoutSuite.utils import manage_dictionary from ScoutSuite.core.utils import recurse diff --git a/ScoutSuite/output/console.py b/ScoutSuite/output/console.py index cdc037601..06c24d239 100644 --- a/ScoutSuite/output/console.py +++ b/ScoutSuite/output/console.py @@ -4,7 +4,7 @@ import re import sys -from opinel.utils.globals import manage_dictionary +from ScoutSuite.utils import manage_dictionary from ScoutSuite.providers.base.configs.browser import get_value_at diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index b2b21bf1a..80d577080 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -14,12 +14,11 @@ from opinel.utils.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response from ScoutSuite.core.console import printException, printInfo -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.base.configs import resource_id_map from ScoutSuite.providers.base.configs.threads import thread_configs from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import format_service_name, is_throttled +from ScoutSuite.utils import format_service_name, is_throttled, manage_dictionary from ScoutSuite.providers.aws.configs.base import BaseConfig from ScoutSuite.output.console import FetchStatusLogger diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index f06280754..200f6784b 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -7,13 +7,12 @@ from opinel.utils.credentials import read_creds from ScoutSuite.core.console import printDebug, printError, printException, printInfo -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.services import AWSServicesConfig from ScoutSuite.providers.base.configs.browser import combine_paths, get_object_at, get_value_at from ScoutSuite.providers.aws.services.vpc import put_cidr_name from ScoutSuite.providers.base.provider import BaseProvider -from ScoutSuite.utils import ec2_classic +from ScoutSuite.utils import ec2_classic, manage_dictionary class AWSProvider(BaseProvider): diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 3ef3da9c8..8ac8371f0 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -10,11 +10,10 @@ from opinel.utils.aws import get_name from ScoutSuite.core.console import printException, printInfo from opinel.utils.fs import load_data -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.providers.base.configs.browser import get_attribute_at -from ScoutSuite.utils import get_keys, ec2_classic +from ScoutSuite.utils import get_keys, ec2_classic, manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/elasticache.py b/ScoutSuite/providers/aws/services/elasticache.py index a3b7c7183..ffdf6a541 100644 --- a/ScoutSuite/providers/aws/services/elasticache.py +++ b/ScoutSuite/providers/aws/services/elasticache.py @@ -1,11 +1,8 @@ # -*- coding: utf-8 -*- -from opinel.utils.globals import manage_dictionary - from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic - +from ScoutSuite.utils import ec2_classic, manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/elb.py b/ScoutSuite/providers/aws/services/elb.py index b9b1deda7..2d35580b4 100644 --- a/ScoutSuite/providers/aws/services/elb.py +++ b/ScoutSuite/providers/aws/services/elb.py @@ -2,12 +2,10 @@ """ ELB-related classes and functions """ -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic, get_keys - +from ScoutSuite.utils import ec2_classic, get_keys, manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index b45647430..df5dc10ab 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -4,11 +4,10 @@ """ from opinel.utils.aws import handle_truncated_response -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic +from ScoutSuite.utils import ec2_classic, manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/emr.py b/ScoutSuite/providers/aws/services/emr.py index dadee2ac3..c7c1f7e63 100644 --- a/ScoutSuite/providers/aws/services/emr.py +++ b/ScoutSuite/providers/aws/services/emr.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.globals import manage_dictionary +from ScoutSuite.utils import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index 0223b8eda..3ef28fd6c 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -3,7 +3,6 @@ from botocore.exceptions import ClientError from opinel.utils.aws import connect_service, handle_truncated_response from ScoutSuite.core.console import printError, printException -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig from ScoutSuite.utils import * diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index da470ae86..bb988a69e 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -2,12 +2,10 @@ from opinel.utils.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic - +from ScoutSuite.utils import ec2_classic, manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index b4d86a4e7..d8a87610c 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -4,11 +4,10 @@ """ from opinel.utils.aws import handle_truncated_response -from opinel.utils.globals import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic +from ScoutSuite.utils import ec2_classic, manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 9033b3373..6b8688a65 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -8,7 +8,7 @@ from botocore.exceptions import ClientError from opinel.utils.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException, printInfo -from opinel.utils.globals import manage_dictionary +from ScoutSuite.utils import manage_dictionary from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/sns.py b/ScoutSuite/providers/aws/services/sns.py index f4fa3342e..20b066784 100644 --- a/ScoutSuite/providers/aws/services/sns.py +++ b/ScoutSuite/providers/aws/services/sns.py @@ -5,7 +5,7 @@ import json -from opinel.utils.globals import manage_dictionary +from ScoutSuite.utils import manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index 776266147..2c41976aa 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -4,11 +4,10 @@ import copy from opinel.utils.aws import get_name -from opinel.utils.globals import manage_dictionary from opinel.utils.fs import load_data, read_ip_ranges from ScoutSuite.providers.base.configs.browser import get_value_at -from ScoutSuite.utils import ec2_classic, get_keys +from ScoutSuite.utils import ec2_classic, get_keys, manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig from ScoutSuite.providers.aws.configs.vpc import VPCConfig as SingleVPCConfig diff --git a/ScoutSuite/utils.py b/ScoutSuite/utils.py index 24ccc0b20..9f15da47e 100644 --- a/ScoutSuite/utils.py +++ b/ScoutSuite/utils.py @@ -91,3 +91,21 @@ def is_throttled(e): :return: True if it's a throttling exception else False """ return True if (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in [ 'Throttling', 'RequestLimitExceeded', 'ThrottlingException' ]) else False + + +def manage_dictionary(dictionary, key, init, callback=None): + """ + + :param dictionary: + :param key: + :param init: + :param callback: + + :return: + """ + if not str(key) in dictionary: + dictionary[str(key)] = init + manage_dictionary(dictionary, key, init) + if callback: + callback(dictionary[key]) + return dictionary \ No newline at end of file diff --git a/opinel/services/cloudformation.py b/opinel/services/cloudformation.py index ea1950ad1..ca4b1f724 100644 --- a/opinel/services/cloudformation.py +++ b/opinel/services/cloudformation.py @@ -7,10 +7,10 @@ from opinel.utils.aws import connect_service, handle_truncated_response from ScoutSuite.core.console import printDebug, printInfo, printError, printException from opinel.utils.fs import read_file -from opinel.utils.globals import snake_to_camel re_iam_capability = re.compile('.*?AWS::IAM.*?', re.DOTALL | re.MULTILINE) + def create_cloudformation_resource_from_template(api_client, resource_type, name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False, need_on_failure=False): """ @@ -22,7 +22,7 @@ def create_cloudformation_resource_from_template(api_client, resource_type, name :return: """ create = getattr(api_client, 'create_%s' % resource_type) - api_resource_type = snake_to_camel(resource_type) + api_resource_type = _snake_to_camel(resource_type) # Add a timestamps tags.append({'Key': 'OpinelTimestamp', 'Value': str(time.time())}) params = prepare_cloudformation_params(name, template_path, template_parameters, api_resource_type, tags) @@ -274,7 +274,7 @@ def update_cloudformation_resource_from_template(api_client, resource_type, name """ try: update = getattr(api_client, 'update_%s' % resource_type) - api_resource_type = snake_to_camel(resource_type) + api_resource_type = _snake_to_camel(resource_type) # Add a timestamps tags.append({'Key': 'OpinelTimestamp', 'Value': str(time.time())}) params = prepare_cloudformation_params(name, template_path, template_parameters, api_resource_type, tags) @@ -371,3 +371,7 @@ def cloudformation_wait(api_client, resource_type, resource_name, operation_id = printInfo('Status: %s... waiting %d seconds until next check...' % (status, increment)) timer += increment time.sleep(increment) + + +def _snake_to_camel(snake): + return "".join(val.title() for val in snake.split('_')) diff --git a/opinel/utils/globals.py b/opinel/utils/globals.py deleted file mode 100644 index 0ba1ad6a8..000000000 --- a/opinel/utils/globals.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- - -import boto3 -from distutils.version import StrictVersion -import os -import re - -from opinel import __version__ as OPINEL_VERSION -from ScoutSuite.core.console import printError - - -######################################## -# Regex -######################################## - -re_opinel = re.compile(r'^opinel>=([0-9.]+),<([0-9.]+).*') -re_boto3 = re.compile(r'^boto3>=([0-9.]+)(,<([0-9.]+).*)?') - - -######################################## -# Functions -######################################## - -def check_requirements(script_path, requirements_file = None): - """ - Check versions of opinel and boto3 - :param script_path: - :return: - """ - script_dir = os.path.dirname(script_path) - opinel_min_version = opinel_max_version = boto3_min_version = boto3_max_version = None - # Requirements file is either next to the script or in data/requirements - if not requirements_file: - requirements_file = os.path.join(script_dir, 'data/requirements.txt') - if not os.path.isfile(requirements_file): - requirements_file = os.path.join(script_dir, 'requirements.txt') - with open(requirements_file, 'rt') as f: - for requirement in f.readlines(): - opinel_requirements = re_opinel.match(requirement) - if opinel_requirements: - opinel_requirements = opinel_requirements.groups() - opinel_min_version = opinel_requirements[0] - opinel_max_version = opinel_requirements[1] - boto3_requirements = re_boto3.match(requirement) - if boto3_requirements: - boto3_requirements = boto3_requirements.groups() - boto3_min_version = boto3_requirements[0] - boto3_max_version = boto3_requirements[1] - if not check_versions(opinel_min_version, OPINEL_VERSION, opinel_max_version, 'opinel'): - return False - if not check_versions(boto3_min_version, boto3.__version__, boto3_max_version, 'boto3'): - return False - return True - - -def check_versions(min_version, installed_version, max_version, package_name, strict = False): - """ - - :param min_version: - :param installed_version: - :param max_version: - :param package_name: - - :return: - """ - if not min_version: - # If no minimum version was specified, pass - return True - if StrictVersion(installed_version) < StrictVersion(min_version): - printError('Error: the version of %s installed on this system (%s) is too old. ' - 'You need at least version %s to run this tool.' % (package_name, OPINEL_VERSION, min_version)) - return False - if max_version and StrictVersion(installed_version) >= StrictVersion(max_version): - printError('Warning: ther version of %s installed on this system (%s) is too recent; ' - 'you may experience unexpected runtime errors as versions above %s have not been tested.' % - (package_name, installed_version, max_version)) - if strict: - printError('Warning treated as error.') - return False - return True - - -def manage_dictionary(dictionary, key, init, callback = None): - """ - - :param dictionary: - :param key: - :param init: - :param callback: - - :return: - """ - if not str(key) in dictionary: - dictionary[str(key)] = init - manage_dictionary(dictionary, key, init) - if callback: - callback(dictionary[key]) - return dictionary - - -def snake_to_camel(snake): - return "".join(val.title() for val in snake.split('_')) - -def snake_to_words(snake, capitalize = False): - return " ".join(val.title() if capitalize else val for val in snake.split('_')) From 36d986acd8b7545907019e33b7a637451c35b910 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Thu, 21 Feb 2019 16:27:32 -0500 Subject: [PATCH 037/154] [Opinel] Some compatibility improvements --- ScoutSuite/core/console.py | 5 +---- opinel/utils/fs.py | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 5ec20328a..598711d51 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -5,10 +5,7 @@ import sys import traceback -try: - input = raw_input -except NameError: - pass +from six.moves import input ######################################## # Globals diff --git a/opinel/utils/fs.py b/opinel/utils/fs.py index d03a949f2..3c0104198 100644 --- a/opinel/utils/fs.py +++ b/opinel/utils/fs.py @@ -10,7 +10,6 @@ from ScoutSuite.core.conditions import _pass_condition - class CustomJSONEncoder(json.JSONEncoder): """ JSON encoder class @@ -104,7 +103,6 @@ def read_file(file_path, mode = 'rt'): :return: Contents of the file """ - contents = '' with open(file_path, mode) as f: contents = f.read() return contents From 71cb7fa31758f44a18b9cf4e191f1f2b14dbefe7 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 11:17:15 -0500 Subject: [PATCH 038/154] Removed opinel.services --- ScoutSuite/providers/aws/services/s3.py | 5 +- opinel/services/__init__.py | 0 opinel/services/cloudformation.py | 377 ------------------------ opinel/services/cloudtrail.py | 4 - opinel/services/iam.py | 294 ------------------ opinel/services/organizations.py | 49 --- 6 files changed, 3 insertions(+), 726 deletions(-) delete mode 100644 opinel/services/__init__.py delete mode 100644 opinel/services/cloudformation.py delete mode 100644 opinel/services/cloudtrail.py delete mode 100644 opinel/services/iam.py delete mode 100644 opinel/services/organizations.py diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 6b8688a65..f8c7330bb 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -246,9 +246,9 @@ def get_s3_bucket_secure_transport(api_client, bucket_name, bucket_info): if 'Condition' in statement and \ 'Bool' in statement['Condition'] and \ 'aws:SecureTransport' in statement['Condition']['Bool'] and \ - ((statement['Condition']['Bool']['aws:SecureTransport'] == 'false' and \ + ((statement['Condition']['Bool']['aws:SecureTransport'] == 'false' and statement['Effect'] == 'Deny') or - (statement['Condition']['Bool']['aws:SecureTransport'] == 'true' and \ + (statement['Condition']['Bool']['aws:SecureTransport'] == 'true' and statement['Effect'] == 'Allow')): bucket_info['secure_transport_enabled'] = True return True @@ -383,6 +383,7 @@ def get_s3_bucket_keys(api_client, bucket_name, bucket, check_encryption, check_ printException(e) continue if check_acls: + # noinspection PyBroadException try: key['grantees'] = get_s3_acls(api_client, bucket_name, bucket, key_name=key['name']) except Exception as e: diff --git a/opinel/services/__init__.py b/opinel/services/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/opinel/services/cloudformation.py b/opinel/services/cloudformation.py deleted file mode 100644 index ca4b1f724..000000000 --- a/opinel/services/cloudformation.py +++ /dev/null @@ -1,377 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import re -import time - -from opinel.utils.aws import connect_service, handle_truncated_response -from ScoutSuite.core.console import printDebug, printInfo, printError, printException -from opinel.utils.fs import read_file - -re_iam_capability = re.compile('.*?AWS::IAM.*?', re.DOTALL | re.MULTILINE) - - -def create_cloudformation_resource_from_template(api_client, resource_type, name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False, need_on_failure=False): - """ - - :param callback: - :param name: - :param template_path: - :param template_parameters: - :param quiet: - :return: - """ - create = getattr(api_client, 'create_%s' % resource_type) - api_resource_type = _snake_to_camel(resource_type) - # Add a timestamps - tags.append({'Key': 'OpinelTimestamp', 'Value': str(time.time())}) - params = prepare_cloudformation_params(name, template_path, template_parameters, api_resource_type, tags) - if not quiet: - printInfo('Creating the %s %s..' % (resource_type, name)) - response = create(**params) - resource_id_attribute = '%sId' % api_resource_type - resource_id = response[resource_id_attribute] if resource_id_attribute in response else None - operation_id = response['OperationId'] if 'OperationId' in response else None - if wait_for_completion: - cloudformation_wait(api_client, resource_type, name, operation_id) - return resource_id - - -def create_stack(api_client, stack_name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False): - """ - - :param api_client: - :param stack_name: - :param template_path: - :param template_parameters: List of parameter keys and values - :param quiet: - :return: - """ - return create_cloudformation_resource_from_template(api_client, 'stack', stack_name, template_path, template_parameters, tags, quiet, wait_for_completion, need_on_failure=True) - - -def create_or_update_stack(api_client, stack_name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False): - """ - - :param api_client: - :param stack_name: - :param template_path: - :param template_parameters: List of parameter keys and values - :param quiet: - :return: - """ - try: - stack = api_client.describe_stacks(StackName = stack_name) - printInfo('Stack already exists... ', newLine = False) - stack_id = update_stack(api_client, stack_name, template_path, template_parameters, quiet, wait_for_completion) - except Exception as e: - if hasattr(e, 'response') and type(e.response) == dict and 'Error' in e.response and e.response['Error']['Code'] == 'ValidationError': - stack_id = create_stack(api_client, stack_name, template_path, template_parameters, tags, quiet, wait_for_completion) - else: - stack_id = None - printException(e) - return stack_id - - - -def create_stack_set(api_client, stack_set_name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False): - """ - - :param api_client: - :param stack_set_name: - :param template_path: - :param template_parameters: - :param quiet: - :return: - """ - return create_cloudformation_resource_from_template(api_client, 'stack_set', stack_set_name, template_path, template_parameters, tags, quiet, wait_for_completion) - - -def create_or_update_stack_set(api_client, stack_set_name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False): - """ - - :param api_client: - :param stack_name: - :param template_path: - :param template_parameters: List of parameter keys and values - :param quiet: - :return: - """ - operation_id = stack_set_id = None - try: - stack_set = api_client.describe_stack_set(StackSetName = stack_set_name) - printInfo('Stack set already exists... ', newLine = False) - operation_id = update_stack_set(api_client, stack_set_name, template_path, template_parameters, quiet, wait_for_completion) - except Exception as e: - if hasattr(e, 'response') and type(e.response) == dict and 'Error' in e.response and e.response['Error']['Code'] == 'StackSetNotFoundException': - stack_set_id = create_stack_set(api_client, stack_set_name, template_path, template_parameters, tags, quiet, wait_for_completion) - else: - printException(e) - return (stack_set_id, operation_id) - - -def create_stack_instances(api_client, stack_set_name, account_ids, regions, quiet=False): - """ - - :param api_client: - :param stack_set_name: - :param account_ids: - :param regions: - :return: - """ - operation_preferences = {'FailureTolerancePercentage': 100, - 'MaxConcurrentPercentage': 100 - } - if not quiet: - printInfo('Creating stack instances in %d regions and %d accounts...' % (len(regions), len(account_ids))) - printDebug(' %s' % ', '.join(regions)) - response = api_client.create_stack_instances(StackSetName=stack_set_name, Accounts=account_ids, Regions=regions, OperationPreferences=operation_preferences) - if not quiet: - printInfo('Successfully started operation Id %s' % response['OperationId']) - return response['OperationId'] - - -def delete_stack_set(api_client, stack_set_name, timeout = 60 * 5): - """ - """ - printDebug('Deleting stack set %s' % stack_set_name) - # Check for instances - stack_instances = handle_truncated_response(api_client.list_stack_instances, {'StackSetName': stack_set_name}, ['Summaries'])['Summaries'] - account_ids = [] - regions = [] - if len(stack_instances) > 0: - for si in stack_instances: - if si['Account'] not in account_ids: - account_ids.append(si['Account']) - if si['Region'] not in regions: - regions.append(si['Region']) - operation_id = api_client.delete_stack_instances(StackSetName = stack_set_name, Accounts = account_ids, Regions = regions, RetainStacks = False)['OperationId'] - wait_for_operation(api_client, stack_set_name, operation_id) - api_client.delete_stack_set(StackSetName = stack_set_name) - - -def get_stackset_ready_accounts(credentials, account_ids, quiet=True): - """ - Verify which AWS accounts have been configured for CloudFormation stack set by attempting to assume the stack set execution role - - :param credentials: AWS credentials to use when calling sts:assumerole - :param org_account_ids: List of AWS accounts to check for Stackset configuration - - :return: List of account IDs in which assuming the stackset execution role worked - """ - api_client = connect_service('sts', credentials, silent=True) - configured_account_ids = [] - for account_id in account_ids: - try: - role_arn = 'arn:aws:iam::%s:role/AWSCloudFormationStackSetExecutionRole' % account_id - api_client.assume_role(RoleArn=role_arn, RoleSessionName='opinel-get_stackset_ready_accounts') - configured_account_ids.append(account_id) - except Exception as e: - pass - - if len(configured_account_ids) != len(account_ids) and not quiet: - printInfo('Only %d of these accounts have the necessary stack set execution role:' % len(configured_account_ids)) - printDebug(str(configured_account_ids)) - return configured_account_ids - - -def make_awsrecipes_stack_name(template_path): - """ - - :param template_path: - :return: - """ - return make_prefixed_stack_name('AWSRecipes', template_path) - - -def make_opinel_stack_name(template_path): - """ - - :param template_path:" - :return: - """ - return make_prefixed_stack_name('Opinel', template_path) - - -def make_prefixed_stack_name(prefix, template_path): - """ - - :param prefix: - :param template_path: - """ - parts = os.path.basename(template_path).split('-') - parts = parts if len(parts) == 1 else parts[:-1] - return ('%s-%s' % (prefix, '-'.join(parts))).split('.')[0] - - -def prepare_cloudformation_params(stack_name, template_path, template_parameters, resource_type, tags=[], need_on_failure=False): - """ - - :param api_client: - :param stack_name: - :param template_path: - :param template_parameters: List of parameter keys and values - :param quiet: - :return: - """ - printDebug('Reading CloudFormation template from %s' % template_path) - template_body = read_file(template_path) - params = {} - params['%sName' % resource_type] = stack_name - params['TemplateBody'] = template_body - if len(template_parameters): - params['Parameters'] = [] - it = iter(template_parameters) - for param in it: - printError('Param:: %s' % param) - params['Parameters'].append({'ParameterKey': param,'ParameterValue': next(it)}) - - if len(tags): - params['Tags'] = tags - if re_iam_capability.match(template_body): - params['Capabilities'] = [ - 'CAPABILITY_NAMED_IAM'] - if need_on_failure: - params['OnFailure'] = 'ROLLBACK' - return params - - -def update_stack(api_client, stack_name, template_path, template_parameters = [], quiet = False, wait_for_completion = False): - """ - - :param api_client: - :param stack_name: - :param template_path: - :param template_parameters: List of parameter keys and values - :param quiet: - :return: - """ - update_cloudformation_resource_from_template(api_client, 'stack', stack_name, template_path, template_parameters, quiet = quiet, wait_for_completion = wait_for_completion) - - -def update_stack_set(api_client, stack_set_name, template_path, template_parameters=[], quiet=False, wait_for_completion = False): - """ - - :param api_client: - :param stack_set_name: - :param template_path: - :param template_parameters: - :param quiet: - :return: - """ - return update_cloudformation_resource_from_template(api_client, 'stack_set', stack_set_name, template_path, template_parameters, [], quiet, wait_for_completion) - - -def update_cloudformation_resource_from_template(api_client, resource_type, name, template_path, template_parameters=[], tags=[], quiet=False, wait_for_completion = False): - """ - - :param callback: - :param name: - :param template_path: - :param template_parameters: - :param quiet: - :return: - """ - try: - update = getattr(api_client, 'update_%s' % resource_type) - api_resource_type = _snake_to_camel(resource_type) - # Add a timestamps - tags.append({'Key': 'OpinelTimestamp', 'Value': str(time.time())}) - params = prepare_cloudformation_params(name, template_path, template_parameters, api_resource_type, tags) - if not quiet: - printInfo('Updating the %s...' % resource_type, newLine=False) - response = update(**params) - operation_id = response['OperationId'] if resource_type == 'stack_set' else None - if wait_for_completion: - cloudformation_wait(api_client, resource_type, name, operation_id) - - except Exception as e: - if api_resource_type == 'Stack' and hasattr(e, 'response') and type(e.response == dict) and e.response['Error']['Code'] == 'ValidationError' and e.response['Error']['Message'] == 'No updates are to be performed.': - printInfo(' Already up to date.') - else: - printException(e) - printError(' Failed.') - - -def wait_for_operation(api_client, stack_set_name, operation_id, timeout = 5 * 60, increment = 5): - printDebug('Waiting for operation %s on stack set %s...' % (operation_id, stack_set_name)) - timer = 0 - status = '' - while True: - if timer >= timeout: - printError('Timed out.') - break - info = api_client.describe_stack_set_operation(StackSetName = stack_set_name, OperationId = operation_id) - status = info['StackSetOperation']['Status'] - if status not in ['RUNNING', 'STOPPING']: - break - printError('Operation status is \'%s\'... waiting %d seconds until next check...' % (status, increment)) - time.sleep(increment) - timer += increment - return 'Operation %s is %s' % (operation_id, status) - - -def wait_for_stack_set(api_client, stack_set_name, timeout = 60, increment = 5): - printDebug('Waiting for stack set %s to be ready...' % stack_set_name) - timer = 0 - while True: - if timer >= timeout: - printError('Timed out.') - break - printError('Checking the stack set\'s status...') - time.sleep(increment) - timer += increment - info = api_client.describe_stack_set(StackSetName = stack_set_name) - if info['StackSet']['Status'] == 'ACTIVE': - break - - -def still_running(callback, params, resource_type): - rc = True - response = callback(**params) - if resource_type == 'stack': - status = response['Stacks'][0]['StackStatus'] - if status.endswith('_COMPLETE') or status.endswith('_FAILED'): - rc = False - elif resource_type == 'stack_set': - status = response['StackSet']['Status'] - if status == 'ACTIVE': - rc = False - elif resource_type == 'operation': - status = response['StackSetOperation']['Status'] - if status != 'RUNNING': - rc = False - return (rc, status) - - -def cloudformation_wait(api_client, resource_type, resource_name, operation_id = None, timeout = 5 * 60, increment = 5): - if resource_type == 'stack': - callback = api_client.describe_stacks - params = {'StackName': resource_name} - elif resource_type == 'stack_set': - params = {'StackSetName': resource_name} - if operation_id: - callback = api_client.describe_stack_set_operation - params['OperationId'] = operation_id - resource_type = 'operation' - else: - callback = api_client.describe_stack_set - else: - printError('Unknown resource type: %s' % resource_type) - return - timer = 0 - while True: - if timer >= timeout: - printError('Timed out.') - break - rc, status = still_running(callback, params, resource_type) - if rc == False: - printInfo('Status: %s' % status) - break - printInfo('Status: %s... waiting %d seconds until next check...' % (status, increment)) - timer += increment - time.sleep(increment) - - -def _snake_to_camel(snake): - return "".join(val.title() for val in snake.split('_')) diff --git a/opinel/services/cloudtrail.py b/opinel/services/cloudtrail.py deleted file mode 100644 index 646802d96..000000000 --- a/opinel/services/cloudtrail.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- - -def get_trails(api_client): - return api_client.describe_trails()['trailList'] diff --git a/opinel/services/iam.py b/opinel/services/iam.py deleted file mode 100644 index 33765c995..000000000 --- a/opinel/services/iam.py +++ /dev/null @@ -1,294 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from opinel.utils.aws import handle_truncated_response -from opinel.utils.credentials import generate_password -from ScoutSuite.core.console import printInfo, printError, printException - - - -def add_user_to_group(iam_client, user, group, quiet = False): - """ - Add an IAM user to an IAM group - - :param iam_client: - :param group: - :param user: - :param user_info: - :param dry_run: - :return: - """ - if not quiet: - printInfo('Adding user to group %s...' % group) - iam_client.add_user_to_group(GroupName = group, UserName = user) - - -def create_groups(iam_client, groups): - """ - Create a number of IAM group, silently handling exceptions when entity already exists - . - :param iam_client: AWS API client for IAM - :param groups: Name of IAM groups to be created. - - :return: None - """ - groups_data = [] - if type(groups) != list: - groups = [ groups ] - for group in groups: - errors = [] - try: - printInfo('Creating group %s...' % group) - iam_client.create_group(GroupName = group) - except Exception as e: - if e.response['Error']['Code'] != 'EntityAlreadyExists': - printException(e) - errors.append('iam:creategroup') - groups_data.append({'groupname': group, 'errors': errors}) - return groups_data - - -def create_user(iam_client, user, groups = [], with_password= False, with_mfa = False, with_access_key = False, require_password_reset = True): - """ - - :param iam_client: AWS API client for IAM - :param user: Name of the user to create - :param groups: Name of the IAM groups to add the user to - :param with_password: Boolean indicating whether creation of a password should be done - :param with_mfa: Boolean indicating whether creation of an MFA device should be done - :param with_access_key: Boolean indicating whether creation of an API access key should be done - :param require_password_reset: Boolean indicating whether users should reset their password after first login - :return: - """ - user_data = {'username': user, 'errors': []} - printInfo('Creating user %s...' % user) - try: - iam_client.create_user(UserName = user) - except Exception as e: - user_data['errors'].append('iam:createuser') - return user_data - # Add user to groups - if type(groups) != list: - groups = [ groups ] - for group in groups: - try: - add_user_to_group(iam_client, user, group) - except Exception as e: - printException(e) - user_data['errors'].append('iam:addusertogroup - %s' % group) - # Generate password - if with_password: - try: - printInfo('Creating a login profile...') - user_data['password'] = generate_password() - iam_client.create_login_profile(UserName = user, Password = user_data['password'] , PasswordResetRequired = require_password_reset) - except Exception as e: - printException(e) - user_data['errors'].append('iam:createloginprofile') - # Enable MFA - if False and with_mfa: - printInfo('Enabling MFA...') - serial = '' - mfa_code1 = '' - mfa_code2 = '' - # Create an MFA device, Display the QR Code, and activate the MFA device - try: - mfa_serial = False # enable_mfa(iam_client, user, '%s/qrcode.png' % user) - except Exception as e: - return 42 - # Request access key - if with_access_key: - try: - printInfo('Creating an API access key...') - access_key = iam_client.create_access_key(UserName=user)['AccessKey'] - user_data['AccessKeyId'] = access_key['AccessKeyId'] - user_data['SecretAccessKey'] = access_key['SecretAccessKey'] - except Exception as e: - printException(e) - user_data['errors'].append('iam:createaccesskey') - return user_data - - -def delete_user(iam_client, user, mfa_serial = None, keep_user = False, terminated_groups = []): - """ - Delete IAM user - - :param iam_client: - :param user: - :param mfa_serial: - :param keep_user: - :param terminated_groups: - :return: - """ - errors = [] - printInfo('Deleting user %s...' % user) - # Delete access keys - try: - aws_keys = get_access_keys(iam_client, user) - for aws_key in aws_keys: - try: - printInfo('Deleting access key ID %s... ' % aws_key['AccessKeyId'], False) - iam_client.delete_access_key(AccessKeyId = aws_key['AccessKeyId'], UserName = user) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - except Exception as e: - printException(e) - printError('Failed to get access keys for user %s.' % user) - # Deactivate and delete MFA devices - try: - mfa_devices = iam_client.list_mfa_devices(UserName = user)['MFADevices'] - for mfa_device in mfa_devices: - serial = mfa_device['SerialNumber'] - try: - printInfo('Deactivating MFA device %s... ' % serial, False) - iam_client.deactivate_mfa_device(SerialNumber = serial, UserName = user) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - delete_virtual_mfa_device(iam_client, serial) - if mfa_serial: - delete_virtual_mfa_device(iam_client, mfa_serial) - except Exception as e: - printException(e) - printError('Faile to fetch/delete MFA device serial number for user %s.' % user) - errors.append(e.response['Error']['Code']) - # Remove IAM user from groups - try: - groups = iam_client.list_groups_for_user(UserName = user)['Groups'] - for group in groups: - try: - printInfo('Removing from group %s... ' % group['GroupName'], False) - iam_client.remove_user_from_group(GroupName = group['GroupName'], UserName = user) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - except Exception as e: - printException(e) - printError('Failed to fetch IAM groups for user %s.' % user) - errors.append(e.response['Error']['Code']) - # Delete login profile - login_profile = [] - try: - login_profile = iam_client.get_login_profile(UserName = user)['LoginProfile'] - except Exception as e: - pass - try: - if len(login_profile): - printInfo('Deleting login profile... ', False) - iam_client.delete_login_profile(UserName = user) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - # Delete inline policies - try: - printInfo('Deleting inline policies... ', False) - policies = iam_client.list_user_policies(UserName = user) - for policy in policies['PolicyNames']: - iam_client.delete_user_policy(UserName = user, PolicyName = policy) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - # Detach managed policies - try: - printInfo('Detaching managed policies... ', False) - policies = iam_client.list_attached_user_policies(UserName = user) - for policy in policies['AttachedPolicies']: - iam_client.detach_user_policy(UserName = user, PolicyArn = policy['PolicyArn']) - printInfo('Success') - except Exception as e: - printInfo('Failed') - printException(e) - errors.append(e.response['Error']['Code']) - # Delete IAM user - try: - if not keep_user: - iam_client.delete_user(UserName = user) - printInfo('User %s deleted.' % user) - else: - for group in terminated_groups: - add_user_to_group(iam_client, group, user) - except Exception as e: - printException(e) - printError('Failed to delete user.') - errors.append(e.response['Error']['Code']) - pass - return errors - - -def delete_virtual_mfa_device(iam_client, mfa_serial): - """ - Delete a vritual MFA device given its serial number - - :param iam_client: - :param mfa_serial: - :return: - """ - try: - printInfo('Deleting MFA device %s...' % mfa_serial) - iam_client.delete_virtual_mfa_device(SerialNumber = mfa_serial) - except Exception as e: - printException(e) - printError('Failed to delete MFA device %s' % mfa_serial) - pass - -def get_access_keys(iam_client, user_name): - """ - - :param iam_client: - :param user_name: - :return: - """ - keys = handle_truncated_response(iam_client.list_access_keys, {'UserName': user_name}, ['AccessKeyMetadata'])['AccessKeyMetadata'] - return keys - - -def init_group_category_regex(category_groups, category_regex_args): - """ - Initialize and compile regular expression for category groups - - :param category_regex_args: List of string regex - - :return: List of compiled regex - """ - category_regex = [] - authorized_empty_regex = 1 - if len(category_regex_args) and len(category_groups) != len(category_regex_args): - printError('Error: you must provide as many regex as category groups.') - return None - for regex in category_regex_args: - if len(regex) < 1: - if authorized_empty_regex > 0: - category_regex.append(None) - authorized_empty_regex -= 1 - else: - printError('Error: you cannot have more than one empty regex to automatically assign groups to users.') - return None - else: - category_regex.append(re.compile(regex)) - return category_regex - - - -def show_access_keys(iam_client, user_name): - """ - - :param iam_client: - :param user_name: - :return: - """ - keys = get_access_keys(iam_client, user_name) - printInfo('User \'%s\' currently has %s access keys:' % (user_name, len(keys))) - for key in keys: - printInfo('\t%s (%s)' % (key['AccessKeyId'], key['Status'])) diff --git a/opinel/services/organizations.py b/opinel/services/organizations.py deleted file mode 100644 index 4578110f8..000000000 --- a/opinel/services/organizations.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -from opinel.utils.aws import handle_truncated_response -from ScoutSuite.core.console import printDebug, printInfo - - -def get_organization_account_ids(api_client, exceptions = [], quiet = True): - - # List all accounts in the organization - org_accounts = get_organization_accounts(api_client, exceptions, quiet) - return [ account['Id'] for account in org_accounts ] - - -def get_organization_accounts(api_client, exceptions = [], quiet = True): - - # List all accounts in the organization - org_accounts = handle_truncated_response(api_client.list_accounts, {}, ['Accounts'])['Accounts'] - if not quiet: - printInfo('Found %d accounts in the organization.' % len(org_accounts)) - for account in org_accounts: - printDebug(str(account)) - if len(exceptions): - filtered_accounts = [] - for account in org_accounts: - if account['Id'] not in exceptions: - filtered_accounts.append(account) - org_accounts = filtered_accounts - return org_accounts - - -def get_organizational_units(api_client): - ous = [] - roots = api_client.list_roots()['Roots'] - return get_children_organizational_units(api_client, roots) - - -def get_children_organizational_units(api_client, parents): - ous = [] - for parent in parents: - children = handle_truncated_response(api_client.list_organizational_units_for_parent, {'ParentId': parent['Id']}, ['OrganizationalUnits'])['OrganizationalUnits'] - if len(children): - ous += get_children_organizational_units(api_client, children) - else: - ous.append(parent) - return ous - -def list_accounts_for_parent(api_client, parent): - return handle_truncated_response(api_client.list_accounts_for_parent, {'ParentId': parent['Id']}, ['Accounts'])['Accounts'] - From 6cb47b9b52ce0c3593cdc803b9bffac0017c56a4 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 13:13:06 -0500 Subject: [PATCH 039/154] Moved aws.py and fixed warnings in it --- .../utils => ScoutSuite/providers/aws}/aws.py | 34 ++++++++++--------- ScoutSuite/providers/aws/configs/base.py | 2 +- ScoutSuite/providers/aws/configs/regions.py | 2 +- ScoutSuite/providers/aws/provider.py | 2 +- ScoutSuite/providers/aws/services/ec2.py | 2 +- ScoutSuite/providers/aws/services/efs.py | 2 +- ScoutSuite/providers/aws/services/elbv2.py | 2 +- ScoutSuite/providers/aws/services/iam.py | 2 +- ScoutSuite/providers/aws/services/rds.py | 2 +- ScoutSuite/providers/aws/services/redshift.py | 2 +- ScoutSuite/providers/aws/services/route53.py | 2 +- ScoutSuite/providers/aws/services/s3.py | 2 +- ScoutSuite/providers/aws/services/vpc.py | 2 +- ScoutSuite/providers/base/configs/base.py | 4 +-- ScoutSuite/providers/base/configs/services.py | 2 +- opinel/utils/credentials.py | 2 +- opinel/utils/profiles.py | 2 +- requirements.txt | 1 + 18 files changed, 36 insertions(+), 33 deletions(-) rename {opinel/utils => ScoutSuite/providers/aws}/aws.py (76%) diff --git a/opinel/utils/aws.py b/ScoutSuite/providers/aws/aws.py similarity index 76% rename from opinel/utils/aws.py rename to ScoutSuite/providers/aws/aws.py index 9eac65958..52bbc702a 100644 --- a/opinel/utils/aws.py +++ b/ScoutSuite/providers/aws/aws.py @@ -8,8 +8,7 @@ from ScoutSuite.core.console import printInfo, printException - -def build_region_list(service, chosen_regions = [], partition_name = 'aws'): +def build_region_list(service, chosen_regions=None, partition_name='aws'): """ Build the list of target region names @@ -19,16 +18,18 @@ def build_region_list(service, chosen_regions = [], partition_name = 'aws'): :return: """ - service = 'ec2containerservice' if service == 'ecs' else service # Of course things aren't that easy... + if chosen_regions is None: + chosen_regions = [] + service = 'ec2containerservice' if service == 'ecs' else service # Get list of regions from botocore - regions = Session().get_available_regions(service, partition_name = partition_name) + regions = Session().get_available_regions(service, partition_name=partition_name) if len(chosen_regions): return list((Counter(regions) & Counter(chosen_regions)).elements()) else: return regions -def connect_service(service, credentials, region_name = None, config = None, silent = False): +def connect_service(service, credentials, region_name=None, config=None, silent=False): """ Instantiates an AWS API client @@ -42,12 +43,10 @@ def connect_service(service, credentials, region_name = None, config = None, sil """ api_client = None try: - client_params = {} - client_params['service_name'] = service.lower() - session_params = {} - session_params['aws_access_key_id'] = credentials['AccessKeyId'] - session_params['aws_secret_access_key'] = credentials['SecretAccessKey'] - session_params['aws_session_token'] = credentials['SessionToken'] + client_params = {'service_name': service.lower()} + session_params = {'aws_access_key_id': credentials['AccessKeyId'], + 'aws_secret_access_key': credentials['SecretAccessKey'], + 'aws_session_token': credentials['SessionToken']} if region_name: client_params['region_name'] = region_name session_params['region_name'] = region_name @@ -55,10 +54,10 @@ def connect_service(service, credentials, region_name = None, config = None, sil client_params['config'] = config aws_session = boto3.session.Session(**session_params) if not silent: - infoMessage = 'Connecting to AWS %s' % service + info_message = 'Connecting to AWS %s' % service if region_name: - infoMessage = infoMessage + ' in %s' % region_name - printInfo('%s...' % infoMessage) + info_message = info_message + ' in %s' % region_name + printInfo('%s...' % info_message) api_client = aws_session.client(**client_params) except Exception as e: printException(e) @@ -86,7 +85,7 @@ def get_name(src, dst, default_attribute): def get_caller_identity(credentials): - api_client = connect_service('sts', credentials, silent = True) + api_client = connect_service('sts', credentials, silent=True) return api_client.get_caller_identity() @@ -132,6 +131,7 @@ def handle_truncated_response(callback, params, entities): if not marker_found: break except Exception as e: + # noinspection PyTypeChecker if is_throttled(e): time.sleep(1) else: @@ -146,4 +146,6 @@ def is_throttled(e): :param e: Exception raised :return: True if it's a throttling exception else False """ - return True if (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in [ 'Throttling', 'RequestLimitExceeded', 'ThrottlingException', 'TooManyRequestsException' ]) else False + return True if (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in + ['Throttling', 'RequestLimitExceeded', 'ThrottlingException', 'TooManyRequestsException']) \ + else False diff --git a/ScoutSuite/providers/aws/configs/base.py b/ScoutSuite/providers/aws/configs/base.py index 0884cacdf..fcd6f4e4a 100644 --- a/ScoutSuite/providers/aws/configs/base.py +++ b/ScoutSuite/providers/aws/configs/base.py @@ -8,7 +8,7 @@ from ScoutSuite.providers.base.configs.base import BaseConfig -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response class AWSBaseConfig(BaseConfig): diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 80d577080..8e918de9d 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -12,7 +12,7 @@ except ImportError: from queue import Queue -from opinel.utils.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response +from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response from ScoutSuite.core.console import printException, printInfo from ScoutSuite.providers.base.configs import resource_id_map diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 200f6784b..55c5d239a 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -3,7 +3,7 @@ import copy import os -from opinel.utils.aws import get_aws_account_id +from ScoutSuite.providers.aws.aws import get_aws_account_id from opinel.utils.credentials import read_creds from ScoutSuite.core.console import printDebug, printError, printException, printInfo diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 8ac8371f0..ce52dfe23 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -7,7 +7,7 @@ import netaddr import base64 -from opinel.utils.aws import get_name +from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.core.console import printException, printInfo from opinel.utils.fs import load_data diff --git a/ScoutSuite/providers/aws/services/efs.py b/ScoutSuite/providers/aws/services/efs.py index 31ee09dd2..58198baf1 100644 --- a/ScoutSuite/providers/aws/services/efs.py +++ b/ScoutSuite/providers/aws/services/efs.py @@ -2,7 +2,7 @@ """ EFS-related classes and functions """ -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index df5dc10ab..b67c1c5f5 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -3,7 +3,7 @@ ELBv2-related classes and functions """ -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index 3ef28fd6c..dd67370a9 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from botocore.exceptions import ClientError -from opinel.utils.aws import connect_service, handle_truncated_response +from ScoutSuite.providers.aws.aws import connect_service, handle_truncated_response from ScoutSuite.core.console import printError, printException from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index bb988a69e..eeff7459c 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index d8a87610c..772d40edc 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -3,7 +3,7 @@ Redshift-related classes and functions """ -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig diff --git a/ScoutSuite/providers/aws/services/route53.py b/ScoutSuite/providers/aws/services/route53.py index c7d658f44..08d502f1c 100644 --- a/ScoutSuite/providers/aws/services/route53.py +++ b/ScoutSuite/providers/aws/services/route53.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.base import AWSBaseConfig diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index f8c7330bb..da9b1d0e2 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -6,7 +6,7 @@ import json from botocore.exceptions import ClientError -from opinel.utils.aws import handle_truncated_response +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException, printInfo from ScoutSuite.utils import manage_dictionary diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index 2c41976aa..e6cf953e9 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -3,7 +3,7 @@ import netaddr import copy -from opinel.utils.aws import get_name +from ScoutSuite.providers.aws.aws import get_name from opinel.utils.fs import load_data, read_ip_ranges from ScoutSuite.providers.base.configs.browser import get_value_at diff --git a/ScoutSuite/providers/base/configs/base.py b/ScoutSuite/providers/base/configs/base.py index 95681f2f2..99dd70a3b 100644 --- a/ScoutSuite/providers/base/configs/base.py +++ b/ScoutSuite/providers/base/configs/base.py @@ -16,11 +16,11 @@ from ScoutSuite.providers.base.configs.threads import thread_configs # TODO do this better without name conflict -from opinel.utils.aws import connect_service +from ScoutSuite.providers.aws.aws import connect_service from ScoutSuite.providers.gcp.utils import gcp_connect_service from ScoutSuite.providers.azure.utils import azure_connect_service -from opinel.utils.aws import build_region_list +from ScoutSuite.providers.aws.aws import build_region_list from ScoutSuite.core.console import printException, printInfo from ScoutSuite.output.console import FetchStatusLogger diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index d6ad4dc4e..5d19b7317 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.core.console import printError, printException, printDebug -from opinel.utils.aws import get_partition_name +from ScoutSuite.providers.aws.aws import get_partition_name class BaseServicesConfig(object): diff --git a/opinel/utils/credentials.py b/opinel/utils/credentials.py index ea4ae2b81..018caf7e5 100644 --- a/opinel/utils/credentials.py +++ b/opinel/utils/credentials.py @@ -14,7 +14,7 @@ from ScoutSuite.core.console import printException, printError, printInfo from ScoutSuite.core.console import prompt_4_mfa_code from opinel.utils.fs import save_blob_as_json -from opinel.utils.aws import connect_service +from ScoutSuite.providers.aws.aws import connect_service ######################################## diff --git a/opinel/utils/profiles.py b/opinel/utils/profiles.py index 3450f1f59..f4714ce67 100644 --- a/opinel/utils/profiles.py +++ b/opinel/utils/profiles.py @@ -3,7 +3,7 @@ import os import re -from opinel.utils.aws import get_aws_account_id +from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.core.console import printDebug from opinel.utils.credentials import read_creds diff --git a/requirements.txt b/requirements.txt index e4ff9ba7f..4fd46b27c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ netaddr>=0.7.11 # AWS Provider boto3>=1.9.60 +botocore>=1.12.60 # Opinel requests>=2.4.0,<3.0.0 From 568d7b5707f39abdd46f414d27713646ba1aa7f1 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 14:11:02 -0500 Subject: [PATCH 040/154] Solved PEP8 issues in regions.py --- ScoutSuite/providers/aws/configs/regions.py | 40 +++++++++++---------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 8e918de9d..c5650cb3e 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -12,7 +12,8 @@ except ImportError: from queue import Queue -from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response +from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, \ + handle_truncated_response from ScoutSuite.core.console import printException, printInfo from ScoutSuite.providers.base.configs import resource_id_map @@ -72,7 +73,7 @@ def __init__(self, service_metadata=None, thread_config=4): continue params = resource_metadata['params'] if 'params' in resource_metadata else {} ignore_exceptions = True if 'no_exceptions' in resource_metadata and \ - resource_metadata['no_exceptions'] == True else False + resource_metadata['no_exceptions'] else False if not only_first_region: self.targets['other_regions'] += ((resource, resource_metadata['response'], @@ -91,14 +92,13 @@ def init_region_config(self, region): :param region: Name of the region """ - self.regions[region] = self.region_config_class(region_name = region, resource_types = self.resource_types) + self.regions[region] = self.region_config_class(region_name=region, resource_types=self.resource_types) - def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets = None): + def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): """ Fetch all the configuration supported by Scout2 for a given service :param credentials: F - :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. @@ -125,7 +125,8 @@ def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower() # Init regions - regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class + # TODO: move this code within this class + regions = build_region_list(api_service, regions, partition_name) self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) # Threading to fetch & parse resources (queue consumer) @@ -156,7 +157,8 @@ def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets for j in range(self.thread_config['list']): qr.put(None) - def _init_threading(self, function, params=None, num_threads=10): + @staticmethod + def _init_threading(function, params=None, num_threads=10): """ Initialize queue and threads @@ -166,7 +168,8 @@ def _init_threading(self, function, params=None, num_threads=10): :return: """ params = {} if params is None else params - q = Queue(maxsize=0) # TODO: find something appropriate + # TODO: find something appropriate + q = Queue(maxsize=0) for i in range(num_threads): worker = Thread(target=function, args=(q, params)) worker.setDaemon(True) @@ -181,10 +184,10 @@ def _fetch_region(self, q, params): region, targets = q.get() or (None, None) if region: self.init_region_config(region) - api_client = connect_service(params['api_service'], params['credentials'], region, silent = True) + api_client = connect_service(params['api_service'], params['credentials'], region, silent=True) api_clients[region] = api_client # TODO : something here for single_region stuff - self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) # params['targets']) + self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) self.fetchstatuslogger.counts['regions']['fetched'] += 1 except Exception as e: printException(e) @@ -247,12 +250,14 @@ def tweak_params(self, params, credentials): ######################################## +# noinspection PyBroadException class RegionConfig(BaseConfig): """ Base class for ... """ - def __init__(self, region_name, resource_types=None): + def __init__(self, region_name, resource_types=None, **kwargs): + super().__init__(**kwargs) resource_types = {} if resource_types is None else resource_types self.region = region_name self.name = region_name @@ -265,7 +270,7 @@ def __init__(self, region_name, resource_types=None): self.vpc_resource_types = resource_types['vpc'] def fetch_all(self, api_client, fetchstatuslogger, q, targets): - ''' + """ Make all API calls as defined in metadata.json :param api_client: @@ -273,9 +278,9 @@ def fetch_all(self, api_client, fetchstatuslogger, q, targets): :param q: :param targets: :return: - ''' + """ self.fetchstatuslogger = fetchstatuslogger - if targets != None: + if targets is not None: # Ensure targets is a tuple if type(targets) != list and type(targets) != tuple: targets = tuple(targets,) @@ -285,7 +290,7 @@ def fetch_all(self, api_client, fetchstatuslogger, q, targets): self._fetch_targets(api_client, q, target) def _fetch_targets(self, api_client, q, target): - ''' + """ Make an API call defined in metadata.json. Parse the returned object as implemented in the "parse_[object name]" method. @@ -293,7 +298,7 @@ def _fetch_targets(self, api_client, q, target): :param q: :param target: :return: - ''' + """ # Handle & format the target type target_type, response_attribute, list_method_name, list_params, ignore_list_error = target list_method = getattr(api_client, list_method_name) @@ -318,7 +323,7 @@ def _fetch_targets(self, api_client, q, target): # Add to the queue q.put((callback, region, target)) - def store_target(self, global_params, region, target): + def store_target(self, target): target_type = target.pop('scout2_target_type') if 'VpcId' in target: vpc_id = target.pop('VpcId') @@ -330,4 +335,3 @@ def store_target(self, global_params, region, target): target_id = target[resource_id_map[target_type]] get_name(target, target, resource_id_map[target_type]) target_dict[target_id] = target - From 11be22c096ac3c2f12a626112853b47ff51622e9 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 14:11:02 -0500 Subject: [PATCH 041/154] Revert "Solved PEP8 issues in regions.py" This reverts commit 225f0c53f924fe7c1ef782145e2e608ce8c7da0e. --- ScoutSuite/providers/aws/configs/regions.py | 40 ++++++++++----------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index c5650cb3e..8e918de9d 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -12,8 +12,7 @@ except ImportError: from queue import Queue -from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, \ - handle_truncated_response +from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response from ScoutSuite.core.console import printException, printInfo from ScoutSuite.providers.base.configs import resource_id_map @@ -73,7 +72,7 @@ def __init__(self, service_metadata=None, thread_config=4): continue params = resource_metadata['params'] if 'params' in resource_metadata else {} ignore_exceptions = True if 'no_exceptions' in resource_metadata and \ - resource_metadata['no_exceptions'] else False + resource_metadata['no_exceptions'] == True else False if not only_first_region: self.targets['other_regions'] += ((resource, resource_metadata['response'], @@ -92,13 +91,14 @@ def init_region_config(self, region): :param region: Name of the region """ - self.regions[region] = self.region_config_class(region_name=region, resource_types=self.resource_types) + self.regions[region] = self.region_config_class(region_name = region, resource_types = self.resource_types) - def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): + def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets = None): """ Fetch all the configuration supported by Scout2 for a given service :param credentials: F + :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. @@ -125,8 +125,7 @@ def fetch_all(self, credentials, regions=None, partition_name='aws', targets=Non api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower() # Init regions - # TODO: move this code within this class - regions = build_region_list(api_service, regions, partition_name) + regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) # Threading to fetch & parse resources (queue consumer) @@ -157,8 +156,7 @@ def fetch_all(self, credentials, regions=None, partition_name='aws', targets=Non for j in range(self.thread_config['list']): qr.put(None) - @staticmethod - def _init_threading(function, params=None, num_threads=10): + def _init_threading(self, function, params=None, num_threads=10): """ Initialize queue and threads @@ -168,8 +166,7 @@ def _init_threading(function, params=None, num_threads=10): :return: """ params = {} if params is None else params - # TODO: find something appropriate - q = Queue(maxsize=0) + q = Queue(maxsize=0) # TODO: find something appropriate for i in range(num_threads): worker = Thread(target=function, args=(q, params)) worker.setDaemon(True) @@ -184,10 +181,10 @@ def _fetch_region(self, q, params): region, targets = q.get() or (None, None) if region: self.init_region_config(region) - api_client = connect_service(params['api_service'], params['credentials'], region, silent=True) + api_client = connect_service(params['api_service'], params['credentials'], region, silent = True) api_clients[region] = api_client # TODO : something here for single_region stuff - self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) + self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) # params['targets']) self.fetchstatuslogger.counts['regions']['fetched'] += 1 except Exception as e: printException(e) @@ -250,14 +247,12 @@ def tweak_params(self, params, credentials): ######################################## -# noinspection PyBroadException class RegionConfig(BaseConfig): """ Base class for ... """ - def __init__(self, region_name, resource_types=None, **kwargs): - super().__init__(**kwargs) + def __init__(self, region_name, resource_types=None): resource_types = {} if resource_types is None else resource_types self.region = region_name self.name = region_name @@ -270,7 +265,7 @@ def __init__(self, region_name, resource_types=None, **kwargs): self.vpc_resource_types = resource_types['vpc'] def fetch_all(self, api_client, fetchstatuslogger, q, targets): - """ + ''' Make all API calls as defined in metadata.json :param api_client: @@ -278,9 +273,9 @@ def fetch_all(self, api_client, fetchstatuslogger, q, targets): :param q: :param targets: :return: - """ + ''' self.fetchstatuslogger = fetchstatuslogger - if targets is not None: + if targets != None: # Ensure targets is a tuple if type(targets) != list and type(targets) != tuple: targets = tuple(targets,) @@ -290,7 +285,7 @@ def fetch_all(self, api_client, fetchstatuslogger, q, targets): self._fetch_targets(api_client, q, target) def _fetch_targets(self, api_client, q, target): - """ + ''' Make an API call defined in metadata.json. Parse the returned object as implemented in the "parse_[object name]" method. @@ -298,7 +293,7 @@ def _fetch_targets(self, api_client, q, target): :param q: :param target: :return: - """ + ''' # Handle & format the target type target_type, response_attribute, list_method_name, list_params, ignore_list_error = target list_method = getattr(api_client, list_method_name) @@ -323,7 +318,7 @@ def _fetch_targets(self, api_client, q, target): # Add to the queue q.put((callback, region, target)) - def store_target(self, target): + def store_target(self, global_params, region, target): target_type = target.pop('scout2_target_type') if 'VpcId' in target: vpc_id = target.pop('VpcId') @@ -335,3 +330,4 @@ def store_target(self, target): target_id = target[resource_id_map[target_type]] get_name(target, target, resource_id_map[target_type]) target_dict[target_id] = target + From 4950147a662b78a45500727fd7b9e7b60d901821 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 14:27:39 -0500 Subject: [PATCH 042/154] PEP8 fixes in regions.py --- ScoutSuite/providers/aws/configs/regions.py | 36 ++++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 8e918de9d..fc9293c0b 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -6,13 +6,15 @@ import copy import re from threading import Thread + # Python2 vs Python3 try: from Queue import Queue except ImportError: from queue import Queue -from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, handle_truncated_response +from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, \ + handle_truncated_response from ScoutSuite.core.console import printException, printInfo from ScoutSuite.providers.base.configs import resource_id_map @@ -31,6 +33,7 @@ first_cap_re = re.compile('(.)([A-Z][a-z]+)') all_caps_re = re.compile('([a-z0-9])([A-Z])') + ######################################## # RegionalServiceConfig ######################################## @@ -50,7 +53,7 @@ def __init__(self, service_metadata=None, thread_config=4): self.regions = {} self.thread_config = thread_configs[thread_config] self.service = re.sub(r'Config$', "", type(self).__name__).lower() - + # Booleans that define if threads should keep running self.run_q_threads = True self.run_qr_threads = True @@ -91,14 +94,13 @@ def init_region_config(self, region): :param region: Name of the region """ - self.regions[region] = self.region_config_class(region_name = region, resource_types = self.resource_types) + self.regions[region] = self.region_config_class(region_name=region, resource_types=self.resource_types) - def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets = None): + def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): """ Fetch all the configuration supported by Scout2 for a given service :param credentials: F - :param service: Name of the service :param regions: Name of regions to fetch data from :param partition_name: AWS partition to connect to :param targets: Type of resources to be fetched; defaults to all. @@ -125,7 +127,7 @@ def fetch_all(self, credentials, regions = None, partition_name = 'aws', targets api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower() # Init regions - regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class + regions = build_region_list(api_service, regions, partition_name) # TODO: move this code within this class self.fetchstatuslogger.counts['regions']['discovered'] = len(regions) # Threading to fetch & parse resources (queue consumer) @@ -166,7 +168,7 @@ def _init_threading(self, function, params=None, num_threads=10): :return: """ params = {} if params is None else params - q = Queue(maxsize=0) # TODO: find something appropriate + q = Queue(maxsize=0) # TODO: find something appropriate for i in range(num_threads): worker = Thread(target=function, args=(q, params)) worker.setDaemon(True) @@ -181,10 +183,11 @@ def _fetch_region(self, q, params): region, targets = q.get() or (None, None) if region: self.init_region_config(region) - api_client = connect_service(params['api_service'], params['credentials'], region, silent = True) + api_client = connect_service(params['api_service'], params['credentials'], region, silent=True) api_clients[region] = api_client # TODO : something here for single_region stuff - self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], targets) # params['targets']) + self.regions[region].fetch_all(api_client, self.fetchstatuslogger, params['q'], + targets) # params['targets']) self.fetchstatuslogger.counts['regions']['fetched'] += 1 except Exception as e: printException(e) @@ -242,11 +245,13 @@ def tweak_params(self, params, credentials): params = get_aws_account_id(credentials) return params + ######################################## # RegionConfig ######################################## +# noinspection PyBroadException class RegionConfig(BaseConfig): """ Base class for ... @@ -265,7 +270,7 @@ def __init__(self, region_name, resource_types=None): self.vpc_resource_types = resource_types['vpc'] def fetch_all(self, api_client, fetchstatuslogger, q, targets): - ''' + """ Make all API calls as defined in metadata.json :param api_client: @@ -273,19 +278,19 @@ def fetch_all(self, api_client, fetchstatuslogger, q, targets): :param q: :param targets: :return: - ''' + """ self.fetchstatuslogger = fetchstatuslogger - if targets != None: + if targets is not None: # Ensure targets is a tuple if type(targets) != list and type(targets) != tuple: - targets = tuple(targets,) + targets = tuple(targets, ) elif type(targets) != tuple: targets = tuple(targets) for target in targets: self._fetch_targets(api_client, q, target) def _fetch_targets(self, api_client, q, target): - ''' + """ Make an API call defined in metadata.json. Parse the returned object as implemented in the "parse_[object name]" method. @@ -293,7 +298,7 @@ def _fetch_targets(self, api_client, q, target): :param q: :param target: :return: - ''' + """ # Handle & format the target type target_type, response_attribute, list_method_name, list_params, ignore_list_error = target list_method = getattr(api_client, list_method_name) @@ -330,4 +335,3 @@ def store_target(self, global_params, region, target): target_id = target[resource_id_map[target_type]] get_name(target, target, resource_id_map[target_type]) target_dict[target_id] = target - From 5d56d28334df6f62f75dd35a4889179732565be9 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 14:33:24 -0500 Subject: [PATCH 043/154] Reformatted a few AWS files --- ScoutSuite/providers/aws/configs/vpc.py | 3 +- ScoutSuite/providers/aws/provider.py | 26 +++++++++++------ .../providers/aws/services/awslambda.py | 4 +-- .../providers/aws/services/cloudformation.py | 6 ++-- .../providers/aws/services/cloudtrail.py | 17 +++++------ .../providers/aws/services/cloudwatch.py | 4 +-- .../providers/aws/services/directconnect.py | 4 +-- ScoutSuite/providers/aws/services/ec2.py | 14 +++++---- ScoutSuite/providers/aws/services/efs.py | 13 +++++---- .../providers/aws/services/elasticache.py | 8 ++--- ScoutSuite/providers/aws/services/elb.py | 10 ++++--- ScoutSuite/providers/aws/services/elbv2.py | 8 +++-- ScoutSuite/providers/aws/services/iam.py | 2 +- ScoutSuite/providers/aws/services/rds.py | 29 ++++++++++--------- ScoutSuite/providers/aws/services/redshift.py | 10 +++---- ScoutSuite/providers/aws/services/route53.py | 23 ++++++--------- ScoutSuite/providers/aws/services/s3.py | 1 + ScoutSuite/providers/aws/services/ses.py | 12 ++++---- ScoutSuite/providers/aws/services/sns.py | 4 +-- ScoutSuite/providers/aws/services/sqs.py | 8 ++--- ScoutSuite/providers/aws/services/vpc.py | 5 ++++ 21 files changed, 108 insertions(+), 103 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/vpc.py b/ScoutSuite/providers/aws/configs/vpc.py index 0d9edaaef..ccc0aa288 100644 --- a/ScoutSuite/providers/aws/configs/vpc.py +++ b/ScoutSuite/providers/aws/configs/vpc.py @@ -3,12 +3,13 @@ Base class for vpc-specific services """ + class VPCConfig(object): """ Configuration for a single VPC in a single service """ - def __init__(self, vpc_resource_types, name = None): + def __init__(self, vpc_resource_types, name=None): self.name = name for resource_type in vpc_resource_types: setattr(self, resource_type, {}) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 55c5d239a..c15cf2647 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -20,7 +20,8 @@ class AWSProvider(BaseProvider): Implements provider for AWS """ - def __init__(self, profile=None, report_dir=None, timestamp=None, services=None, skipped_services=None, thread_config=4, **kwargs): + def __init__(self, profile=None, report_dir=None, timestamp=None, services=None, skipped_services=None, + thread_config=4, **kwargs): services = [] if services is None else services skipped_services = [] if skipped_services is None else skipped_services @@ -112,11 +113,13 @@ def _check_ec2_zone_distribution(self): def _add_last_snapshot_date_to_ec2_volumes(self): for region in self.services['ec2']['regions'].values(): for volumeId, volume in region.get('volumes').items(): - completed_snapshots = [s for s in region['snapshots'].values() if s['VolumeId'] == volumeId and s['State'] == 'completed'] + completed_snapshots = [s for s in region['snapshots'].values() if + s['VolumeId'] == volumeId and s['State'] == 'completed'] sorted_snapshots = sorted(completed_snapshots, key=lambda s: s['StartTime'], reverse=True) volume['LastSnapshotDate'] = sorted_snapshots[0]['StartTime'] if len(sorted_snapshots) > 0 else None - def add_security_group_name_to_ec2_grants_callback(self, current_config, path, current_path, ec2_grant, callback_args): + def add_security_group_name_to_ec2_grants_callback(self, current_config, path, current_path, ec2_grant, + callback_args): sg_id = ec2_grant['GroupId'] if sg_id in current_path: target = current_path[:(current_path.index(sg_id) + 1)] @@ -239,8 +242,9 @@ def _match_iam_policies_and_buckets(self): parts = full_path.split('/') bucket_name = parts[0].split(':')[-1] self._update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, - iam_info['permissions']['Action'][action][iam_entity][ - 'Allow'][allowed_iam_entity]['Resource'][full_path]) + iam_info['permissions']['Action'][action][iam_entity][ + 'Allow'][allowed_iam_entity]['Resource'][ + full_path]) # For notresource statements, we must fetch the policy document to determine which buckets are not protected if 'NotResource' in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ allowed_iam_entity]: @@ -352,7 +356,8 @@ def _match_instances_and_roles(self): role_instances[instance_profile_id] iam_config['roles'][role_id]['instances_count'] += len(role_instances[instance_profile_id]) - def match_roles_and_cloudformation_stacks_callback(self, current_config, path, current_path, stack_id, callback_args): + def match_roles_and_cloudformation_stacks_callback(self, current_config, path, current_path, stack_id, + callback_args): if 'RoleARN' not in current_config: return role_arn = current_config.pop('RoleARN') @@ -376,7 +381,8 @@ def _get_role_info(self, attribute_name, attribute_value): def process_vpc_peering_connections_callback(self, current_config, path, current_path, pc_id, callback_args): # Create a list of peering connection IDs in each VPC - info = 'AccepterVpcInfo' if current_config['AccepterVpcInfo']['OwnerId'] == self.aws_account_id else 'RequesterVpcInfo' + info = 'AccepterVpcInfo' if current_config['AccepterVpcInfo'][ + 'OwnerId'] == self.aws_account_id else 'RequesterVpcInfo' region = current_path[current_path.index('regions') + 1] vpc_id = current_config[info]['VpcId'] if vpc_id not in self.services['vpc']['regions'][region]['vpcs']: @@ -399,7 +405,8 @@ def process_vpc_peering_connections_callback(self, current_config, path, current else: current_config['peer_info']['name'] = current_config['peer_info']['OwnerId'] - def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id, callback_args): + def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id, + callback_args): service = current_path[1] original_resource_path = combine_paths(copy.deepcopy(current_path), [resource_id]) resource = get_object_at(self, original_resource_path) @@ -574,7 +581,8 @@ def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_l attached_vpc['subnets'][subnet_id]['flow_logs'].append(flow_log_id) elif attached_resource.startswith('subnet-'): subnet_path = combine_paths(current_path[0:4], - ['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets', attached_resource]) + ['vpcs', self.subnet_map[attached_resource]['vpc_id'], 'subnets', + attached_resource]) subnet = get_object_at(self, subnet_path) manage_dictionary(subnet, 'flow_logs', []) if flow_log_id not in subnet['flow_logs']: diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index bb567c5ae..7eb9ff54b 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -6,7 +6,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # LambdaRegionConfig ######################################## @@ -18,7 +17,6 @@ def parse_function(self, global_params, region, function): self.functions[function['name']] = function - ######################################## # LambdaConfig ######################################## @@ -30,5 +28,5 @@ class LambdaConfig(RegionalServiceConfig): region_config_class = LambdaRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(LambdaConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/cloudformation.py b/ScoutSuite/providers/aws/services/cloudformation.py index bc770b890..2b683c937 100644 --- a/ScoutSuite/providers/aws/services/cloudformation.py +++ b/ScoutSuite/providers/aws/services/cloudformation.py @@ -5,7 +5,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # CloudFormationRegionConfig ######################################## @@ -25,7 +24,7 @@ def parse_stack(self, global_params, region, stack): """ stack['id'] = stack.pop('StackId') stack['name'] = stack.pop('StackName') - + stack_description = api_clients[region].describe_stacks(StackName=stack['name']) stack['termination_protection'] = stack_description['Stacks'][0]['EnableTerminationProtection'] @@ -35,7 +34,6 @@ def parse_stack(self, global_params, region, stack): self.stacks[stack['name']] = stack - ######################################## # CloudFormationConfig ######################################## @@ -47,5 +45,5 @@ class CloudFormationConfig(RegionalServiceConfig): region_config_class = CloudFormationRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(CloudFormationConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/cloudtrail.py b/ScoutSuite/providers/aws/services/cloudtrail.py index 0738c2662..0f085e2e6 100644 --- a/ScoutSuite/providers/aws/services/cloudtrail.py +++ b/ScoutSuite/providers/aws/services/cloudtrail.py @@ -7,7 +7,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # CloudTrailRegionConfig ######################################## @@ -45,22 +44,22 @@ def parse_trail(self, global_params, region, trail): if key not in trail_config: trail_config[key] = False trail_details = api_client.get_trail_status(Name=trail['TrailARN']) - for key in ['KMSKeyId', 'IsLogging', 'LatestDeliveryTime', 'LatestDeliveryError', 'StartLoggingTime', 'StopLoggingTime', 'LatestNotificationTime', 'LatestNotificationError', 'LatestCloudWatchLogsDeliveryError', 'LatestCloudWatchLogsDeliveryTime']: + for key in ['KMSKeyId', 'IsLogging', 'LatestDeliveryTime', 'LatestDeliveryError', 'StartLoggingTime', + 'StopLoggingTime', 'LatestNotificationTime', 'LatestNotificationError', + 'LatestCloudWatchLogsDeliveryError', 'LatestCloudWatchLogsDeliveryTime']: trail_config[key] = trail_details[key] if key in trail_details else None - if trail_details: # using trail ARN instead of name as with Organizations the trail would be located in another account trail_config['wildcard_data_logging'] = self.data_logging_status(trail_config['TrailARN'], trail_details, api_client) - + for es in api_client.get_event_selectors(TrailName=trail_config['TrailARN'])['EventSelectors']: trail_config['DataEventsEnabled'] = len(es['DataResources']) > 0 trail_config['ManagementEventsEnabled'] = es['IncludeManagementEvents'] - - self.trails[trail_id] = trail_config + self.trails[trail_id] = trail_config def data_logging_status(self, trail_name, trail_details, api_client): for es in api_client.get_event_selectors(TrailName=trail_name)['EventSelectors']: @@ -90,11 +89,10 @@ class CloudTrailConfig(RegionalServiceConfig): region_config_class = CloudTrailRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(CloudTrailConfig, self).__init__(service_metadata, thread_config) - ######################################## # Post Processing ######################################## @@ -108,7 +106,8 @@ def cloudtrail_postprocessing(aws_config): cloudtrail_config['violations']['cloudtrail-duplicated-global-services-logging']['flagged_items'] = 0 # Global services logging disabled if 'cloudtrail-no-global-services-logging' in cloudtrail_config['violations']: - if len(cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items']) != cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['checked_items']: + if len(cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items']) != \ + cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['checked_items']: cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['items'] = [] cloudtrail_config['violations']['cloudtrail-no-global-services-logging']['flagged_items'] = 0 # CloudTrail not enabled at all... diff --git a/ScoutSuite/providers/aws/services/cloudwatch.py b/ScoutSuite/providers/aws/services/cloudwatch.py index e5ccaf98a..95ce45dc6 100644 --- a/ScoutSuite/providers/aws/services/cloudwatch.py +++ b/ScoutSuite/providers/aws/services/cloudwatch.py @@ -6,7 +6,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # CloudWatchRegionConfig ######################################## @@ -33,7 +32,6 @@ def parse_alarm(self, global_params, region, alarm): self.alarms[alarm_id] = alarm - ######################################## # CloudWatchConfig ######################################## @@ -45,5 +43,5 @@ class CloudWatchConfig(RegionalServiceConfig): region_config_class = CloudWatchRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(CloudWatchConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/directconnect.py b/ScoutSuite/providers/aws/services/directconnect.py index f848930d7..761e8d4c0 100644 --- a/ScoutSuite/providers/aws/services/directconnect.py +++ b/ScoutSuite/providers/aws/services/directconnect.py @@ -3,7 +3,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # DirectConnectRegionConfig ######################################## @@ -26,7 +25,6 @@ def parse_connection(self, global_params, region, connection): self.connections[connection['id']] = connection - ######################################## # DirectConnectConfig ######################################## @@ -38,5 +36,5 @@ class DirectConnectConfig(RegionalServiceConfig): region_config_class = DirectConnectRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(DirectConnectConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index ce52dfe23..662c00066 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -16,7 +16,6 @@ from ScoutSuite.utils import get_keys, ec2_classic, manage_dictionary from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # Globals ######################################## @@ -71,7 +70,8 @@ def parse_instance(self, global_params, region, reservation): self.vpcs[vpc_id].instances[i['InstanceId']] = instance def _get_user_data(self, region, instance_id): - user_data_response = api_clients[region].describe_instance_attribute(Attribute='userData', InstanceId=instance_id) + user_data_response = api_clients[region].describe_instance_attribute(Attribute='userData', + InstanceId=instance_id) if 'Value' not in user_data_response['UserData'].keys(): return None @@ -92,7 +92,7 @@ def parse_image(self, global_params, region, image): image['id'] = id image['name'] = name - self.images[id] = image + self.images[id] = image def parse_security_group(self, global_params, region, group): """ @@ -172,8 +172,9 @@ def parse_snapshot(self, global_params, region, snapshot): self.snapshots[snapshot['id']] = snapshot # Get snapshot attribute snapshot['createVolumePermission'] = \ - api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', SnapshotId=snapshot['id'])[ - 'CreateVolumePermissions'] + api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', + SnapshotId=snapshot['id'])[ + 'CreateVolumePermissions'] snapshot['public'] = self._is_public(snapshot) def _is_public(self, snapshot): @@ -225,6 +226,7 @@ def analyze_ec2_config(ec2_info, aws_account_id, force_write): printInfo('Error') printException(e) + def add_security_group_name_to_ec2_grants_callback(ec2_config, current_config, path, current_path, ec2_grant, callback_args): """ @@ -290,7 +292,7 @@ def link_elastic_ips_callback2(ec2_config, current_config, path, current_path, i current_config['PublicIpAddress'] = callback_args['elastic_ip'] elif current_config['PublicIpAddress'] != callback_args['elastic_ip']: printInfo('Warning: public IP address exists (%s) for an instance associated with an elastic IP (%s)' % ( - current_config['PublicIpAddress'], callback_args['elastic_ip'])) + current_config['PublicIpAddress'], callback_args['elastic_ip'])) # This can happen... fix it diff --git a/ScoutSuite/providers/aws/services/efs.py b/ScoutSuite/providers/aws/services/efs.py index 58198baf1..ab7e3dd85 100644 --- a/ScoutSuite/providers/aws/services/efs.py +++ b/ScoutSuite/providers/aws/services/efs.py @@ -7,7 +7,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # EFSRegionConfig ######################################## @@ -31,19 +30,21 @@ def parse_file_system(self, global_params, region, file_system): else: file_system['name'] = None # Get tags - file_system['tags'] = handle_truncated_response(api_clients[region].describe_tags, {'FileSystemId': fs_id}, ['Tags'])['Tags'] + file_system['tags'] = \ + handle_truncated_response(api_clients[region].describe_tags, {'FileSystemId': fs_id}, ['Tags'])['Tags'] # Get mount targets - mount_targets = handle_truncated_response(api_clients[region].describe_mount_targets, {'FileSystemId': fs_id}, ['MountTargets'])['MountTargets'] + mount_targets = handle_truncated_response(api_clients[region].describe_mount_targets, {'FileSystemId': fs_id}, + ['MountTargets'])['MountTargets'] file_system['mount_targets'] = {} for mt in mount_targets: mt_id = mt['MountTargetId'] file_system['mount_targets'][mt_id] = mt # Get security groups - file_system['mount_targets'][mt_id]['security_groups'] = api_clients[region].describe_mount_target_security_groups(MountTargetId = mt_id)['SecurityGroups'] + file_system['mount_targets'][mt_id]['security_groups'] = \ + api_clients[region].describe_mount_target_security_groups(MountTargetId=mt_id)['SecurityGroups'] self.file_systems[fs_id] = file_system - ######################################## # EFSConfig ######################################## @@ -55,5 +56,5 @@ class EFSConfig(RegionalServiceConfig): region_config_class = EFSRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(EFSConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/elasticache.py b/ScoutSuite/providers/aws/services/elasticache.py index ffdf6a541..9af741cc0 100644 --- a/ScoutSuite/providers/aws/services/elasticache.py +++ b/ScoutSuite/providers/aws/services/elasticache.py @@ -26,7 +26,9 @@ def parse_cluster(self, global_params, region, cluster): cluster['name'] = cluster_name # Must fetch info about the subnet group to retrieve the VPC ID... if 'CacheSubnetGroupName' in cluster: - subnet_group = api_clients[region].describe_cache_subnet_groups(CacheSubnetGroupName = cluster['CacheSubnetGroupName'])['CacheSubnetGroups'][0] + subnet_group = \ + api_clients[region].describe_cache_subnet_groups(CacheSubnetGroupName=cluster['CacheSubnetGroupName'])[ + 'CacheSubnetGroups'][0] vpc_id = subnet_group['VpcId'] else: vpc_id = ec2_classic @@ -36,7 +38,6 @@ def parse_cluster(self, global_params, region, cluster): if subnet_group: self.vpcs[vpc_id].subnet_groups[subnet_group['CacheSubnetGroupName']] = subnet_group - def parse_security_group(self, global_params, region, security_group): """ Parse a single ElastiCache security group @@ -50,7 +51,6 @@ def parse_security_group(self, global_params, region, security_group): self.security_groups[security_group['name']] = security_group - ######################################## # ElastiCacheConfig ######################################## @@ -62,5 +62,5 @@ class ElastiCacheConfig(RegionalServiceConfig): region_config_class = ElastiCacheRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(ElastiCacheConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/elb.py b/ScoutSuite/providers/aws/services/elb.py index 2d35580b4..5e32137be 100644 --- a/ScoutSuite/providers/aws/services/elb.py +++ b/ScoutSuite/providers/aws/services/elb.py @@ -46,7 +46,9 @@ def parse_elb(self, global_params, region, lb): elb['listeners'][l['Listener']['LoadBalancerPort']] = listener # Fetch LB policies here. This is not ideal, but the alternative is to download all policies and clean up after... if len(policy_names): - policies = api_clients[region].describe_load_balancer_policies(LoadBalancerName = elb['name'], PolicyNames = policy_names)['PolicyDescriptions'] + policies = \ + api_clients[region].describe_load_balancer_policies(LoadBalancerName=elb['name'], PolicyNames=policy_names)[ + 'PolicyDescriptions'] for policy in policies: policy['name'] = policy.pop('PolicyName') policy_id = self.get_non_provider_id(policy['name']) @@ -55,11 +57,11 @@ def parse_elb(self, global_params, region, lb): for i in lb['Instances']: elb['instances'].append(i['InstanceId']) # Get attributes - elb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerName=elb['name'])['LoadBalancerAttributes'] + elb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerName=elb['name'])[ + 'LoadBalancerAttributes'] self.vpcs[vpc_id].elbs[self.get_non_provider_id(elb['name'])] = elb - ######################################## # ELBConfig ######################################## @@ -71,5 +73,5 @@ class ELBConfig(RegionalServiceConfig): region_config_class = ELBRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(ELBConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index b67c1c5f5..f0166d781 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -43,14 +43,16 @@ def parse_lb(self, global_params, region, lb): pass lb['listeners'] = {} # Get listeners - listeners = handle_truncated_response(api_clients[region].describe_listeners, {'LoadBalancerArn': lb['arn']}, ['Listeners'])['Listeners'] + listeners = handle_truncated_response(api_clients[region].describe_listeners, {'LoadBalancerArn': lb['arn']}, + ['Listeners'])['Listeners'] for listener in listeners: listener.pop('ListenerArn') listener.pop('LoadBalancerArn') port = listener.pop('Port') lb['listeners'][port] = listener # Get attributes - lb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerArn = lb['arn'])['Attributes'] + lb['attributes'] = api_clients[region].describe_load_balancer_attributes(LoadBalancerArn=lb['arn'])[ + 'Attributes'] self.vpcs[vpc_id].lbs[self.get_non_provider_id(lb['name'])] = lb def parse_ssl_policie(self, global_params, region, policy): @@ -68,5 +70,5 @@ class ELBv2Config(RegionalServiceConfig): """ region_config_class = ELBv2RegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(ELBv2Config, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index dd67370a9..c0c4fe16c 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -425,4 +425,4 @@ def __parse_resource(self, effect, action_string, action, resource_string, resou self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ resource][policy_type], policy_name, {}) self.permissions[action_string][action][iam_resource_type][effect][iam_resource_name][resource_string][ - resource][policy_type][policy_name]['condition'] = condition \ No newline at end of file + resource][policy_type][policy_name]['condition'] = condition diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index eeff7459c..70425fe68 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -25,25 +25,25 @@ def parse_instance(self, global_params, region, dbi): :param region: Name of the AWS region :param instance: Instance """ - vpc_id = dbi['DBSubnetGroup']['VpcId'] if 'DBSubnetGroup' in dbi and 'VpcId' in dbi['DBSubnetGroup'] and dbi['DBSubnetGroup']['VpcId'] else ec2_classic + vpc_id = dbi['DBSubnetGroup']['VpcId'] if 'DBSubnetGroup' in dbi and 'VpcId' in dbi['DBSubnetGroup'] and \ + dbi['DBSubnetGroup']['VpcId'] else ec2_classic instance = {} instance['name'] = dbi.pop('DBInstanceIdentifier') for key in ['InstanceCreateTime', 'Engine', 'DBInstanceStatus', 'AutoMinorVersionUpgrade', 'DBInstanceClass', 'MultiAZ', 'Endpoint', 'BackupRetentionPeriod', 'PubliclyAccessible', 'StorageEncrypted', 'VpcSecurityGroups', 'DBSecurityGroups', 'DBParameterGroups', 'EnhancedMonitoringResourceArn', 'StorageEncrypted']: - # parameter_groups , security_groups, vpc_security_groups + # parameter_groups , security_groups, vpc_security_groups instance[key] = dbi[key] if key in dbi else None # If part of a cluster, multi AZ information is only available via cluster information if 'DBClusterIdentifier' in dbi: api_client = api_clients[region] - cluster = api_client.describe_db_clusters(DBClusterIdentifier = dbi['DBClusterIdentifier'])['DBClusters'][0] + cluster = api_client.describe_db_clusters(DBClusterIdentifier=dbi['DBClusterIdentifier'])['DBClusters'][0] instance['MultiAZ'] = cluster['MultiAZ'] # Save manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) self.vpcs[vpc_id].instances[instance['name']] = instance - def parse_snapshot(self, global_params, region, dbs): """ @@ -64,19 +64,21 @@ def parse_snapshot(self, global_params, region, dbs): for attribute in attributes: snapshot[attribute] = dbs[attribute] if attribute in dbs else None api_client = api_clients[region] - attributes = api_client.describe_db_snapshot_attributes(DBSnapshotIdentifier = snapshot_id)['DBSnapshotAttributesResult'] + attributes = api_client.describe_db_snapshot_attributes(DBSnapshotIdentifier=snapshot_id)[ + 'DBSnapshotAttributesResult'] snapshot['attributes'] = attributes['DBSnapshotAttributes'] if 'DBSnapshotAttributes' in attributes else {} # Save manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) self.vpcs[vpc_id].snapshots[snapshot_id] = snapshot - def parse_parameter_group(self, global_params, region, parameter_group): parameter_group['arn'] = parameter_group.pop('DBParameterGroupArn') parameter_group['name'] = parameter_group.pop('DBParameterGroupName') api_client = api_clients[region] try: - parameters = handle_truncated_response(api_client.describe_db_parameters, {'DBParameterGroupName': parameter_group['name']}, ['Parameters'])['Parameters'] + parameters = handle_truncated_response(api_client.describe_db_parameters, + {'DBParameterGroupName': parameter_group['name']}, ['Parameters'])[ + 'Parameters'] manage_dictionary(parameter_group, 'parameters', {}) for parameter in parameters: if not parameter['IsModifiable']: @@ -91,7 +93,6 @@ def parse_parameter_group(self, global_params, region, parameter_group): parameter_group_id = self.get_non_provider_id(parameter_group['name']) (self).parameter_groups[parameter_group_id] = parameter_group - def parse_security_group(self, global_params, region, security_group): """ Parse a single Redsfhit security group @@ -100,17 +101,16 @@ def parse_security_group(self, global_params, region, security_group): :param region: Name of the AWS region :param security)_group: Security group """ - #vpc_id = security_group.pop('VpcId') if 'VpcId' in security_group else ec2_classic - #manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) + # vpc_id = security_group.pop('VpcId') if 'VpcId' in security_group else ec2_classic + # manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) security_group['arn'] = security_group.pop('DBSecurityGroupArn') security_group['name'] = security_group.pop('DBSecurityGroupName') # Save - #manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - #self.vpcs[vpc_id].security_groups[security_group['name']] = security_group + # manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) + # self.vpcs[vpc_id].security_groups[security_group['name']] = security_group self.security_groups[security_group['name']] = security_group - ######################################## # RDSConfig ######################################## @@ -122,7 +122,7 @@ class RDSConfig(RegionalServiceConfig): region_config_class = RDSRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(RDSConfig, self).__init__(service_metadata, thread_config) @@ -136,6 +136,7 @@ def get_security_groups_info(rds_client, region_info): for group in groups: region_info['vpcs'][ec2_classic]['security_groups'][group['DBSecurityGroupName']] = parse_security_group(group) + def parse_security_group(group): security_group = {} security_group['name'] = group['DBSecurityGroupName'] diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index 772d40edc..505ea1848 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -33,7 +33,6 @@ def parse_cluster(self, global_params, region, cluster): cluster['name'] = name self.vpcs[vpc_id].clusters[name] = cluster - def parse_parameter_group(self, global_params, region, parameter_group): """ Parse a single Redshift parameter group and fetch all of its parameters @@ -43,11 +42,12 @@ def parse_parameter_group(self, global_params, region, parameter_group): :param parameter_group: Parameter group """ pg_name = parameter_group.pop('ParameterGroupName') - pg_id = self.get_non_provider_id(pg_name) # Name could be used as only letters digits or hyphens + pg_id = self.get_non_provider_id(pg_name) # Name could be used as only letters digits or hyphens parameter_group['name'] = pg_name parameter_group['parameters'] = {} api_client = api_clients[region] - parameters = handle_truncated_response(api_client.describe_cluster_parameters, {'ParameterGroupName': pg_name}, ['Parameters'])['Parameters'] + parameters = handle_truncated_response(api_client.describe_cluster_parameters, {'ParameterGroupName': pg_name}, + ['Parameters'])['Parameters'] for parameter in parameters: param = {} param['value'] = parameter['ParameterValue'] @@ -55,7 +55,6 @@ def parse_parameter_group(self, global_params, region, parameter_group): parameter_group['parameters'][parameter['ParameterName']] = param (self).parameter_groups[pg_id] = parameter_group - def parse_security_group(self, global_params, region, security_group): """ Parse a single Redsfhit security group @@ -69,7 +68,6 @@ def parse_security_group(self, global_params, region, security_group): self.security_groups['name'] = security_group - ######################################## # RedshiftConfig ######################################## @@ -81,5 +79,5 @@ class RedshiftConfig(RegionalServiceConfig): region_config_class = RedshiftRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(RedshiftConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/route53.py b/ScoutSuite/providers/aws/services/route53.py index 08d502f1c..b7290af1e 100644 --- a/ScoutSuite/providers/aws/services/route53.py +++ b/ScoutSuite/providers/aws/services/route53.py @@ -5,7 +5,6 @@ from ScoutSuite.providers.aws.configs.base import AWSBaseConfig - class Route53DomainsConfig(AWSBaseConfig): """ Object that holds the Route53Domains configuration @@ -20,8 +19,6 @@ def __init__(self, target_config): self.domains_count = 0 super(Route53DomainsConfig, self).__init__(target_config) - - ######################################## ##### Domains ######################################## @@ -31,14 +28,13 @@ def parse_domains(self, domain, params): """ domain_id = self.get_non_provider_id(domain['DomainName']) domain['name'] = domain.pop('DomainName') - #TODO: Get Dnssec info when available - #api_client = params['api_client'] - #details = api_client.get_domain_detail(DomainName = domain['name']) - #get_keys(details, domain, ['Dnssec']) + # TODO: Get Dnssec info when available + # api_client = params['api_client'] + # details = api_client.get_domain_detail(DomainName = domain['name']) + # get_keys(details, domain, ['Dnssec']) self.domains[domain_id] = domain - class Route53Config(AWSBaseConfig): """ Object that holds the Route53 configuration @@ -53,8 +49,6 @@ def __init__(self, target_config): self.hosted_zones_count = 0 super(Route53Config, self).__init__(target_config) - - ######################################## ##### hosted_zoness ######################################## @@ -66,9 +60,10 @@ def parse_hosted_zones(self, hosted_zone, params): hosted_zone_id = hosted_zone.pop('Id') hosted_zone['name'] = hosted_zone.pop('Name') api_client = params['api_client'] - record_sets = handle_truncated_response(api_client.list_resource_record_sets, {'HostedZoneId': hosted_zone_id}, ['ResourceRecordSets']) + record_sets = handle_truncated_response(api_client.list_resource_record_sets, {'HostedZoneId': hosted_zone_id}, + ['ResourceRecordSets']) hosted_zone.update(record_sets) - #print(str(record_sets)) - #record_sets = api_client.list_resource_record_sets() - #hosted_zone['RecordSets'] = record_sets['Resourc'] + # print(str(record_sets)) + # record_sets = api_client.list_resource_record_sets() + # hosted_zone['RecordSets'] = record_sets['Resourc'] self.hosted_zones[hosted_zone_id] = hosted_zone diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index da9b1d0e2..e7befa31e 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -272,6 +272,7 @@ def get_s3_bucket_versioning(api_client, bucket_name, bucket_info): bucket_info['version_mfa_delete_enabled'] = None return False + def _status_to_bool(value): """ Converts a string to True if it is equal to 'Enabled' or to False otherwise. """ return value == 'Enabled' diff --git a/ScoutSuite/providers/aws/services/ses.py b/ScoutSuite/providers/aws/services/ses.py index eff667512..237139cba 100644 --- a/ScoutSuite/providers/aws/services/ses.py +++ b/ScoutSuite/providers/aws/services/ses.py @@ -5,7 +5,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # SESRegionConfig ######################################## @@ -23,18 +22,19 @@ def parse_identitie(self, global_params, region, identity_name): :param region: Name of the AWS region """ identity = {'name': identity_name, 'policies': {}} - policy_names = api_clients[region].list_identity_policies(Identity = identity_name)['PolicyNames'] + policy_names = api_clients[region].list_identity_policies(Identity=identity_name)['PolicyNames'] if len(policy_names): - policies = api_clients[region].get_identity_policies(Identity = identity_name, PolicyNames = policy_names)['Policies'] + policies = api_clients[region].get_identity_policies(Identity=identity_name, PolicyNames=policy_names)[ + 'Policies'] for policy_name in policies: identity['policies'][policy_name] = json.loads(policies[policy_name]) - dkim = api_clients[region].get_identity_dkim_attributes(Identities = [ identity_name ])['DkimAttributes'][identity_name] + dkim = api_clients[region].get_identity_dkim_attributes(Identities=[identity_name])['DkimAttributes'][ + identity_name] identity['DkimEnabled'] = dkim['DkimEnabled'] identity['DkimVerificationStatus'] = dkim['DkimVerificationStatus'] self.identities[self.get_non_provider_id(identity_name)] = identity - ######################################## # SESConfig ######################################## @@ -46,5 +46,5 @@ class SESConfig(RegionalServiceConfig): region_config_class = SESRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(SESConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/sns.py b/ScoutSuite/providers/aws/services/sns.py index 20b066784..511a5282e 100644 --- a/ScoutSuite/providers/aws/services/sns.py +++ b/ScoutSuite/providers/aws/services/sns.py @@ -36,7 +36,6 @@ def parse_subscription(self, params, region, subscription): topic['subscriptions']['protocol'][protocol].append(subscription) topic['subscriptions_count'] += 1 - def parse_topic(self, params, region, topic): """ Parse a single topic and fetch additional attributes @@ -59,7 +58,6 @@ def parse_topic(self, params, region, topic): self.topics[topic['name']] = topic - ######################################## # SNSConfig ######################################## @@ -71,5 +69,5 @@ class SNSConfig(RegionalServiceConfig): region_config_class = SNSRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(SNSConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/sqs.py b/ScoutSuite/providers/aws/services/sqs.py index eab64bc6d..d4d512d8a 100644 --- a/ScoutSuite/providers/aws/services/sqs.py +++ b/ScoutSuite/providers/aws/services/sqs.py @@ -5,7 +5,6 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients - ######################################## # SQSRegionConfig ######################################## @@ -24,7 +23,9 @@ def parse_queue(self, global_params, region, queue_url): :param queue_url: URL of the AWS queue """ queue = {'QueueUrl': queue_url} - attributes = api_clients[region].get_queue_attributes(QueueUrl = queue_url, AttributeNames = ['CreatedTimestamp', 'Policy', 'QueueArn', 'KmsMasterKeyId'])['Attributes'] + attributes = api_clients[region].get_queue_attributes(QueueUrl=queue_url, + AttributeNames=['CreatedTimestamp', 'Policy', 'QueueArn', + 'KmsMasterKeyId'])['Attributes'] queue['arn'] = attributes.pop('QueueArn') queue['kms_master_key_id'] = attributes.pop('KmsMasterKeyId', None) queue['CreatedTimestamp'] = attributes.pop('CreatedTimestamp', None) @@ -38,7 +39,6 @@ def parse_queue(self, global_params, region, queue_url): self.queues[queue['name']] = queue - ######################################## # SQSConfig ######################################## @@ -50,5 +50,5 @@ class SQSConfig(RegionalServiceConfig): region_config_class = SQSRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(SQSConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index e6cf953e9..461b8bee4 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -17,6 +17,7 @@ protocols_dict = load_data('protocols.json', 'protocols') + ######################################## # VPCRegionConfig ######################################## @@ -159,6 +160,7 @@ def __init__(self, service_metadata, thread_config): known_cidrs = {'0.0.0.0/0': 'All'} + def put_cidr_name(aws_config, current_config, path, current_path, resource_id, callback_args): """ Add a display name for all known CIDRs @@ -186,6 +188,7 @@ def put_cidr_name(aws_config, current_config, path, current_path, resource_id, c aws_ip_ranges = {} # read_ip_ranges(aws_ip_ranges_filename, False) + def get_cidr_name(cidr, ip_ranges_files, ip_ranges_name_key): """ Read display name for CIDRs from ip-ranges files @@ -208,6 +211,7 @@ def get_cidr_name(cidr, ip_ranges_files, ip_ranges_name_key): return 'Unknown CIDR in %s %s' % (ip_range['service'], ip_range['region']) return 'Unknown CIDR' + def propagate_vpc_names(aws_config, current_config, path, current_path, resource_id, callback_args): """ Propagate VPC names in VPC-related services (info only fetched during EC2 calls) @@ -229,6 +233,7 @@ def propagate_vpc_names(aws_config, current_config, path, current_path, resource target_path = '.'.join(target_path) current_config['name'] = get_value_at(aws_config, target_path, target_path) + def get_subnet_flow_logs_list(current_config, subnet): """ Return the flow logs that cover a given subnet From fa454dcf4427ac4c9c89f5bc1201fb52d6288528 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 15:19:16 -0500 Subject: [PATCH 044/154] Y'all need PEP8 --- ScoutSuite/providers/aws/configs/services.py | 7 ++- .../providers/aws/services/awslambda.py | 2 +- .../providers/aws/services/cloudwatch.py | 4 +- ScoutSuite/providers/aws/services/ec2.py | 11 ++-- ScoutSuite/providers/aws/services/efs.py | 4 +- .../providers/aws/services/elasticache.py | 4 +- ScoutSuite/providers/aws/services/elb.py | 12 ++--- ScoutSuite/providers/aws/services/elbv2.py | 2 +- ScoutSuite/providers/aws/services/emr.py | 8 +-- ScoutSuite/providers/aws/services/iam.py | 50 ++++++++----------- ScoutSuite/providers/aws/services/rds.py | 13 ++--- ScoutSuite/providers/aws/services/redshift.py | 2 +- ScoutSuite/providers/aws/services/route53.py | 4 +- ScoutSuite/providers/aws/services/s3.py | 16 +++--- ScoutSuite/providers/aws/services/ses.py | 1 + ScoutSuite/providers/aws/services/vpc.py | 2 +- 16 files changed, 63 insertions(+), 79 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 4705bd665..2839ef90f 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -27,7 +27,10 @@ from ScoutSuite.providers.aws.services.dynamodb_private import DynamoDBConfig from ScoutSuite.providers.aws.services.kms_private import KMSConfig except ImportError: - pass + ConfigConfig = None + DynamoDBConfig = None + KMSConfig = None + class AWSServicesConfig(BaseServicesConfig): """ @@ -80,4 +83,4 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): pass def _is_provider(self, provider_name): - return provider_name == 'aws' \ No newline at end of file + return provider_name == 'aws' diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index 7eb9ff54b..02ac22568 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -3,7 +3,7 @@ Lambda-related classes and functions """ -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients +from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig ######################################## diff --git a/ScoutSuite/providers/aws/services/cloudwatch.py b/ScoutSuite/providers/aws/services/cloudwatch.py index 95ce45dc6..d9bf11acb 100644 --- a/ScoutSuite/providers/aws/services/cloudwatch.py +++ b/ScoutSuite/providers/aws/services/cloudwatch.py @@ -3,7 +3,7 @@ CloudWatch-related classes and functions """ -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients +from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig ######################################## @@ -27,7 +27,7 @@ def parse_alarm(self, global_params, region, alarm): alarm['name'] = alarm.pop('AlarmName') # Drop some data for k in ['AlarmConfigurationUpdatedTimestamp', 'StateReason', 'StateReasonData', 'StateUpdatedTimestamp']: - foo = alarm.pop(k) if k in alarm else None + alarm.pop(k) if k in alarm else None alarm_id = self.get_non_provider_id(alarm['arn']) self.alarms[alarm_id] = alarm diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 662c00066..27f33389c 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -84,7 +84,6 @@ def parse_image(self, global_params, region, image): :param global_params: Parameters shared for all regions :param region: Name of the AWS region - :param snapshot: Single image """ id = image['ImageId'] name = image['Name'] @@ -104,12 +103,8 @@ def parse_security_group(self, global_params, region, group): """ vpc_id = group['VpcId'] if 'VpcId' in group and group['VpcId'] else ec2_classic manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) - security_group = {} - security_group['name'] = group['GroupName'] - security_group['id'] = group['GroupId'] - security_group['description'] = group['Description'] - security_group['owner_id'] = group['OwnerId'] - security_group['rules'] = {'ingress': {}, 'egress': {}} + security_group = {'name': group['GroupName'], 'id': group['GroupId'], 'description': group['Description'], + 'owner_id': group['OwnerId'], 'rules': {'ingress': {}, 'egress': {}}} security_group['rules']['ingress']['protocols'], security_group['rules']['ingress'][ 'count'] = self.__parse_security_group_rules(group['IpPermissions']) security_group['rules']['egress']['protocols'], security_group['rules']['egress'][ @@ -209,7 +204,7 @@ def __init__(self, service_metadata, thread_config=4): ######################################## -##### EC2 analysis functions +# EC2 analysis functions ######################################## def analyze_ec2_config(ec2_info, aws_account_id, force_write): diff --git a/ScoutSuite/providers/aws/services/efs.py b/ScoutSuite/providers/aws/services/efs.py index ab7e3dd85..2c6568f6b 100644 --- a/ScoutSuite/providers/aws/services/efs.py +++ b/ScoutSuite/providers/aws/services/efs.py @@ -31,7 +31,7 @@ def parse_file_system(self, global_params, region, file_system): file_system['name'] = None # Get tags file_system['tags'] = \ - handle_truncated_response(api_clients[region].describe_tags, {'FileSystemId': fs_id}, ['Tags'])['Tags'] + handle_truncated_response(api_clients[region].describe_tags, {'FileSystemId': fs_id}, ['Tags'])['Tags'] # Get mount targets mount_targets = handle_truncated_response(api_clients[region].describe_mount_targets, {'FileSystemId': fs_id}, ['MountTargets'])['MountTargets'] @@ -41,7 +41,7 @@ def parse_file_system(self, global_params, region, file_system): file_system['mount_targets'][mt_id] = mt # Get security groups file_system['mount_targets'][mt_id]['security_groups'] = \ - api_clients[region].describe_mount_target_security_groups(MountTargetId=mt_id)['SecurityGroups'] + api_clients[region].describe_mount_target_security_groups(MountTargetId=mt_id)['SecurityGroups'] self.file_systems[fs_id] = file_system diff --git a/ScoutSuite/providers/aws/services/elasticache.py b/ScoutSuite/providers/aws/services/elasticache.py index 9af741cc0..3c642de0c 100644 --- a/ScoutSuite/providers/aws/services/elasticache.py +++ b/ScoutSuite/providers/aws/services/elasticache.py @@ -27,8 +27,8 @@ def parse_cluster(self, global_params, region, cluster): # Must fetch info about the subnet group to retrieve the VPC ID... if 'CacheSubnetGroupName' in cluster: subnet_group = \ - api_clients[region].describe_cache_subnet_groups(CacheSubnetGroupName=cluster['CacheSubnetGroupName'])[ - 'CacheSubnetGroups'][0] + api_clients[region].describe_cache_subnet_groups(CacheSubnetGroupName=cluster['CacheSubnetGroupName'])[ + 'CacheSubnetGroups'][0] vpc_id = subnet_group['VpcId'] else: vpc_id = ec2_classic diff --git a/ScoutSuite/providers/aws/services/elb.py b/ScoutSuite/providers/aws/services/elb.py index 5e32137be..ab81f48bf 100644 --- a/ScoutSuite/providers/aws/services/elb.py +++ b/ScoutSuite/providers/aws/services/elb.py @@ -20,13 +20,12 @@ class ELBRegionConfig(RegionConfig): def parse_elb(self, global_params, region, lb): """ + :param lb: :param global_params: :param region: - :param elb: :return: """ - elb = {} - elb['name'] = lb.pop('LoadBalancerName') + elb = {'name': lb.pop('LoadBalancerName')} vpc_id = lb['VPCId'] if 'VPCId' in lb and lb['VPCId'] else ec2_classic manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) get_keys(lb, elb, ['DNSName', 'CreatedTime', 'AvailabilityZones', 'Subnets', 'Scheme']) @@ -44,11 +43,12 @@ def parse_elb(self, global_params, region, lb): if policy_id not in self.elb_policies: policy_names.append(policy_name) elb['listeners'][l['Listener']['LoadBalancerPort']] = listener - # Fetch LB policies here. This is not ideal, but the alternative is to download all policies and clean up after... + # Fetch LB policies here. This is not ideal, but the alternative is to download all policies and clean up + # after... if len(policy_names): policies = \ - api_clients[region].describe_load_balancer_policies(LoadBalancerName=elb['name'], PolicyNames=policy_names)[ - 'PolicyDescriptions'] + api_clients[region].describe_load_balancer_policies(LoadBalancerName=elb['name'], + PolicyNames=policy_names)['PolicyDescriptions'] for policy in policies: policy['name'] = policy.pop('PolicyName') policy_id = self.get_non_provider_id(policy['name']) diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index f0166d781..a12da5aec 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -14,6 +14,7 @@ # ELBv2RegionConfig ######################################## +# noinspection PyBroadException class ELBv2RegionConfig(RegionConfig): """ ELBv2 configuration for a single AWS region @@ -26,7 +27,6 @@ def parse_lb(self, global_params, region, lb): :param global_params: :param region: - :param source: :return: """ lb['arn'] = lb.pop('LoadBalancerArn') diff --git a/ScoutSuite/providers/aws/services/emr.py b/ScoutSuite/providers/aws/services/emr.py index c7c1f7e63..0632407f5 100644 --- a/ScoutSuite/providers/aws/services/emr.py +++ b/ScoutSuite/providers/aws/services/emr.py @@ -6,7 +6,6 @@ from ScoutSuite.providers.aws.configs.vpc import VPCConfig - ######################################## # EMRRegionConfig ######################################## @@ -25,10 +24,11 @@ def parse_cluster(self, global_params, region, cluster): :param cluster: EMR cluster """ cluster_id = cluster['Id'] - cluster = api_clients[region].describe_cluster(ClusterId = cluster_id)['Cluster'] + cluster = api_clients[region].describe_cluster(ClusterId=cluster_id)['Cluster'] cluster['id'] = cluster.pop('Id') cluster['name'] = cluster.pop('Name') - vpc_id = 'TODO' # The EMR API won't disclose the VPC ID, so wait until all configs have been fetch and look up the VPC based on the subnet ID + vpc_id = 'TODO' # The EMR API won't disclose the VPC ID, so wait until all configs have been fetch and look + # up the VPC based on the subnet ID manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) self.vpcs[vpc_id].clusters[cluster_id] = cluster @@ -44,5 +44,5 @@ class EMRConfig(RegionalServiceConfig): region_config_class = EMRRegionConfig - def __init__(self, service_metadata, thread_config = 4): + def __init__(self, service_metadata, thread_config=4): super(EMRConfig, self).__init__(service_metadata, thread_config) diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index c0c4fe16c..68c73984a 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -8,6 +8,7 @@ from ScoutSuite.utils import * +# noinspection PyBroadException class IAMConfig(AWSBaseConfig): """ Object that holds the IAM configuration @@ -47,7 +48,7 @@ def __init__(self, target_config): super(IAMConfig, self).__init__(target_config) ######################################## - ##### Overload to fetch credentials report before and after + # Overload to fetch credentials report before and after ######################################## def fetch_all(self, credentials, regions=None, partition_name='aws', targets=None): @@ -59,7 +60,7 @@ def fetch_all(self, credentials, regions=None, partition_name='aws', targets=Non self.fetchstatuslogger.show(True) ######################################## - ##### Credential report + # Credential report ######################################## def fetch_credential_reports(self, credentials, ignore_exception=False): @@ -126,7 +127,7 @@ def _sanitize_date(self, date): return date if date != 'no_information' and date != 'N/A' else None ######################################## - ##### Groups + # Groups ######################################## def parse_groups(self, group, params): @@ -151,7 +152,7 @@ def parse_groups(self, group, params): self.groups[group['id']] = group ######################################## - ##### Managed policies + # Managed policies ######################################## def parse_policies(self, fetched_policy, params): @@ -159,10 +160,8 @@ def parse_policies(self, fetched_policy, params): Parse a single IAM policy and fetch additional information """ api_client = params['api_client'] - policy = {} - policy['name'] = fetched_policy.pop('PolicyName') - policy['id'] = fetched_policy.pop('PolicyId') - policy['arn'] = fetched_policy.pop('Arn') + policy = {'name': fetched_policy.pop('PolicyName'), 'id': fetched_policy.pop('PolicyId'), + 'arn': fetched_policy.pop('Arn')} # Download version and document policy_version = api_client.get_policy_version(PolicyArn=policy['arn'], VersionId=fetched_policy['DefaultVersionId']) @@ -184,7 +183,7 @@ def parse_policies(self, fetched_policy, params): self.policies[policy['id']] = policy ######################################## - ##### Password policy + # Password policy ######################################## def fetch_password_policy(self, credentials): @@ -209,30 +208,24 @@ def fetch_password_policy(self, credentials): except ClientError as e: if e.response['Error']['Code'] == 'NoSuchEntity': - self.password_policy = {} - self.password_policy[ - 'MinimumPasswordLength'] = '1' # As of 10/10/2016, 1-character passwords were authorized when no policy exists, even though the console displays 6 - self.password_policy['RequireUppercaseCharacters'] = False - self.password_policy['RequireLowercaseCharacters'] = False - self.password_policy['RequireNumbers'] = False - self.password_policy['RequireSymbols'] = False - self.password_policy['PasswordReusePrevention'] = False - self.password_policy['ExpirePasswords'] = False + self.password_policy = {'MinimumPasswordLength': '1', 'RequireUppercaseCharacters': False, + 'RequireLowercaseCharacters': False, 'RequireNumbers': False, + 'RequireSymbols': False, 'PasswordReusePrevention': False, + 'ExpirePasswords': False} else: raise e except Exception as e: printError(str(e)) ######################################## - ##### Roles + # Roles ######################################## def parse_roles(self, fetched_role, params): """ Parse a single IAM role and fetch additional data """ - role = {} - role['instances_count'] = 'N/A' + role = {'instances_count': 'N/A'} # When resuming upon throttling error, skip if already fetched if fetched_role['RoleName'] in self.roles: return @@ -263,7 +256,7 @@ def parse_roles(self, fetched_role, params): self.roles[role['id']] = role ######################################## - ##### Users + # Users ######################################## def parse_users(self, user, params): @@ -288,7 +281,7 @@ def parse_users(self, user, params): user['groups'].append(group['GroupName']) try: user['LoginProfile'] = api_client.get_login_profile(UserName=user['name'])['LoginProfile'] - except Exception as e: + except Exception: pass user['AccessKeys'] = api_client.list_access_keys(UserName=user['name'])['AccessKeyMetadata'] user['MFADevices'] = api_client.list_mfa_devices(UserName=user['name'])['MFADevices'] @@ -296,7 +289,7 @@ def parse_users(self, user, params): self.users[user['id']] = user ######################################## - ##### Finalize IAM config + # Finalize IAM config ######################################## def finalize(self): @@ -318,7 +311,7 @@ def finalize(self): super(IAMConfig, self).finalize() ######################################## - ##### Class helpers + # Class helpers ######################################## def get_id_for_resource(self, iam_resource_type, resource_name): @@ -334,15 +327,14 @@ def __fetch_group_users(self, api_client, group_name): return users ######################################## - ##### Inline policies + # Inline policies ######################################## def __get_inline_policies(self, api_client, iam_resource_type, resource_id, resource_name): fetched_policies = {} get_policy_method = getattr(api_client, 'get_' + iam_resource_type + '_policy') list_policy_method = getattr(api_client, 'list_' + iam_resource_type + '_policies') - args = {} - args[iam_resource_type.title() + 'Name'] = resource_name + args = {iam_resource_type.title() + 'Name': resource_name} try: policy_names = list_policy_method(**args)['PolicyNames'] except Exception as e: @@ -389,7 +381,7 @@ def __parse_statement(self, policy_name, statement, policy_type, iam_resource_ty # Condition condition = statement['Condition'] if 'Condition' in statement else None manage_dictionary(self.permissions, action_string, {}) - if iam_resource_type == None: + if iam_resource_type is None: return self.__parse_actions(effect, action_string, statement[action_string], resource_string, statement[resource_string], iam_resource_type, resource_name, policy_name, policy_type, diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index 70425fe68..57b8c5ef8 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -21,14 +21,13 @@ def parse_instance(self, global_params, region, dbi): """ Parse a single RDS instance + :param dbi: :param global_params: Parameters shared for all regions :param region: Name of the AWS region - :param instance: Instance """ vpc_id = dbi['DBSubnetGroup']['VpcId'] if 'DBSubnetGroup' in dbi and 'VpcId' in dbi['DBSubnetGroup'] and \ dbi['DBSubnetGroup']['VpcId'] else ec2_classic - instance = {} - instance['name'] = dbi.pop('DBInstanceIdentifier') + instance = {'name': dbi.pop('DBInstanceIdentifier')} for key in ['InstanceCreateTime', 'Engine', 'DBInstanceStatus', 'AutoMinorVersionUpgrade', 'DBInstanceClass', 'MultiAZ', 'Endpoint', 'BackupRetentionPeriod', 'PubliclyAccessible', 'StorageEncrypted', 'VpcSecurityGroups', 'DBSecurityGroups', 'DBParameterGroups', @@ -91,7 +90,7 @@ def parse_parameter_group(self, global_params, region, parameter_group): printError('Failed fetching DB parameters for %s' % parameter_group['name']) # Save parameter_group_id = self.get_non_provider_id(parameter_group['name']) - (self).parameter_groups[parameter_group_id] = parameter_group + self.parameter_groups[parameter_group_id] = parameter_group def parse_security_group(self, global_params, region, security_group): """ @@ -138,10 +137,8 @@ def get_security_groups_info(rds_client, region_info): def parse_security_group(group): - security_group = {} - security_group['name'] = group['DBSecurityGroupName'] - security_group['description'] = group['DBSecurityGroupDescription'] - security_group['ec2_groups'] = {} + security_group = {'name': group['DBSecurityGroupName'], 'description': group['DBSecurityGroupDescription'], + 'ec2_groups': {}} for grant in group['EC2SecurityGroups']: if 'EC2SecurityGroupId' in grant: group_id = grant.pop('EC2SecurityGroupId') diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index 505ea1848..224e92d20 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -59,9 +59,9 @@ def parse_security_group(self, global_params, region, security_group): """ Parse a single Redsfhit security group + :param security_group: :param global_params: Parameters shared for all regions :param region: Name of the AWS region - :param security)_group: Security group """ name = security_group.pop('ClusterSecurityGroupName') security_group['name'] = name diff --git a/ScoutSuite/providers/aws/services/route53.py b/ScoutSuite/providers/aws/services/route53.py index b7290af1e..4f4cd6680 100644 --- a/ScoutSuite/providers/aws/services/route53.py +++ b/ScoutSuite/providers/aws/services/route53.py @@ -20,7 +20,7 @@ def __init__(self, target_config): super(Route53DomainsConfig, self).__init__(target_config) ######################################## - ##### Domains + # Domains ######################################## def parse_domains(self, domain, params): """ @@ -50,7 +50,7 @@ def __init__(self, target_config): super(Route53Config, self).__init__(target_config) ######################################## - ##### hosted_zoness + # Hosted_zoness ######################################## def parse_hosted_zones(self, hosted_zone, params): """ diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index e7befa31e..7c209af46 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -70,9 +70,6 @@ def parse_buckets(self, bucket, params): get_s3_bucket_policy(api_client, bucket['name'], bucket) get_s3_bucket_secure_transport(api_client, bucket['name'], bucket) # If requested, get key properties - # if params['check_encryption'] or params['check_acls']: - # get_s3_bucket_keys(api_client, bucket['name'], bucket, params['check_encryption'], - # params['check_acls']) bucket['id'] = self.get_non_provider_id(bucket['name']) self.buckets[bucket['id']] = bucket @@ -156,8 +153,7 @@ def update_bucket_permissions(s3_info, iam_info, action, iam_entity, allowed_iam allowed_buckets.remove(bucket_name) elif bucket_name == '*': allowed_buckets = [] - policy_info = {} - policy_info[policy_type] = {} + policy_info = {policy_type: {}} policy_info[policy_type][policy_name] = \ iam_info['permissions']['Action'][action][iam_entity]['Allow'][allowed_iam_entity]['NotResource'][full_path][ policy_type][policy_name] @@ -261,13 +257,14 @@ def get_s3_bucket_secure_transport(api_client, bucket_name, bucket_info): return False +# noinspection PyBroadException def get_s3_bucket_versioning(api_client, bucket_name, bucket_info): try: versioning = api_client.get_bucket_versioning(Bucket=bucket_name) bucket_info['versioning_status_enabled'] = _status_to_bool(versioning.get('Status')) bucket_info['version_mfa_delete_enabled'] = _status_to_bool(versioning.get('MFADelete')) return True - except Exception as e: + except Exception: bucket_info['versioning_status_enabled'] = None bucket_info['version_mfa_delete_enabled'] = None return False @@ -294,12 +291,13 @@ def get_s3_bucket_logging(api_client, bucket_name, bucket_info): return False +# noinspection PyBroadException def get_s3_bucket_webhosting(api_client, bucket_name, bucket_info): try: result = api_client.get_bucket_website(Bucket=bucket_name) bucket_info['web_hosting_enabled'] = 'IndexDocument' in result return True - except Exception as e: + except Exception: # TODO: distinguish permission denied from 'NoSuchWebsiteConfiguration' errors bucket_info['web_hosting_enabled'] = False return False @@ -387,12 +385,10 @@ def get_s3_bucket_keys(api_client, bucket_name, bucket, check_encryption, check_ # noinspection PyBroadException try: key['grantees'] = get_s3_acls(api_client, bucket_name, bucket, key_name=key['name']) - except Exception as e: + except Exception: continue # Save it bucket['keys'].append(key) - # FIXME - commented for now as this method doesn't seem to be defined anywhere' - # update_status(key_count, bucket['keys_count'], 'keys') def get_s3_list_region(region): diff --git a/ScoutSuite/providers/aws/services/ses.py b/ScoutSuite/providers/aws/services/ses.py index 237139cba..f0af048a5 100644 --- a/ScoutSuite/providers/aws/services/ses.py +++ b/ScoutSuite/providers/aws/services/ses.py @@ -18,6 +18,7 @@ def parse_identitie(self, global_params, region, identity_name): """ Parse a single identity and fetch additional attributes + :param identity_name: :param global_params: Parameters shared for all regions :param region: Name of the AWS region """ diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index 461b8bee4..30c631e14 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -155,7 +155,7 @@ def __init__(self, service_metadata, thread_config): ######################################## -##### VPC analysis functions +# VPC analysis functions ######################################## known_cidrs = {'0.0.0.0/0': 'All'} From 55130e9a2d17b6aeaee11eb17aee44ad28215eb4 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 15:32:42 -0500 Subject: [PATCH 045/154] More cleaning for AWS --- ScoutSuite/providers/aws/services/cloudtrail.py | 5 ++--- ScoutSuite/providers/aws/services/directconnect.py | 2 +- ScoutSuite/providers/aws/services/ec2.py | 3 ++- ScoutSuite/providers/aws/services/elbv2.py | 1 + ScoutSuite/providers/aws/services/rds.py | 2 +- ScoutSuite/providers/aws/services/redshift.py | 6 ++---- ScoutSuite/providers/aws/services/s3.py | 14 +++++++------- 7 files changed, 16 insertions(+), 17 deletions(-) diff --git a/ScoutSuite/providers/aws/services/cloudtrail.py b/ScoutSuite/providers/aws/services/cloudtrail.py index 0f085e2e6..c6dcecd71 100644 --- a/ScoutSuite/providers/aws/services/cloudtrail.py +++ b/ScoutSuite/providers/aws/services/cloudtrail.py @@ -20,12 +20,11 @@ def parse_trail(self, global_params, region, trail): """ Parse a single CloudTrail trail + :param trail: :param global_params: Parameters shared for all regions :param region: Name of the AWS region - :param cluster: Trail """ - trail_config = {} - trail_config['name'] = trail.pop('Name') + trail_config = {'name': trail.pop('Name')} trail_id = self.get_non_provider_id(trail_config['name']) trail_details = None diff --git a/ScoutSuite/providers/aws/services/directconnect.py b/ScoutSuite/providers/aws/services/directconnect.py index 761e8d4c0..ba951a138 100644 --- a/ScoutSuite/providers/aws/services/directconnect.py +++ b/ScoutSuite/providers/aws/services/directconnect.py @@ -16,9 +16,9 @@ def parse_connection(self, global_params, region, connection): """ Parse a single connection and fetch additional attributes + :param connection: :param global_params: Parameters shared for all regions :param region: Name of the AWS region - :param connection_url: URL of the AWS connection """ connection['id'] = connection.pop('connectionId') connection['name'] = connection.pop('connectionName') diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 27f33389c..f46ccb5f4 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -47,9 +47,9 @@ def parse_instance(self, global_params, region, reservation): """ Parse a single EC2 instance + :param reservation: :param global_params: Parameters shared for all regions :param region: Name of the AWS region - :param instance: Cluster """ for i in reservation['Instances']: instance = {} @@ -82,6 +82,7 @@ def parse_image(self, global_params, region, image): """ Parses a single AMI (Amazon Machine Image) + :param image: :param global_params: Parameters shared for all regions :param region: Name of the AWS region """ diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index a12da5aec..de26cca3b 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -25,6 +25,7 @@ class ELBv2RegionConfig(RegionConfig): def parse_lb(self, global_params, region, lb): """ + :param lb: :param global_params: :param region: :return: diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index 57b8c5ef8..b530d99d7 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -96,9 +96,9 @@ def parse_security_group(self, global_params, region, security_group): """ Parse a single Redsfhit security group + :param security_group: :param global_params: Parameters shared for all regions :param region: Name of the AWS region - :param security)_group: Security group """ # vpc_id = security_group.pop('VpcId') if 'VpcId' in security_group else ec2_classic # manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index 224e92d20..29b8576a9 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -49,11 +49,9 @@ def parse_parameter_group(self, global_params, region, parameter_group): parameters = handle_truncated_response(api_client.describe_cluster_parameters, {'ParameterGroupName': pg_name}, ['Parameters'])['Parameters'] for parameter in parameters: - param = {} - param['value'] = parameter['ParameterValue'] - param['source'] = parameter['Source'] + param = {'value': parameter['ParameterValue'], 'source': parameter['Source']} parameter_group['parameters'][parameter['ParameterName']] = param - (self).parameter_groups[pg_id] = parameter_group + self.parameter_groups[pg_id] = parameter_group def parse_security_group(self, global_params, region, security_group): """ diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 7c209af46..682390ee9 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -82,10 +82,10 @@ def match_iam_policies_and_buckets(s3_info, iam_info): if 'Allow' in iam_info['permissions']['Action'][action][iam_entity]: for allowed_iam_entity in iam_info['permissions']['Action'][action][iam_entity]['Allow']: # For resource statements, we can easily rely on the existing permissions structure - if 'Resource' in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]: + if 'Resource' in \ + iam_info['permissions']['Action'][action][iam_entity]['Allow'][allowed_iam_entity]: for full_path in (x for x in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]['Resource'] if x.startswith('arn:aws:s3:') or x == '*'): + allowed_iam_entity]['Resource'] if x.startswith('arn:aws:s3:') or x == '*'): parts = full_path.split('/') bucket_name = parts[0].split(':')[-1] update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, @@ -94,14 +94,14 @@ def match_iam_policies_and_buckets(s3_info, iam_info): # For notresource statements, we must fetch the policy document to determine which buckets are # not protected if 'NotResource' in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]: + allowed_iam_entity]: for full_path in (x for x in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]['NotResource'] if x.startswith('arn:aws:s3:') or x == '*'): + allowed_iam_entity]['NotResource'] if x.startswith('arn:aws:s3:') or x == '*'): for policy_type in ['InlinePolicies', 'ManagedPolicies']: if policy_type in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]['NotResource'][full_path]: + allowed_iam_entity]['NotResource'][full_path]: for policy in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]['NotResource'][full_path][policy_type]: + allowed_iam_entity]['NotResource'][full_path][policy_type]: update_bucket_permissions(s3_info, iam_info, action, iam_entity, allowed_iam_entity, full_path, policy_type, policy) From f95352afba714b18d19f1aff0e7acf1dc08fae99 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 16:26:22 -0500 Subject: [PATCH 046/154] Moving stuff around --- ScoutSuite/__main__.py | 2 +- {opinel/utils => ScoutSuite/providers/aws}/credentials.py | 0 {opinel/utils => ScoutSuite/providers/aws}/profiles.py | 2 +- ScoutSuite/providers/aws/provider.py | 2 +- tests/test-scoutsuite.py | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename {opinel/utils => ScoutSuite/providers/aws}/credentials.py (100%) rename {opinel/utils => ScoutSuite/providers/aws}/profiles.py (98%) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 6f9b102de..1f2c402f7 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -7,7 +7,7 @@ import webbrowser from ScoutSuite.core.console import configPrintException, printInfo, printDebug -from opinel.utils.profiles import AWSProfiles +from ScoutSuite.providers.aws.profiles import AWSProfiles from ScoutSuite import AWSCONFIG from ScoutSuite.output.html import Scout2Report diff --git a/opinel/utils/credentials.py b/ScoutSuite/providers/aws/credentials.py similarity index 100% rename from opinel/utils/credentials.py rename to ScoutSuite/providers/aws/credentials.py diff --git a/opinel/utils/profiles.py b/ScoutSuite/providers/aws/profiles.py similarity index 98% rename from opinel/utils/profiles.py rename to ScoutSuite/providers/aws/profiles.py index f4714ce67..566532a45 100644 --- a/opinel/utils/profiles.py +++ b/ScoutSuite/providers/aws/profiles.py @@ -5,7 +5,7 @@ from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.core.console import printDebug -from opinel.utils.credentials import read_creds +from ScoutSuite.providers.aws.credentials import read_creds aws_dir = os.path.join(os.path.expanduser('~'), '.aws') aws_credentials_file = os.path.join(aws_dir, 'credentials') diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index c15cf2647..768d02372 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -4,7 +4,7 @@ import os from ScoutSuite.providers.aws.aws import get_aws_account_id -from opinel.utils.credentials import read_creds +from ScoutSuite.providers.aws.credentials import read_creds from ScoutSuite.core.console import printDebug, printError, printException, printInfo diff --git a/tests/test-scoutsuite.py b/tests/test-scoutsuite.py index 1af03246d..37b40b81b 100644 --- a/tests/test-scoutsuite.py +++ b/tests/test-scoutsuite.py @@ -3,7 +3,7 @@ import mock from nose.plugins.attrib import attr -from opinel.utils.credentials import read_creds_from_environment_variables +from ScoutSuite.providers.aws.credentials import read_creds_from_environment_variables from ScoutSuite.__main__ import * From e02dfa407d7cdc9a8e300a5aeacb0de92aba8241 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 16:41:23 -0500 Subject: [PATCH 047/154] Cleaning in credentials and profiles.py --- ScoutSuite/providers/aws/credentials.py | 69 ++++++++++++------------- ScoutSuite/providers/aws/profiles.py | 25 ++++----- 2 files changed, 42 insertions(+), 52 deletions(-) diff --git a/ScoutSuite/providers/aws/credentials.py b/ScoutSuite/providers/aws/credentials.py index 018caf7e5..183cdcc6d 100644 --- a/ScoutSuite/providers/aws/credentials.py +++ b/ScoutSuite/providers/aws/credentials.py @@ -8,7 +8,7 @@ import fileinput import os import re -import requests # TODO: get rid of that and make sure urllib2 validates certs ? +import requests # TODO: get rid of that and make sure urllib2 validates certs ? import string from ScoutSuite.core.console import printException, printError, printInfo @@ -16,7 +16,6 @@ from opinel.utils.fs import save_blob_as_json from ScoutSuite.providers.aws.aws import connect_service - ######################################## # Globals ######################################## @@ -42,7 +41,6 @@ re_mfa_serial = re.compile(mfa_serial) re_mfa_serial_format = re.compile(mfa_serial_format) - aws_config_dir = os.path.join(os.path.expanduser('~'), '.aws') aws_credentials_file = os.path.join(aws_config_dir, 'credentials') aws_credentials_file_tmp = os.path.join(aws_config_dir, 'credentials.tmp') @@ -54,7 +52,7 @@ ######################################## -def assume_role(role_name, credentials, role_arn, role_session_name, silent = False): +def assume_role(role_name, credentials, role_arn, role_session_name, silent=False): """ Assume role and save credentials @@ -67,29 +65,29 @@ def assume_role(role_name, credentials, role_arn, role_session_name, silent = Fa """ external_id = credentials.pop('ExternalId') if 'ExternalId' in credentials else None # Connect to STS - sts_client = connect_service('sts', credentials, silent = silent) + sts_client = connect_service('sts', credentials, silent=silent) # Set required arguments for assume role call sts_args = { - 'RoleArn': role_arn, - 'RoleSessionName': role_session_name + 'RoleArn': role_arn, + 'RoleSessionName': role_session_name } # MFA used ? if 'mfa_serial' in credentials and 'mfa_code' in credentials: - sts_args['TokenCode'] = credentials['mfa_code'] - sts_args['SerialNumber'] = credentials['mfa_serial'] + sts_args['TokenCode'] = credentials['mfa_code'] + sts_args['SerialNumber'] = credentials['mfa_serial'] # External ID used ? if external_id: - sts_args['ExternalId'] = external_id + sts_args['ExternalId'] = external_id # Assume the role sts_response = sts_client.assume_role(**sts_args) credentials = sts_response['Credentials'] cached_credentials_filename = get_cached_credentials_filename(role_name, role_arn) - #with open(cached_credentials_filename, 'wt+') as f: + # with open(cached_credentials_filename, 'wt+') as f: # write_data_to_file(f, sts_response, True, False) cached_credentials_path = os.path.dirname(cached_credentials_filename) if not os.path.isdir(cached_credentials_path): os.makedirs(cached_credentials_path) - save_blob_as_json(cached_credentials_filename, sts_response, True, False) # blob, force_write, debug): + save_blob_as_json(cached_credentials_filename, sts_response, True, False) # blob, force_write, debug): return credentials @@ -101,13 +99,13 @@ def get_cached_credentials_filename(role_name, role_arn): :param role_arn: :return: """ - filename_p1 = role_name.replace('/','-') + filename_p1 = role_name.replace('/', '-') filename_p2 = role_arn.replace('/', '-').replace(':', '_') return os.path.join(os.path.join(os.path.expanduser('~'), '.aws'), 'cli/cache/%s--%s.json' % (filename_p1, filename_p2)) -def get_profiles_from_aws_credentials_file(credentials_files = [aws_credentials_file, aws_config_file]): +def get_profiles_from_aws_credentials_file(credentials_files=[aws_credentials_file, aws_config_file]): """ :param credentials_files: @@ -148,11 +146,11 @@ def init_creds(): :return: """ - return { 'AccessKeyId': None, 'SecretAccessKey': None, 'SessionToken': None, - 'Expiration': None, 'SerialNumber': None, 'TokenCode': None } + return {'AccessKeyId': None, 'SecretAccessKey': None, 'SessionToken': None, + 'Expiration': None, 'SerialNumber': None, 'TokenCode': None} -def init_sts_session(profile_name, credentials, duration = 28800, session_name = None, save_creds = True): +def init_sts_session(profile_name, credentials, duration=28800, session_name=None, save_creds=True): """ Fetch STS credentials @@ -187,7 +185,7 @@ def init_sts_session(profile_name, credentials, duration = 28800, session_name = return sts_response['Credentials'] -def read_creds_from_aws_credentials_file(profile_name, credentials_file = aws_credentials_file): +def read_creds_from_aws_credentials_file(profile_name, credentials_file=aws_credentials_file): """ Read credentials from AWS config file @@ -253,6 +251,7 @@ def read_creds_from_csv(filename): return key_id, secret, mfa_serial +# noinspection PyBroadException def read_creds_from_ec2_instance_metadata(): """ Read credentials from EC2 instance metadata (IAM role) @@ -261,7 +260,7 @@ def read_creds_from_ec2_instance_metadata(): """ creds = init_creds() try: - has_role = requests.get('http://169.254.169.254/latest/meta-data/iam/security-credentials', timeout = 1) + has_role = requests.get('http://169.254.169.254/latest/meta-data/iam/security-credentials', timeout=1) if has_role.status_code == 200: iam_role = has_role.text credentials = requests.get('http://169.254.169.254/latest/meta-data/iam/security-credentials/%s/' % @@ -270,10 +269,11 @@ def read_creds_from_ec2_instance_metadata(): creds[c] = credentials[c] creds['SessionToken'] = credentials['Token'] return creds - except Exception as e: + except Exception: return False +# noinspection PyBroadException def read_creds_from_ecs_container_metadata(): """ Read credentials from ECS instance metadata (IAM role) @@ -283,12 +283,12 @@ def read_creds_from_ecs_container_metadata(): creds = init_creds() try: ecs_metadata_relative_uri = os.environ['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] - credentials = requests.get('http://169.254.170.2' + ecs_metadata_relative_uri, timeout = 1).json() + credentials = requests.get('http://169.254.170.2' + ecs_metadata_relative_uri, timeout=1).json() for c in ['AccessKeyId', 'SecretAccessKey']: creds[c] = credentials[c] creds['SessionToken'] = credentials['Token'] return creds - except Exception as e: + except Exception: return False @@ -319,7 +319,7 @@ def read_profile_from_environment_variables(): return role_arn, external_id -def read_profile_from_aws_config_file(profile_name, config_file = aws_config_file): +def read_profile_from_aws_config_file(profile_name, config_file=aws_config_file): """ Read profiles from AWS config file @@ -358,7 +358,7 @@ def read_profile_from_aws_config_file(profile_name, config_file = aws_config_fil return role_arn, source_profile, mfa_serial, external_id -def show_profiles_from_aws_credentials_file(credentials_files = [aws_credentials_file, aws_config_file]): +def show_profiles_from_aws_credentials_file(credentials_files=[aws_credentials_file, aws_config_file]): """ Show profile names from ~/.aws/credentials @@ -370,7 +370,7 @@ def show_profiles_from_aws_credentials_file(credentials_files = [aws_credentials printInfo(' * %s' % profile) -def write_creds_to_aws_credentials_file(profile_name, credentials, credentials_file = aws_credentials_file): +def write_creds_to_aws_credentials_file(profile_name, credentials, credentials_file=aws_credentials_file): """ Write credentials to AWS config file @@ -382,9 +382,7 @@ def write_creds_to_aws_credentials_file(profile_name, credentials, credentials_f profile_found = False profile_ever_found = False session_token_written = False - security_token_written = False mfa_serial_written = False - expiration_written = False # Create the .aws folder if needed if not os.path.isdir(aws_config_dir): os.mkdir(aws_config_dir) @@ -414,10 +412,8 @@ def write_creds_to_aws_credentials_file(profile_name, credentials, credentials_f session_token_written = True elif re_security_token.match(line) and 'SessionToken' in credentials and credentials['SessionToken']: print('aws_security_token = %s' % credentials['SessionToken']) - security_token_written = True elif re_expiration.match(line) and 'Expiration' in credentials and credentials['Expiration']: print('expiration = %s' % credentials['Expiration']) - expiration_written = True else: print(line.rstrip()) else: @@ -454,13 +450,14 @@ def complete_profile(f, credentials, session_token_written, mfa_serial_written): if mfa_serial and not mfa_serial_written: f.write('aws_mfa_serial = %s\n' % mfa_serial) + ######################################## # Main function ######################################## -def read_creds(profile_name, csv_file = None, mfa_serial_arg = None, mfa_code = None, force_init = False, - role_session_name = 'opinel'): +def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, force_init=False, + role_session_name='opinel'): """ Read credentials from anywhere (CSV, Environment, Instance metadata, config/credentials) @@ -503,7 +500,7 @@ def read_creds(profile_name, csv_file = None, mfa_serial_arg = None, mfa_code = # Scout2 issue 237 - credentials file may be used to configure role-based profiles... if not role_arn: role_arn, source_profile, role_mfa_serial, external_id = \ - read_profile_from_aws_config_file(profile_name, config_file = aws_credentials_file) + read_profile_from_aws_config_file(profile_name, config_file=aws_credentials_file) if role_arn: # Lookup cached credentials try: @@ -561,9 +558,9 @@ def read_creds(profile_name, csv_file = None, mfa_serial_arg = None, mfa_code = if 'AccessKeyId' in credentials and credentials['AccessKeyId']: credentials = init_sts_session(profile_name, credentials) # If we don't have valid creds by now, print an error message - if 'AccessKeyId' not in credentials or credentials['AccessKeyId'] == None or \ - 'SecretAccessKey' not in credentials or credentials['SecretAccessKey'] == None: + if 'AccessKeyId' not in credentials or credentials['AccessKeyId'] is None or \ + 'SecretAccessKey' not in credentials or credentials['SecretAccessKey'] is None: printError('Error: could not find AWS credentials. Use the --help option for more information.') - if not 'AccessKeyId' in credentials: - credentials = { 'AccessKeyId': None } + if 'AccessKeyId' not in credentials: + credentials = {'AccessKeyId': None} return credentials diff --git a/ScoutSuite/providers/aws/profiles.py b/ScoutSuite/providers/aws/profiles.py index 566532a45..d316abd17 100644 --- a/ScoutSuite/providers/aws/profiles.py +++ b/ScoutSuite/providers/aws/profiles.py @@ -13,9 +13,10 @@ re_profile_name = re.compile(r'(\[(profile\s+)?(.*?)\])') + class AWSProfile(object): - def __init__(self, filename = None, raw_profile = None, name = None, credentials = None, account_id = None): + def __init__(self, filename=None, raw_profile=None, name=None, credentials=None, account_id=None): self.filename = filename self.raw_profile = raw_profile self.name = name @@ -24,7 +25,6 @@ def __init__(self, filename = None, raw_profile = None, name = None, credentials if self.raw_profile: self.parse_raw_profile() - def get_credentials(self): # For now, use the existing code... self.credentials = read_creds(self.name) @@ -34,11 +34,9 @@ def get_credentials(self): pass return self.credentials - def set_attribute(self, attribute, value): self.attributes[attribute] = value - def parse_raw_profile(self): for line in self.raw_profile.split('\n')[1:]: line = line.strip() @@ -48,9 +46,8 @@ def parse_raw_profile(self): value = ''.join(values[1:]).strip() self.attributes[attribute] = value - def write(self): - tmp = AWSProfiles.get(self.name, quiet = True) + tmp = AWSProfiles.get(self.name, quiet=True) if not self.raw_profile: self.raw_profile = tmp[0].raw_profile if len(tmp) else None if not self.filename: @@ -84,11 +81,10 @@ def write(self): f.write(contents) - class AWSProfiles(object): @staticmethod - def list(names = []): + def list(names=[]): """ @brief @@ -96,9 +92,8 @@ def list(names = []): """ return [p.name for p in AWSProfiles.get(names)] - @staticmethod - def get(names = [], quiet = False): + def get(names=[], quiet=False): """ """ profiles = [] @@ -106,12 +101,11 @@ def get(names = [], quiet = False): profiles += AWSProfiles.find_profiles_in_file(aws_config_file, names, quiet) return profiles - @staticmethod - def find_profiles_in_file(filename, names = [], quiet = True): + def find_profiles_in_file(filename, names=[], quiet=True): profiles = [] if type(names) != list: - names = [ names ] + names = [names] if not quiet: printDebug('Searching for profiles matching %s in %s ... ' % (str(names), filename)) name_filters = [] @@ -130,11 +124,10 @@ def find_profiles_in_file(filename, names = [], quiet = True): matching_profile = True i1 = aws_credentials.index(profile[0]) if i < profile_count: - i2 = aws_credentials.index(existing_profiles[i+1][0]) + i2 = aws_credentials.index(existing_profiles[i + 1][0]) raw_profile = aws_credentials[i1:i2] else: raw_profile = aws_credentials[i1:] if len(name_filters) == 0 or matching_profile: - profiles.append(AWSProfile(filename = filename, raw_profile = raw_profile, name = profile[2])) + profiles.append(AWSProfile(filename=filename, raw_profile=raw_profile, name=profile[2])) return profiles - From 4b8716937341be904aa57496ed2d33c6383345fa Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 16:50:44 -0500 Subject: [PATCH 048/154] Getting rid of more warnings... --- ScoutSuite/providers/aws/credentials.py | 5 +++-- ScoutSuite/providers/aws/profiles.py | 13 ++++++++--- ScoutSuite/providers/aws/provider.py | 30 +++++++++---------------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/ScoutSuite/providers/aws/credentials.py b/ScoutSuite/providers/aws/credentials.py index 183cdcc6d..1c37c6205 100644 --- a/ScoutSuite/providers/aws/credentials.py +++ b/ScoutSuite/providers/aws/credentials.py @@ -456,6 +456,7 @@ def complete_profile(f, credentials, session_token_written, mfa_serial_written): ######################################## +# noinspection PyBroadException,PyBroadException def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, force_init=False, role_session_name='opinel'): """ @@ -515,9 +516,9 @@ def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, if expiration < current: print('Role\'s credentials have expired on %s' % credentials['Expiration']) credentials = oldcred - except Exception as e: + except Exception: pass - if not expiration or expiration < current or credentials['AccessKeyId'] == None: + if not expiration or expiration < current or credentials['AccessKeyId'] is None: if source_profile: credentials = read_creds(source_profile) if role_mfa_serial: diff --git a/ScoutSuite/providers/aws/profiles.py b/ScoutSuite/providers/aws/profiles.py index d316abd17..0f971179a 100644 --- a/ScoutSuite/providers/aws/profiles.py +++ b/ScoutSuite/providers/aws/profiles.py @@ -14,6 +14,7 @@ re_profile_name = re.compile(r'(\[(profile\s+)?(.*?)\])') +# noinspection PyBroadException class AWSProfile(object): def __init__(self, filename=None, raw_profile=None, name=None, credentials=None, account_id=None): @@ -84,25 +85,31 @@ def write(self): class AWSProfiles(object): @staticmethod - def list(names=[]): + def list(names=None): """ @brief :return: List of all profile names found in .aws/config and .aws/credentials """ + if names is None: + names = [] return [p.name for p in AWSProfiles.get(names)] @staticmethod - def get(names=[], quiet=False): + def get(names=None, quiet=False): """ """ + if names is None: + names = [] profiles = [] profiles += AWSProfiles.find_profiles_in_file(aws_credentials_file, names, quiet) profiles += AWSProfiles.find_profiles_in_file(aws_config_file, names, quiet) return profiles @staticmethod - def find_profiles_in_file(filename, names=[], quiet=True): + def find_profiles_in_file(filename, names=None, quiet=True): + if names is None: + names = [] profiles = [] if type(names) != list: names = [names] diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 768d02372..8250f7469 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -15,6 +15,7 @@ from ScoutSuite.utils import ec2_classic, manage_dictionary +# noinspection PyBroadException class AWSProvider(BaseProvider): """ Implements provider for AWS @@ -178,7 +179,8 @@ def _process_network_acls_check_for_allow_all(self, network_acl, direction): def _process_network_acls_check_for_aws_default(self, network_acl, direction): if len(network_acl['rules'][direction]) == 2 and int( network_acl['allow_all_%s_traffic' % direction]) > 0 and '100' in network_acl['rules'][direction]: - # Assume it is AWS' default rules because there are 2 rules (100 and 65535) and the first rule allows all traffic + # Assume it is AWS' default rules because there are 2 rules (100 and 65535) and the first rule allows all + # traffic network_acl['use_default_%s_rules' % direction] = True else: network_acl['use_default_%s_rules' % direction] = False @@ -234,7 +236,7 @@ def _match_iam_policies_and_buckets(self): for allowed_iam_entity in iam_info['permissions']['Action'][action][iam_entity]['Allow']: # For resource statements, we can easily rely on the existing permissions structure if 'Resource' in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]: + allowed_iam_entity]: for full_path in (x for x in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ allowed_iam_entity]['Resource'] if @@ -245,7 +247,8 @@ def _match_iam_policies_and_buckets(self): iam_info['permissions']['Action'][action][iam_entity][ 'Allow'][allowed_iam_entity]['Resource'][ full_path]) - # For notresource statements, we must fetch the policy document to determine which buckets are not protected + # For notresource statements, we must fetch the policy document to determine which + # buckets are not protected if 'NotResource' in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ allowed_iam_entity]: for full_path in (x for x in @@ -567,7 +570,7 @@ def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_l vpc_path = combine_paths(current_path[0:4], ['vpcs', attached_resource]) try: attached_vpc = get_object_at(self, vpc_path) - except Exception as e: + except Exception: printDebug( 'It appears that the flow log %s is attached to a resource that was previously deleted (%s).' % ( flow_log_id, attached_resource)) @@ -587,14 +590,6 @@ def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_l manage_dictionary(subnet, 'flow_logs', []) if flow_log_id not in subnet['flow_logs']: subnet['flow_logs'].append(flow_log_id) - # TODO this is pre-merge (from Loic) code - # all_vpcs = get_object_at(self, combine_paths(current_path[0:2], ['vpcs'])) - # for vpc in self.services['vpc']: - # if attached_resource in all_vpcs[vpc]['subnets']: - # manage_dictionary(all_vpcs[vpc]['subnets'][attached_resource], 'flow_logs', []) - # if flow_log_id not in all_vpcs[vpc]['subnets'][attached_resource]['flow_logs']: - # all_vpcs[vpc]['subnets'][attached_resource]['flow_logs'].append(flow_log_id) - # break else: printError('Resource %s attached to flow logs is not handled' % attached_resource) @@ -603,7 +598,7 @@ def get_db_attack_surface(self, current_config, path, current_path, db_id, callb service_config = self.services[service] manage_dictionary(service_config, 'external_attack_surface', {}) if (service == 'redshift' or service == 'rds') and 'PubliclyAccessible' in current_config and current_config[ - 'PubliclyAccessible']: + 'PubliclyAccessible']: public_dns = current_config['Endpoint']['Address'] listeners = [current_config['Endpoint']['Port']] security_groups = current_config['VpcSecurityGroups'] @@ -611,8 +606,8 @@ def get_db_attack_surface(self, current_config, path, current_path, db_id, callb current_path, [g['VpcSecurityGroupId'] for g in security_groups], listeners) elif 'ConfigurationEndpoint' in current_config: - public_dns = current_config['ConfigurationEndpoint']['Address'].replace('.cfg', - '') # TODO : get the proper addresss + # TODO : get the proper addresss + public_dns = current_config['ConfigurationEndpoint']['Address'].replace('.cfg', '') listeners = [current_config['ConfigurationEndpoint']['Port']] security_groups = current_config['SecurityGroups'] self._security_group_to_attack_surface(service_config['external_attack_surface'], public_dns, @@ -636,7 +631,6 @@ def get_lb_attack_surface(self, current_config, path, current_path, elb_id, call elb_config['external_attack_surface'][public_dns]['protocols'][protocol]['ports'][listener][ 'cidrs'].append({'CIDR': '0.0.0.0/0'}) elif current_path[1] == 'elbv2' and current_config['Scheme'] == 'internet-facing': - vpc_id = current_path[5] elb_config['external_attack_surface'][public_dns] = {'protocols': {}} security_groups = [g['GroupId'] for g in current_config['security_groups']] listeners = [] @@ -666,7 +660,6 @@ def _security_group_to_attack_surface(self, attack_surface_config, public_ip, cu sg_path.append('rules') sg_path.append('ingress') ingress_rules = get_object_at(self, sg_path) - public_ip_grants = {} for p in ingress_rules['protocols']: for port in ingress_rules['protocols'][p]['ports']: if len(listeners) == 0 and 'cidrs' in ingress_rules['protocols'][p]['ports'][port]: @@ -688,8 +681,7 @@ def _security_group_to_attack_surface(self, attack_surface_config, public_ip, cu else: port_min = port_max = int(port) for listener in listeners: - if (port_min and port_max) and \ - int(listener) > port_min and int(listener) < port_max and \ + if (port_min and port_max) and port_min < int(listener) < port_max and \ 'cidrs' in ingress_rules['protocols'][p]['ports'][port]: manage_dictionary(attack_surface_config[public_ip]['protocols'], p, {'ports': {}}) manage_dictionary(attack_surface_config[public_ip]['protocols'][p]['ports'], From 7553f2f1f3dc92bbdebe544a41f3fad41af1abd5 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 17:10:14 -0500 Subject: [PATCH 049/154] Removing unused parameters from credentials.py --- ScoutSuite/providers/aws/credentials.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/aws/credentials.py b/ScoutSuite/providers/aws/credentials.py index 1c37c6205..1887dee04 100644 --- a/ScoutSuite/providers/aws/credentials.py +++ b/ScoutSuite/providers/aws/credentials.py @@ -33,7 +33,7 @@ re_gov_region = re.compile(r'(.*?)-gov-(.*?)') re_cn_region = re.compile(r'^cn-(.*?)') -re_port_range = re.compile(r'(\d+)\-(\d+)') +re_port_range = re.compile(r"(\d+)-(\d+)") re_single_port = re.compile(r'(\d+)') mfa_serial = r'(aws_mfa_serial|mfa_serial)' @@ -105,13 +105,15 @@ def get_cached_credentials_filename(role_name, role_arn): (filename_p1, filename_p2)) -def get_profiles_from_aws_credentials_file(credentials_files=[aws_credentials_file, aws_config_file]): +def get_profiles_from_aws_credentials_file(credentials_files=None): """ :param credentials_files: :return: """ + if credentials_files is None: + credentials_files = [aws_credentials_file, aws_config_file] profiles = [] for filename in credentials_files: if os.path.isfile(filename): @@ -124,11 +126,10 @@ def get_profiles_from_aws_credentials_file(credentials_files=[aws_credentials_fi return sorted(profiles) -def generate_password(length=16): +def generate_password(): """ Generate a password using random characters from uppercase, lowercase, digits, and symbols - :param length: Length of the password to be generated :return: The random password """ chars = string.ascii_letters + string.digits + '!@#$%^&*()_+-=[]{};:,<.>?|' @@ -150,14 +151,13 @@ def init_creds(): 'Expiration': None, 'SerialNumber': None, 'TokenCode': None} -def init_sts_session(profile_name, credentials, duration=28800, session_name=None, save_creds=True): +def init_sts_session(profile_name, credentials, duration=28800, save_creds=True): """ Fetch STS credentials :param profile_name: :param credentials: :param duration: - :param session_name: :param save_creds: :return: """ From fff523c1805b814844c6152dade99823d9a47330 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 17:16:24 -0500 Subject: [PATCH 050/154] No more warnings in credentials.py --- ScoutSuite/providers/aws/credentials.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/credentials.py b/ScoutSuite/providers/aws/credentials.py index 1887dee04..7ddc1046b 100644 --- a/ScoutSuite/providers/aws/credentials.py +++ b/ScoutSuite/providers/aws/credentials.py @@ -185,6 +185,7 @@ def init_sts_session(profile_name, credentials, duration=28800, save_creds=True) return sts_response['Credentials'] +# noinspection PyTypeChecker def read_creds_from_aws_credentials_file(profile_name, credentials_file=aws_credentials_file): """ Read credentials from AWS config file @@ -358,13 +359,15 @@ def read_profile_from_aws_config_file(profile_name, config_file=aws_config_file) return role_arn, source_profile, mfa_serial, external_id -def show_profiles_from_aws_credentials_file(credentials_files=[aws_credentials_file, aws_config_file]): +def show_profiles_from_aws_credentials_file(credentials_files=None): """ Show profile names from ~/.aws/credentials :param credentials_files: :return: """ + if credentials_files is None: + credentials_files = [aws_credentials_file, aws_config_file] profiles = get_profiles_from_aws_credentials_file(credentials_files) for profile in set(profiles): printInfo(' * %s' % profile) @@ -471,6 +474,7 @@ def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, :return: """ + global sts_credentials, current first_sts_session = False source_profile = None role_mfa_serial = None From dabf930d1a55074cd832ea12db5ea7ebabcf89cb Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 17:23:02 -0500 Subject: [PATCH 051/154] No more warnings in profiles.py --- ScoutSuite/providers/aws/profiles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/aws/profiles.py b/ScoutSuite/providers/aws/profiles.py index 0f971179a..93be904a7 100644 --- a/ScoutSuite/providers/aws/profiles.py +++ b/ScoutSuite/providers/aws/profiles.py @@ -18,6 +18,7 @@ class AWSProfile(object): def __init__(self, filename=None, raw_profile=None, name=None, credentials=None, account_id=None): + self.credentials = credentials self.filename = filename self.raw_profile = raw_profile self.name = name From 74247731a4dcfebc7dd0e04c968fe2e50f09072b Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 17:38:38 -0500 Subject: [PATCH 052/154] Removed unused parameters in provider.py --- ScoutSuite/providers/aws/provider.py | 32 ++++++++++++---------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 8250f7469..5fb9ee420 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -119,8 +119,7 @@ def _add_last_snapshot_date_to_ec2_volumes(self): sorted_snapshots = sorted(completed_snapshots, key=lambda s: s['StartTime'], reverse=True) volume['LastSnapshotDate'] = sorted_snapshots[0]['StartTime'] if len(sorted_snapshots) > 0 else None - def add_security_group_name_to_ec2_grants_callback(self, current_config, path, current_path, ec2_grant, - callback_args): + def add_security_group_name_to_ec2_grants_callback(self, current_path, ec2_grant, callback_args): sg_id = ec2_grant['GroupId'] if sg_id in current_path: target = current_path[:(current_path.index(sg_id) + 1)] @@ -185,7 +184,7 @@ def _process_network_acls_check_for_aws_default(self, network_acl, direction): else: network_acl['use_default_%s_rules' % direction] = False - def list_ec2_network_attack_surface_callback(self, current_config, path, current_path, privateip_id, callback_args): + def list_ec2_network_attack_surface_callback(self, current_config, current_path): manage_dictionary(self.services['ec2'], 'external_attack_surface', {}) if 'Association' in current_config and current_config['Association']: public_ip = current_config['Association']['PublicIp'] @@ -219,7 +218,7 @@ def _map_all_subnets(self): {'map': subnet_map}) self.subnet_map = subnet_map - def map_resource(self, current_config, path, current_path, resource_id, callback_args): + def map_resource(self, current_path, resource_id, callback_args): if resource_id not in callback_args['map']: callback_args['map'][resource_id] = {'region': current_path[3]} if len(current_path) > 5: @@ -290,8 +289,7 @@ def _update_bucket_permissions(self, s3_info, iam_info, action, iam_entity, allo allowed_buckets.remove(bucket_name) elif bucket_name == '*': allowed_buckets = [] - policy_info = {} - policy_info[policy_type] = {} + policy_info = {policy_type: {}} policy_info[policy_type][policy_name] = \ iam_info['permissions']['Action'][action][iam_entity]['Allow'][allowed_iam_entity]['NotResource'][ full_path][ @@ -304,7 +302,7 @@ def _update_iam_permissions(self, s3_info, bucket_name, iam_entity, allowed_iam_ bucket = s3_info['buckets'][bucket_name] manage_dictionary(bucket, iam_entity, {}) manage_dictionary(bucket, iam_entity + '_count', 0) - if not allowed_iam_entity in bucket[iam_entity]: + if allowed_iam_entity not in bucket[iam_entity]: bucket[iam_entity][allowed_iam_entity] = {} bucket[iam_entity + '_count'] = bucket[iam_entity + '_count'] + 1 @@ -322,13 +320,13 @@ def _update_iam_permissions(self, s3_info, bucket_name, iam_entity, allowed_iam_ # Could be an error or cross-account access, ignore... pass - def match_network_acls_and_subnets_callback(self, current_config, path, current_path, acl_id, callback_args): + def match_network_acls_and_subnets_callback(self, current_config, current_path, acl_id): for association in current_config['Associations']: subnet_path = current_path[:-1] + ['subnets', association['SubnetId']] subnet = get_object_at(self, subnet_path) subnet['network_acl'] = acl_id - def match_instances_and_subnets_callback(self, current_config, path, current_path, instance_id, callback_args): + def match_instances_and_subnets_callback(self, current_config, instance_id): subnet_id = current_config['SubnetId'] if subnet_id: vpc = self.subnet_map[subnet_id] @@ -359,14 +357,13 @@ def _match_instances_and_roles(self): role_instances[instance_profile_id] iam_config['roles'][role_id]['instances_count'] += len(role_instances[instance_profile_id]) - def match_roles_and_cloudformation_stacks_callback(self, current_config, path, current_path, stack_id, - callback_args): + def match_roles_and_cloudformation_stacks_callback(self, current_config): if 'RoleARN' not in current_config: return role_arn = current_config.pop('RoleARN') current_config['iam_role'] = self._get_role_info('arn', role_arn) - def match_roles_and_vpc_flowlogs_callback(self, current_config, path, current_path, flowlog_id, callback_args): + def match_roles_and_vpc_flowlogs_callback(self, current_config): if 'DeliverLogsPermissionArn' not in current_config: return delivery_role_arn = current_config.pop('DeliverLogsPermissionArn') @@ -381,7 +378,7 @@ def _get_role_info(self, attribute_name, attribute_value): break return iam_role_info - def process_vpc_peering_connections_callback(self, current_config, path, current_path, pc_id, callback_args): + def process_vpc_peering_connections_callback(self, current_config, current_path, pc_id): # Create a list of peering connection IDs in each VPC info = 'AccepterVpcInfo' if current_config['AccepterVpcInfo'][ @@ -408,12 +405,11 @@ def process_vpc_peering_connections_callback(self, current_config, path, current else: current_config['peer_info']['name'] = current_config['peer_info']['OwnerId'] - def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id, - callback_args): + def match_security_groups_and_resources_callback(self, current_path, resource_id, callback_args): service = current_path[1] original_resource_path = combine_paths(copy.deepcopy(current_path), [resource_id]) resource = get_object_at(self, original_resource_path) - if not 'resource_id_path' in callback_args: + if 'resource_id_path' not in callback_args: resource_type = current_path[-1] resource_path = copy.deepcopy(current_path) resource_path.append(resource_id) @@ -593,7 +589,7 @@ def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_l else: printError('Resource %s attached to flow logs is not handled' % attached_resource) - def get_db_attack_surface(self, current_config, path, current_path, db_id, callback_args): + def get_db_attack_surface(self, current_config, current_path): service = current_path[1] service_config = self.services[service] manage_dictionary(service_config, 'external_attack_surface', {}) @@ -615,7 +611,7 @@ def get_db_attack_surface(self, current_config, path, current_path, db_id, callb listeners) # TODO :: Get Redis endpoint information - def get_lb_attack_surface(self, current_config, path, current_path, elb_id, callback_args): + def get_lb_attack_surface(self, current_config, current_path): public_dns = current_config['DNSName'] elb_config = self.services[current_path[1]] manage_dictionary(elb_config, 'external_attack_surface', {}) From d033fbb5afcb26904f31b9880dc1a5b0cdf91a0b Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 17:38:38 -0500 Subject: [PATCH 053/154] Revert "Removed unused parameters in provider.py" This reverts commit 74247731a4dcfebc7dd0e04c968fe2e50f09072b. --- ScoutSuite/providers/aws/provider.py | 32 ++++++++++++++++------------ 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 5fb9ee420..8250f7469 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -119,7 +119,8 @@ def _add_last_snapshot_date_to_ec2_volumes(self): sorted_snapshots = sorted(completed_snapshots, key=lambda s: s['StartTime'], reverse=True) volume['LastSnapshotDate'] = sorted_snapshots[0]['StartTime'] if len(sorted_snapshots) > 0 else None - def add_security_group_name_to_ec2_grants_callback(self, current_path, ec2_grant, callback_args): + def add_security_group_name_to_ec2_grants_callback(self, current_config, path, current_path, ec2_grant, + callback_args): sg_id = ec2_grant['GroupId'] if sg_id in current_path: target = current_path[:(current_path.index(sg_id) + 1)] @@ -184,7 +185,7 @@ def _process_network_acls_check_for_aws_default(self, network_acl, direction): else: network_acl['use_default_%s_rules' % direction] = False - def list_ec2_network_attack_surface_callback(self, current_config, current_path): + def list_ec2_network_attack_surface_callback(self, current_config, path, current_path, privateip_id, callback_args): manage_dictionary(self.services['ec2'], 'external_attack_surface', {}) if 'Association' in current_config and current_config['Association']: public_ip = current_config['Association']['PublicIp'] @@ -218,7 +219,7 @@ def _map_all_subnets(self): {'map': subnet_map}) self.subnet_map = subnet_map - def map_resource(self, current_path, resource_id, callback_args): + def map_resource(self, current_config, path, current_path, resource_id, callback_args): if resource_id not in callback_args['map']: callback_args['map'][resource_id] = {'region': current_path[3]} if len(current_path) > 5: @@ -289,7 +290,8 @@ def _update_bucket_permissions(self, s3_info, iam_info, action, iam_entity, allo allowed_buckets.remove(bucket_name) elif bucket_name == '*': allowed_buckets = [] - policy_info = {policy_type: {}} + policy_info = {} + policy_info[policy_type] = {} policy_info[policy_type][policy_name] = \ iam_info['permissions']['Action'][action][iam_entity]['Allow'][allowed_iam_entity]['NotResource'][ full_path][ @@ -302,7 +304,7 @@ def _update_iam_permissions(self, s3_info, bucket_name, iam_entity, allowed_iam_ bucket = s3_info['buckets'][bucket_name] manage_dictionary(bucket, iam_entity, {}) manage_dictionary(bucket, iam_entity + '_count', 0) - if allowed_iam_entity not in bucket[iam_entity]: + if not allowed_iam_entity in bucket[iam_entity]: bucket[iam_entity][allowed_iam_entity] = {} bucket[iam_entity + '_count'] = bucket[iam_entity + '_count'] + 1 @@ -320,13 +322,13 @@ def _update_iam_permissions(self, s3_info, bucket_name, iam_entity, allowed_iam_ # Could be an error or cross-account access, ignore... pass - def match_network_acls_and_subnets_callback(self, current_config, current_path, acl_id): + def match_network_acls_and_subnets_callback(self, current_config, path, current_path, acl_id, callback_args): for association in current_config['Associations']: subnet_path = current_path[:-1] + ['subnets', association['SubnetId']] subnet = get_object_at(self, subnet_path) subnet['network_acl'] = acl_id - def match_instances_and_subnets_callback(self, current_config, instance_id): + def match_instances_and_subnets_callback(self, current_config, path, current_path, instance_id, callback_args): subnet_id = current_config['SubnetId'] if subnet_id: vpc = self.subnet_map[subnet_id] @@ -357,13 +359,14 @@ def _match_instances_and_roles(self): role_instances[instance_profile_id] iam_config['roles'][role_id]['instances_count'] += len(role_instances[instance_profile_id]) - def match_roles_and_cloudformation_stacks_callback(self, current_config): + def match_roles_and_cloudformation_stacks_callback(self, current_config, path, current_path, stack_id, + callback_args): if 'RoleARN' not in current_config: return role_arn = current_config.pop('RoleARN') current_config['iam_role'] = self._get_role_info('arn', role_arn) - def match_roles_and_vpc_flowlogs_callback(self, current_config): + def match_roles_and_vpc_flowlogs_callback(self, current_config, path, current_path, flowlog_id, callback_args): if 'DeliverLogsPermissionArn' not in current_config: return delivery_role_arn = current_config.pop('DeliverLogsPermissionArn') @@ -378,7 +381,7 @@ def _get_role_info(self, attribute_name, attribute_value): break return iam_role_info - def process_vpc_peering_connections_callback(self, current_config, current_path, pc_id): + def process_vpc_peering_connections_callback(self, current_config, path, current_path, pc_id, callback_args): # Create a list of peering connection IDs in each VPC info = 'AccepterVpcInfo' if current_config['AccepterVpcInfo'][ @@ -405,11 +408,12 @@ def process_vpc_peering_connections_callback(self, current_config, current_path, else: current_config['peer_info']['name'] = current_config['peer_info']['OwnerId'] - def match_security_groups_and_resources_callback(self, current_path, resource_id, callback_args): + def match_security_groups_and_resources_callback(self, current_config, path, current_path, resource_id, + callback_args): service = current_path[1] original_resource_path = combine_paths(copy.deepcopy(current_path), [resource_id]) resource = get_object_at(self, original_resource_path) - if 'resource_id_path' not in callback_args: + if not 'resource_id_path' in callback_args: resource_type = current_path[-1] resource_path = copy.deepcopy(current_path) resource_path.append(resource_id) @@ -589,7 +593,7 @@ def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_l else: printError('Resource %s attached to flow logs is not handled' % attached_resource) - def get_db_attack_surface(self, current_config, current_path): + def get_db_attack_surface(self, current_config, path, current_path, db_id, callback_args): service = current_path[1] service_config = self.services[service] manage_dictionary(service_config, 'external_attack_surface', {}) @@ -611,7 +615,7 @@ def get_db_attack_surface(self, current_config, current_path): listeners) # TODO :: Get Redis endpoint information - def get_lb_attack_surface(self, current_config, current_path): + def get_lb_attack_surface(self, current_config, path, current_path, elb_id, callback_args): public_dns = current_config['DNSName'] elb_config = self.services[current_path[1]] manage_dictionary(elb_config, 'external_attack_surface', {}) From ae6db8e70c47e558b5880fac505c04f8428c3c53 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 18:14:25 -0500 Subject: [PATCH 054/154] More small fixes --- ScoutSuite/providers/aws/provider.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 8250f7469..4e2c23da3 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -250,7 +250,7 @@ def _match_iam_policies_and_buckets(self): # For notresource statements, we must fetch the policy document to determine which # buckets are not protected if 'NotResource' in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ - allowed_iam_entity]: + allowed_iam_entity]: for full_path in (x for x in iam_info['permissions']['Action'][action][iam_entity]['Allow'][ allowed_iam_entity]['NotResource'] if @@ -290,8 +290,7 @@ def _update_bucket_permissions(self, s3_info, iam_info, action, iam_entity, allo allowed_buckets.remove(bucket_name) elif bucket_name == '*': allowed_buckets = [] - policy_info = {} - policy_info[policy_type] = {} + policy_info = {policy_type: {}} policy_info[policy_type][policy_name] = \ iam_info['permissions']['Action'][action][iam_entity]['Allow'][allowed_iam_entity]['NotResource'][ full_path][ @@ -413,7 +412,7 @@ def match_security_groups_and_resources_callback(self, current_config, path, cur service = current_path[1] original_resource_path = combine_paths(copy.deepcopy(current_path), [resource_id]) resource = get_object_at(self, original_resource_path) - if not 'resource_id_path' in callback_args: + if 'resource_id_path' not in callback_args: resource_type = current_path[-1] resource_path = copy.deepcopy(current_path) resource_path.append(resource_id) @@ -459,7 +458,7 @@ def match_security_groups_and_resources_callback(self, current_config, path, cur manage_dictionary(sg['used_by'][service]['resource_type'], resource_type, {} if resource_status else []) if resource_status: manage_dictionary(sg['used_by'][service]['resource_type'][resource_type], resource_status, []) - if not resource_id in sg['used_by'][service]['resource_type'][resource_type][resource_status]: + if resource_id not in sg['used_by'][service]['resource_type'][resource_type][resource_status]: sg['used_by'][service]['resource_type'][resource_type][resource_status].append(resource_id) else: sg['used_by'][service]['resource_type'][resource_type].append(resource_id) From 16c61469a77beb42133ffb34f27cd799bfe5d46e Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 18:33:27 -0500 Subject: [PATCH 055/154] Made some methods static in credentials.py --- ScoutSuite/providers/aws/provider.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 4e2c23da3..6071eeeca 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -136,7 +136,8 @@ def add_security_group_name_to_ec2_grants_callback(self, current_config, path, c target.append(sg_id) ec2_grant['GroupName'] = get_value_at(self.services['ec2'], target, 'name') - def _process_cloudtrail_trails(self, cloudtrail_config): + @staticmethod + def _process_cloudtrail_trails(cloudtrail_config): printInfo('Processing CloudTrail config...') global_events_logging = [] data_logging_trails_count = 0 @@ -164,7 +165,8 @@ def process_network_acls_callback(self, current_config, path, current_path, priv self._process_network_acls_check_for_aws_default(current_config, 'ingress') self._process_network_acls_check_for_aws_default(current_config, 'egress') - def _process_network_acls_check_for_allow_all(self, network_acl, direction): + @staticmethod + def _process_network_acls_check_for_allow_all(network_acl, direction): network_acl['allow_all_%s_traffic' % direction] = 0 for rule_number in network_acl['rules'][direction]: rule = network_acl['rules'][direction][rule_number] @@ -176,7 +178,8 @@ def _process_network_acls_check_for_allow_all(self, network_acl, direction): network_acl['allow_all_%s_traffic' % direction] = rule_number break - def _process_network_acls_check_for_aws_default(self, network_acl, direction): + @staticmethod + def _process_network_acls_check_for_aws_default(network_acl, direction): if len(network_acl['rules'][direction]) == 2 and int( network_acl['allow_all_%s_traffic' % direction]) > 0 and '100' in network_acl['rules'][direction]: # Assume it is AWS' default rules because there are 2 rules (100 and 65535) and the first rule allows all @@ -219,7 +222,8 @@ def _map_all_subnets(self): {'map': subnet_map}) self.subnet_map = subnet_map - def map_resource(self, current_config, path, current_path, resource_id, callback_args): + @staticmethod + def map_resource(current_config, path, current_path, resource_id, callback_args): if resource_id not in callback_args['map']: callback_args['map'][resource_id] = {'region': current_path[3]} if len(current_path) > 5: @@ -303,7 +307,7 @@ def _update_iam_permissions(self, s3_info, bucket_name, iam_entity, allowed_iam_ bucket = s3_info['buckets'][bucket_name] manage_dictionary(bucket, iam_entity, {}) manage_dictionary(bucket, iam_entity + '_count', 0) - if not allowed_iam_entity in bucket[iam_entity]: + if allowed_iam_entity not in bucket[iam_entity]: bucket[iam_entity][allowed_iam_entity] = {} bucket[iam_entity + '_count'] = bucket[iam_entity + '_count'] + 1 @@ -537,7 +541,8 @@ def _parse_elb_policies(self): # if 'elbv2' in self.config['services']: # Do something too here... - def parse_elb_policies_callback(self, current_config, path, current_path, region_id, callback_args): + @staticmethod + def parse_elb_policies_callback(current_config, path, current_path, region_id, callback_args): region_config = get_object_at(['services', 'elb', ] + current_path + [region_id]) region_config['elb_policies'] = current_config['elb_policies'] for policy_id in region_config['elb_policies']: From e367d5856db348275f1925db36e83427df851bf8 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 18:38:28 -0500 Subject: [PATCH 056/154] Solved yet more warnings --- ScoutSuite/providers/aws/configs/regions.py | 10 +++++++--- ScoutSuite/providers/aws/configs/services.py | 1 + ScoutSuite/providers/aws/provider.py | 3 +++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index fc9293c0b..3b141af3c 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -53,6 +53,7 @@ def __init__(self, service_metadata=None, thread_config=4): self.regions = {} self.thread_config = thread_configs[thread_config] self.service = re.sub(r'Config$', "", type(self).__name__).lower() + self.fetchstatuslogger = None # Booleans that define if threads should keep running self.run_q_threads = True @@ -75,7 +76,7 @@ def __init__(self, service_metadata=None, thread_config=4): continue params = resource_metadata['params'] if 'params' in resource_metadata else {} ignore_exceptions = True if 'no_exceptions' in resource_metadata and \ - resource_metadata['no_exceptions'] == True else False + resource_metadata['no_exceptions'] else False if not only_first_region: self.targets['other_regions'] += ((resource, resource_metadata['response'], @@ -158,7 +159,8 @@ def fetch_all(self, credentials, regions=None, partition_name='aws', targets=Non for j in range(self.thread_config['list']): qr.put(None) - def _init_threading(self, function, params=None, num_threads=10): + @staticmethod + def _init_threading(function, params=None, num_threads=10): """ Initialize queue and threads @@ -257,11 +259,13 @@ class RegionConfig(BaseConfig): Base class for ... """ - def __init__(self, region_name, resource_types=None): + def __init__(self, region_name, resource_types=None, **kwargs): + super().__init__(**kwargs) resource_types = {} if resource_types is None else resource_types self.region = region_name self.name = region_name self.id = region_name + self.fetchstatuslogger = None for resource_type in resource_types['region'] + resource_types['global']: setattr(self, resource_type, {}) setattr(self, '%s_count' % resource_type, 0) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 2839ef90f..6ed498d85 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -53,6 +53,7 @@ class AWSServicesConfig(BaseServicesConfig): def __init__(self, metadata=None, thread_config=4, **kwargs): + super().__init__(metadata, thread_config) self.cloudformation = CloudFormationConfig(metadata['management']['cloudformation'], thread_config) self.cloudtrail = CloudTrailConfig(metadata['management']['cloudtrail'], thread_config) self.cloudwatch = CloudWatchConfig(metadata['management']['cloudwatch'], thread_config) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 6071eeeca..7fdb64578 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -274,6 +274,7 @@ def _match_iam_policies_and_buckets(self): def _update_bucket_permissions(self, s3_info, iam_info, action, iam_entity, allowed_iam_entity, full_path, policy_type, policy_name): + global policy allowed_buckets = [] # By default, all buckets are allowed for bucket_name in s3_info['buckets']: @@ -522,6 +523,7 @@ def set_emr_vpc_ids_callback(self, current_config, path, current_path, vpc_id, c printError('Unable to determine VPC id for %s' % (str(subnet_id) if subnet_id else str(sg_id))) continue if vpc_id: + # noinspection PyArgumentList region_vpcs_config = get_object_at(current_path) manage_dictionary(region_vpcs_config, vpc_id, {'clusters': {}}) region_vpcs_config[vpc_id]['clusters'][cluster_id] = cluster @@ -541,6 +543,7 @@ def _parse_elb_policies(self): # if 'elbv2' in self.config['services']: # Do something too here... + # noinspection PyArgumentList @staticmethod def parse_elb_policies_callback(current_config, path, current_path, region_id, callback_args): region_config = get_object_at(['services', 'elb', ] + current_path + [region_id]) From 1b655264911f8ff278ee4d553bbe927551825347 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Fri, 22 Feb 2019 18:52:46 -0500 Subject: [PATCH 057/154] Optimized imports in AWS files --- ScoutSuite/providers/aws/aws.py | 5 +++-- ScoutSuite/providers/aws/configs/base.py | 3 +-- ScoutSuite/providers/aws/credentials.py | 11 ++++++----- ScoutSuite/providers/aws/profiles.py | 2 +- ScoutSuite/providers/aws/provider.py | 8 +++----- ScoutSuite/providers/aws/services/directconnect.py | 2 +- ScoutSuite/providers/aws/services/ec2.py | 10 +++++----- ScoutSuite/providers/aws/services/emr.py | 3 +-- ScoutSuite/providers/aws/services/iam.py | 4 ++-- ScoutSuite/providers/aws/services/rds.py | 3 +-- ScoutSuite/providers/aws/services/s3.py | 6 +++--- ScoutSuite/providers/aws/services/sns.py | 3 +-- ScoutSuite/providers/aws/services/vpc.py | 10 +++++----- 13 files changed, 33 insertions(+), 37 deletions(-) diff --git a/ScoutSuite/providers/aws/aws.py b/ScoutSuite/providers/aws/aws.py index 52bbc702a..4355f8d9f 100644 --- a/ScoutSuite/providers/aws/aws.py +++ b/ScoutSuite/providers/aws/aws.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- +import time +from collections import Counter + import boto3 from botocore.session import Session -from collections import Counter -import time from ScoutSuite.core.console import printInfo, printException diff --git a/ScoutSuite/providers/aws/configs/base.py b/ScoutSuite/providers/aws/configs/base.py index fcd6f4e4a..29d7c86f0 100644 --- a/ScoutSuite/providers/aws/configs/base.py +++ b/ScoutSuite/providers/aws/configs/base.py @@ -6,9 +6,8 @@ except ImportError: from queue import Queue -from ScoutSuite.providers.base.configs.base import BaseConfig - from ScoutSuite.providers.aws.aws import handle_truncated_response +from ScoutSuite.providers.base.configs.base import BaseConfig class AWSBaseConfig(BaseConfig): diff --git a/ScoutSuite/providers/aws/credentials.py b/ScoutSuite/providers/aws/credentials.py index 7ddc1046b..aa566f352 100644 --- a/ScoutSuite/providers/aws/credentials.py +++ b/ScoutSuite/providers/aws/credentials.py @@ -1,20 +1,21 @@ # Import future print from __future__ import print_function -import boto3 import datetime -import dateutil.parser -import json import fileinput +import json import os import re -import requests # TODO: get rid of that and make sure urllib2 validates certs ? import string +import boto3 +import dateutil.parser +import requests # TODO: get rid of that and make sure urllib2 validates certs ? + from ScoutSuite.core.console import printException, printError, printInfo from ScoutSuite.core.console import prompt_4_mfa_code -from opinel.utils.fs import save_blob_as_json from ScoutSuite.providers.aws.aws import connect_service +from opinel.utils.fs import save_blob_as_json ######################################## # Globals diff --git a/ScoutSuite/providers/aws/profiles.py b/ScoutSuite/providers/aws/profiles.py index 93be904a7..057e4b98b 100644 --- a/ScoutSuite/providers/aws/profiles.py +++ b/ScoutSuite/providers/aws/profiles.py @@ -3,8 +3,8 @@ import os import re -from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.core.console import printDebug +from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.providers.aws.credentials import read_creds aws_dir = os.path.join(os.path.expanduser('~'), '.aws') diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index 7fdb64578..eb50bc268 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -3,14 +3,12 @@ import copy import os -from ScoutSuite.providers.aws.aws import get_aws_account_id -from ScoutSuite.providers.aws.credentials import read_creds - from ScoutSuite.core.console import printDebug, printError, printException, printInfo - +from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.providers.aws.configs.services import AWSServicesConfig -from ScoutSuite.providers.base.configs.browser import combine_paths, get_object_at, get_value_at +from ScoutSuite.providers.aws.credentials import read_creds from ScoutSuite.providers.aws.services.vpc import put_cidr_name +from ScoutSuite.providers.base.configs.browser import combine_paths, get_object_at, get_value_at from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.utils import ec2_classic, manage_dictionary diff --git a/ScoutSuite/providers/aws/services/directconnect.py b/ScoutSuite/providers/aws/services/directconnect.py index ba951a138..3320c50af 100644 --- a/ScoutSuite/providers/aws/services/directconnect.py +++ b/ScoutSuite/providers/aws/services/directconnect.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients +from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig ######################################## diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index f46ccb5f4..09fbce283 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -3,18 +3,18 @@ EC2-related classes and functions """ +import base64 + # TODO: move a lot of this to VPCconfig, and use some sort of filter to only list SGs in EC2 classic import netaddr -import base64 -from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.core.console import printException, printInfo -from opinel.utils.fs import load_data - +from ScoutSuite.providers.aws.aws import get_name +from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.providers.base.configs.browser import get_attribute_at from ScoutSuite.utils import get_keys, ec2_classic, manage_dictionary -from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients +from opinel.utils.fs import load_data ######################################## # Globals diff --git a/ScoutSuite/providers/aws/services/emr.py b/ScoutSuite/providers/aws/services/emr.py index 0632407f5..814cbfdc3 100644 --- a/ScoutSuite/providers/aws/services/emr.py +++ b/ScoutSuite/providers/aws/services/emr.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- -from ScoutSuite.utils import manage_dictionary - from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig +from ScoutSuite.utils import manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index 68c73984a..c113f5cd5 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- from botocore.exceptions import ClientError -from ScoutSuite.providers.aws.aws import connect_service, handle_truncated_response -from ScoutSuite.core.console import printError, printException +from ScoutSuite.core.console import printError, printException +from ScoutSuite.providers.aws.aws import connect_service, handle_truncated_response from ScoutSuite.providers.aws.configs.base import AWSBaseConfig from ScoutSuite.utils import * diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index b530d99d7..96a0f75d7 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- -from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.core.console import printError, printException - +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.utils import ec2_classic, manage_dictionary diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 682390ee9..0642cae1b 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -6,11 +6,11 @@ import json from botocore.exceptions import ClientError -from ScoutSuite.providers.aws.aws import handle_truncated_response -from ScoutSuite.core.console import printError, printException, printInfo -from ScoutSuite.utils import manage_dictionary +from ScoutSuite.core.console import printError, printException, printInfo +from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.base import AWSBaseConfig +from ScoutSuite.utils import manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/sns.py b/ScoutSuite/providers/aws/services/sns.py index 511a5282e..8dce5e096 100644 --- a/ScoutSuite/providers/aws/services/sns.py +++ b/ScoutSuite/providers/aws/services/sns.py @@ -5,9 +5,8 @@ import json -from ScoutSuite.utils import manage_dictionary - from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients +from ScoutSuite.utils import manage_dictionary ######################################## diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index 30c631e14..0cebb39b1 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -1,15 +1,15 @@ # -*- coding: utf-8 -*- -import netaddr import copy -from ScoutSuite.providers.aws.aws import get_name -from opinel.utils.fs import load_data, read_ip_ranges +import netaddr -from ScoutSuite.providers.base.configs.browser import get_value_at -from ScoutSuite.utils import ec2_classic, get_keys, manage_dictionary +from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig from ScoutSuite.providers.aws.configs.vpc import VPCConfig as SingleVPCConfig +from ScoutSuite.providers.base.configs.browser import get_value_at +from ScoutSuite.utils import ec2_classic, get_keys, manage_dictionary +from opinel.utils.fs import load_data, read_ip_ranges ######################################## # Globals From a35edabc325ef851bbd8ad3d57994b9329cb02c0 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sat, 23 Feb 2019 11:05:15 -0500 Subject: [PATCH 058/154] [Opinel] Removed some unused functions --- ScoutSuite/core/console.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 598711d51..c20a36b35 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -115,18 +115,6 @@ def prompt_4_mfa_code(activate=False, input=None): return mfa_code -def prompt_4_mfa_serial(input=None): - """ - Prompt for an MFA serial number - - :param input: Used for unit testing - - :return: The MFA serial number - """ - return prompt_4_value('Enter your MFA serial:', required=False, regex=re_mfa_serial_format, - regex_format=mfa_serial_format, input=input) - - def prompt_4_overwrite(filename, force_write, input=None): """ Prompt whether the file should be overwritten From d2796cf5c4b190f9adee4df347c0c1f7e4c9e7e3 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sat, 23 Feb 2019 11:19:23 -0500 Subject: [PATCH 059/154] [Opinel] Reformated fs.py --- opinel/utils/fs.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/opinel/utils/fs.py b/opinel/utils/fs.py index 3c0104198..0ee8bd5e2 100644 --- a/opinel/utils/fs.py +++ b/opinel/utils/fs.py @@ -14,6 +14,7 @@ class CustomJSONEncoder(json.JSONEncoder): """ JSON encoder class """ + def default(self, o): if type(o) == datetime.datetime: return str(o) @@ -21,7 +22,7 @@ def default(self, o): return o.__dict__ -def load_data(data_file, key_name = None, local_file = False, format = 'json'): +def load_data(data_file, key_name=None, local_file=False, format='json'): """ Load a JSON data file @@ -56,7 +57,7 @@ def load_data(data_file, key_name = None, local_file = False, format = 'json'): return data -def read_ip_ranges(filename, local_file = True, ip_only = False, conditions = []): +def read_ip_ranges(filename, local_file=True, ip_only=False, conditions=[]): """ Returns the list of IP prefixes from an ip-ranges file @@ -67,12 +68,12 @@ def read_ip_ranges(filename, local_file = True, ip_only = False, conditions = [] :return: """ targets = [] - data = load_data(filename, local_file = local_file) + data = load_data(filename, local_file=local_file) if 'source' in data: # Filtered IP ranges conditions = data['conditions'] local_file = data['local_file'] if 'local_file' in data else False - data = load_data(data['source'], local_file = local_file, key_name = 'prefixes') + data = load_data(data['source'], local_file=local_file, key_name='prefixes') else: # Plain IP ranges data = data['prefixes'] @@ -95,7 +96,7 @@ def read_ip_ranges(filename, local_file = True, ip_only = False, conditions = [] return targets -def read_file(file_path, mode = 'rt'): +def read_file(file_path, mode='rt'): """ Read the contents of a file @@ -122,13 +123,14 @@ def save_blob_as_json(filename, blob, force_write, debug): try: if prompt_4_overwrite(filename, force_write): with open(filename, 'wt') as f: - print('%s' % json.dumps(blob, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=CustomJSONEncoder), file=f) + print('%s' % json.dumps(blob, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, + cls=CustomJSONEncoder), file=f) except Exception as e: printException(e) pass -def save_ip_ranges(profile_name, prefixes, force_write, debug, output_format = 'json'): +def save_ip_ranges(profile_name, prefixes, force_write, debug, output_format='json'): """ Creates/Modifies an ip-range-XXX.json file @@ -157,6 +159,7 @@ def save_ip_ranges(profile_name, prefixes, force_write, debug, output_format = ' # Write as CSV output = 'account_id, region, ip, instance_id, instance_name\n' for prefix in unique_prefixes: - output += '%s, %s, %s, %s, %s\n' % (prefix['account_id'], prefix['region'], prefix['ip_prefix'], prefix['instance_id'], prefix['name']) + output += '%s, %s, %s, %s, %s\n' % ( + prefix['account_id'], prefix['region'], prefix['ip_prefix'], prefix['instance_id'], prefix['name']) with open('ip-ranges-%s.csv' % profile_name, 'wt') as f: f.write(output) From 0160ed61c7e99f97a00847041abab96bf85fb271 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sat, 23 Feb 2019 12:59:13 -0500 Subject: [PATCH 060/154] Merged develop --- ScoutSuite/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 4c0b534f7..d1ec7f742 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -9,7 +9,7 @@ from ScoutSuite.core.console import configPrintException, printInfo, printDebug from ScoutSuite.providers.aws.profiles import AWSProfiles -from ScoutSuite.cli_parser import ScoutSuiteArgumentParser +from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser from ScoutSuite import AWSCONFIG from ScoutSuite.output.html import Scout2Report from ScoutSuite.core.exceptions import RuleExceptions From b7211229960dd6de42f0aa2a8cdb9109bf1a1e49 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sat, 23 Feb 2019 13:02:34 -0500 Subject: [PATCH 061/154] Merged develop --- ScoutSuite/providers/aws/configs/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index 6ed498d85..f2e180cf0 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -80,7 +80,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.config = ConfigConfig(metadata['management']['config'], thread_config) self.dynamodb = DynamoDBConfig(metadata['database']['dynamodb'], thread_config) self.kms = KMSConfig(metadata['security']['kms'], thread_config) - except NameError as e: + except: pass def _is_provider(self, provider_name): From 2937c3d1a30d58d7b6e4406f7f2bc706e6578197 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sat, 23 Feb 2019 13:07:28 -0500 Subject: [PATCH 062/154] Fixed broken try/except in AWS --- ScoutSuite/providers/aws/configs/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/configs/services.py b/ScoutSuite/providers/aws/configs/services.py index f2e180cf0..f63728be0 100644 --- a/ScoutSuite/providers/aws/configs/services.py +++ b/ScoutSuite/providers/aws/configs/services.py @@ -80,7 +80,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs): self.config = ConfigConfig(metadata['management']['config'], thread_config) self.dynamodb = DynamoDBConfig(metadata['database']['dynamodb'], thread_config) self.kms = KMSConfig(metadata['security']['kms'], thread_config) - except: + except (NameError, TypeError): pass def _is_provider(self, provider_name): From d52eb23b93434a6d8734843bf904cd84d7fc9340 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sat, 23 Feb 2019 13:35:54 -0500 Subject: [PATCH 063/154] [Opinel] Removed data/ and fs.py --- ScoutSuite/__main__.py | 1 + {opinel/utils => ScoutSuite/core}/fs.py | 16 +++------------- ScoutSuite/core/rule.py | 2 +- .../data/aws}/ip-ranges/aws-in-ec2.json | 2 +- .../data/aws}/ip-ranges/aws-in-us.json | 2 +- .../data/aws}/ip-ranges/aws.json | 0 .../data/icmp_message_types.json | 0 {opinel => ScoutSuite}/data/protocols.json | 0 ScoutSuite/providers/aws/credentials.py | 2 +- ...oup-whitelists-aws-ip-from-banned-region.json | 4 ++-- .../ec2-security-group-whitelists-aws.json | 2 +- ...c2-security-group-whitelists-unknown-aws.json | 2 +- ...-security-group-whitelists-unknown-cidrs.json | 2 +- ScoutSuite/providers/aws/services/ec2.py | 2 +- ScoutSuite/providers/aws/services/vpc.py | 2 +- opinel/__init__.py | 2 -- opinel/utils/__init__.py | 0 17 files changed, 15 insertions(+), 26 deletions(-) rename {opinel/utils => ScoutSuite/core}/fs.py (88%) rename {opinel/data => ScoutSuite/data/aws}/ip-ranges/aws-in-ec2.json (65%) rename {opinel/data => ScoutSuite/data/aws}/ip-ranges/aws-in-us.json (66%) rename {opinel/data => ScoutSuite/data/aws}/ip-ranges/aws.json (100%) rename {opinel => ScoutSuite}/data/icmp_message_types.json (100%) rename {opinel => ScoutSuite}/data/protocols.json (100%) delete mode 100644 opinel/__init__.py delete mode 100644 opinel/utils/__init__.py diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index d1ec7f742..b75d6e950 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -128,6 +128,7 @@ def main(args=None): exceptions = exceptions.exceptions except Exception as e: printDebug('Warning, failed to load exceptions. The file may not exist or may have an invalid format.') + print(e) exceptions = {} # Finalize diff --git a/opinel/utils/fs.py b/ScoutSuite/core/fs.py similarity index 88% rename from opinel/utils/fs.py rename to ScoutSuite/core/fs.py index 0ee8bd5e2..e4eb2c333 100644 --- a/opinel/utils/fs.py +++ b/ScoutSuite/core/fs.py @@ -22,7 +22,7 @@ def default(self, o): return o.__dict__ -def load_data(data_file, key_name=None, local_file=False, format='json'): +def load_data(data_file, key_name=None, local_file=False): """ Load a JSON data file @@ -38,20 +38,10 @@ def load_data(data_file, key_name=None, local_file=False, format='json'): src_dir = os.getcwd() src_file = os.path.join(src_dir, data_file) else: - src_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data') - if not os.path.isdir(src_dir): - src_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../data') + src_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../data') src_file = os.path.join(src_dir, data_file) with open(src_file) as f: - if format == 'json': - data = json.load(f) - elif format == 'yaml': - data = yaml.load(f) - elif format not in ['json', 'yaml'] and not key_name: - data = f.read() - else: - printError('Error, argument \'key_name\' may not be used with data in %s format.' % format) - return None + data = json.load(f) if key_name: data = data[key_name] return data diff --git a/ScoutSuite/core/rule.py b/ScoutSuite/core/rule.py index 6492d3962..3717f6cb0 100644 --- a/ScoutSuite/core/rule.py +++ b/ScoutSuite/core/rule.py @@ -3,7 +3,7 @@ import json import re -from opinel.utils.fs import read_ip_ranges +from ScoutSuite.core.fs import read_ip_ranges from ScoutSuite.core.console import printError from ScoutSuite.utils import format_service_name diff --git a/opinel/data/ip-ranges/aws-in-ec2.json b/ScoutSuite/data/aws/ip-ranges/aws-in-ec2.json similarity index 65% rename from opinel/data/ip-ranges/aws-in-ec2.json rename to ScoutSuite/data/aws/ip-ranges/aws-in-ec2.json index bcd8005ee..ca58b1500 100644 --- a/opinel/data/ip-ranges/aws-in-ec2.json +++ b/ScoutSuite/data/aws/ip-ranges/aws-in-ec2.json @@ -1,5 +1,5 @@ { - "source": "ip-ranges/aws.json", + "source": "aws/ip-ranges/aws.json", "conditions": [ "and", [ "service", "equal", "EC2" ] ] diff --git a/opinel/data/ip-ranges/aws-in-us.json b/ScoutSuite/data/aws/ip-ranges/aws-in-us.json similarity index 66% rename from opinel/data/ip-ranges/aws-in-us.json rename to ScoutSuite/data/aws/ip-ranges/aws-in-us.json index daf21e41c..21241d1fa 100644 --- a/opinel/data/ip-ranges/aws-in-us.json +++ b/ScoutSuite/data/aws/ip-ranges/aws-in-us.json @@ -1,5 +1,5 @@ { - "source": "ip-ranges/aws.json", + "source": "aws/ip-ranges/aws.json", "conditions": [ "and", [ "region", "match", [ "us-.*" ] ] ] diff --git a/opinel/data/ip-ranges/aws.json b/ScoutSuite/data/aws/ip-ranges/aws.json similarity index 100% rename from opinel/data/ip-ranges/aws.json rename to ScoutSuite/data/aws/ip-ranges/aws.json diff --git a/opinel/data/icmp_message_types.json b/ScoutSuite/data/icmp_message_types.json similarity index 100% rename from opinel/data/icmp_message_types.json rename to ScoutSuite/data/icmp_message_types.json diff --git a/opinel/data/protocols.json b/ScoutSuite/data/protocols.json similarity index 100% rename from opinel/data/protocols.json rename to ScoutSuite/data/protocols.json diff --git a/ScoutSuite/providers/aws/credentials.py b/ScoutSuite/providers/aws/credentials.py index aa566f352..5f70e0f71 100644 --- a/ScoutSuite/providers/aws/credentials.py +++ b/ScoutSuite/providers/aws/credentials.py @@ -15,7 +15,7 @@ from ScoutSuite.core.console import printException, printError, printInfo from ScoutSuite.core.console import prompt_4_mfa_code from ScoutSuite.providers.aws.aws import connect_service -from opinel.utils.fs import save_blob_as_json +from ScoutSuite.core.fs import save_blob_as_json ######################################## # Globals diff --git a/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-aws-ip-from-banned-region.json b/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-aws-ip-from-banned-region.json index 3b6df009f..a1ec88b0e 100644 --- a/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-aws-ip-from-banned-region.json +++ b/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-aws-ip-from-banned-region.json @@ -3,7 +3,7 @@ "path": "ec2.regions.id.vpcs.id.security_groups.id.rules.id.protocols.id.ports.id.cidrs.id.CIDR", "dashboard_name": "Rules", "conditions": [ "and", - [ "this", "inSubnets", "_IP_RANGES_FROM_FILE_(ip-ranges/aws.json, [])" ], - [ "this", "notInSubnets", "_IP_RANGES_FROM_FILE_(ip-ranges/aws-in-us.json, [])" ] + [ "this", "inSubnets", "_IP_RANGES_FROM_FILE_(aws/ip-ranges/aws.json, [])" ], + [ "this", "notInSubnets", "_IP_RANGES_FROM_FILE_(aws/ip-ranges/aws-in-us.json, [])" ] ] } diff --git a/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-aws.json b/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-aws.json index 1e37d41cb..1b788363d 100644 --- a/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-aws.json +++ b/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-aws.json @@ -4,6 +4,6 @@ "display_path": "ec2.regions.id.vpcs.id.security_groups.id", "dashboard_name": "Rules", "conditions": [ "and", - [ "this", "inSubnets", "_IP_RANGES_FROM_FILE_(ip-ranges/aws.json, [])" ] + [ "this", "inSubnets", "_IP_RANGES_FROM_FILE_(aws/ip-ranges/aws.json, [])" ] ] } diff --git a/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-unknown-aws.json b/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-unknown-aws.json index fe43b9f80..eae2cd5e7 100644 --- a/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-unknown-aws.json +++ b/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-unknown-aws.json @@ -4,7 +4,7 @@ "display_path": "ec2.regions.id.vpcs.id.security_groups.id", "dashboard_name": "Rules", "conditions": [ "and", - [ "this", "inSubnets", "_IP_RANGES_FROM_FILE_(ip-ranges/aws.json, [])" ], + [ "this", "inSubnets", "_IP_RANGES_FROM_FILE_(aws/ip-ranges/aws.json, [])" ], [ "this", "notInSubnets", "_IP_RANGES_FROM_FILE_(ip-ranges-from-args, [])" ] ] } diff --git a/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-unknown-cidrs.json b/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-unknown-cidrs.json index 16b7378de..74d0bd966 100644 --- a/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-unknown-cidrs.json +++ b/ScoutSuite/providers/aws/rules/findings/ec2-security-group-whitelists-unknown-cidrs.json @@ -6,7 +6,7 @@ "conditions": [ "and", [ "this", "notEqual", "0.0.0.0/0" ], [ "this", "notEqual", "::/0" ], - [ "this", "notInSubnets", "_IP_RANGES_FROM_FILE_(ip-ranges/aws.json, [])" ], + [ "this", "notInSubnets", "_IP_RANGES_FROM_FILE_(aws/ip-ranges/aws.json, [])" ], [ "this", "notInSubnets", "_IP_RANGES_FROM_FILE_(ip-ranges-from-args, [])" ], [ "_INCLUDE_(conditions/ip-not-in-private-space.json)", "", "" ] ] diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 09fbce283..a2af3494c 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -14,7 +14,7 @@ from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.providers.base.configs.browser import get_attribute_at from ScoutSuite.utils import get_keys, ec2_classic, manage_dictionary -from opinel.utils.fs import load_data +from ScoutSuite.core.fs import load_data ######################################## # Globals diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index 0cebb39b1..febdbd4df 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -9,7 +9,7 @@ from ScoutSuite.providers.aws.configs.vpc import VPCConfig as SingleVPCConfig from ScoutSuite.providers.base.configs.browser import get_value_at from ScoutSuite.utils import ec2_classic, get_keys, manage_dictionary -from opinel.utils.fs import load_data, read_ip_ranges +from ScoutSuite.core.fs import load_data, read_ip_ranges ######################################## # Globals diff --git a/opinel/__init__.py b/opinel/__init__.py deleted file mode 100644 index e094ea076..000000000 --- a/opinel/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -__author__ = 'l01cd3v' -__version__ = '3.3.4' diff --git a/opinel/utils/__init__.py b/opinel/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 633f9135d243cd515f62e21d75a3d5a2da20e85f Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sat, 23 Feb 2019 13:40:22 -0500 Subject: [PATCH 064/154] [Opinel] Updated Manifest --- MANIFEST.in | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 212e13fe3..cda9385c9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,7 @@ include LICENSE include README.md include requirements.txt -recursive-include opinel/data * +recursive-include ScoutSuite/data * recursive-include ScoutSuite/output/data * recursive-include ScoutSuite/providers/aws * recursive-include ScoutSuite/providers/aws/rules * diff --git a/setup.py b/setup.py index f0133b71d..3c2bbe842 100644 --- a/setup.py +++ b/setup.py @@ -58,8 +58,8 @@ 'gcp/rules/findings/*.json', 'gcp/rules/rulesets/*.json' ], - 'opinel': [ - 'data/*.json', + 'ScoutSuite.data': [ + '*.json' ] }, include_package_data=True, From aade383a1e32292467192a2728c9ba9ecc3b2b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pelletier?= Date: Sat, 23 Feb 2019 17:57:40 -0500 Subject: [PATCH 065/154] Added GCE CIS findings checks --- .../providers/gcp/services/computeengine.py | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/gcp/services/computeengine.py b/ScoutSuite/providers/gcp/services/computeengine.py index 72c82b28a..a478107f5 100644 --- a/ScoutSuite/providers/gcp/services/computeengine.py +++ b/ScoutSuite/providers/gcp/services/computeengine.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig +from ScoutSuite.providers.gcp.utils import gcp_connect_service from googleapiclient.errors import HttpError import json @@ -79,9 +80,11 @@ def __init__(self, thread_config): # return None def parse_instances(self, instance, params): + project_id = self._get_project_id(instance) + instance_dict = {} instance_dict['id'] = self.get_non_provider_id(instance['name']) - instance_dict['project_id'] = instance['selfLink'].split('/')[-5] + instance_dict['project_id'] = project_id instance_dict['name'] = instance['name'] instance_dict['description'] = instance['description'] if 'description' in instance and instance['description'] else 'N/A' instance_dict['creation_timestamp'] = instance['creationTimestamp'] @@ -92,6 +95,11 @@ def parse_instances(self, instance, params): instance_dict['network_interfaces'] = instance['networkInterfaces'] instance_dict['service_accounts'] = instance['serviceAccounts'] instance_dict['deletion_protection_enabled'] = instance['deletionProtection'] + instance_dict['block_project_ssh_keys_enabled'] = self._is_block_project_ssh_keys_enabled(instance) + instance_dict['oslogin_enabled'] = self._is_oslogin_enabled(instance, project_id) + instance_dict['ip_forwarding_enabled'] = instance['canIpForward'] + instance_dict['serial_port_enabled'] = self._is_serial_port_enabled(instance) + instance_dict['has_full_access_cloud_apis'] = self._has_full_access_to_all_cloud_apis(instance) # TODO this should be done in it's own getter method and merged with instances during postprocessing instance_dict['disks'] = {} @@ -102,8 +110,9 @@ def parse_instances(self, instance, params): 'mode': disk['mode'], 'source_url': disk['source'], 'source_device_name': disk['deviceName'], - 'bootable': disk['boot'] - } + 'bootable': disk['boot'], + 'encrypted_with_csek': self._is_encrypted_with_csek(disk) + } self.instances[instance_dict['id']] = instance_dict @@ -188,3 +197,39 @@ def parse_firewalls(self, firewall, params): firewall_dict[direction_string][rule['IPProtocol']] = ['0-65535'] self.firewalls[firewall_dict['id']] = firewall_dict + + def _get_project_id(self, instance): + return instance['selfLink'].split('/')[-5] + + def _metadata_to_dict(self, metadata): + return dict((item['key'], item['value']) for item in metadata['items']) if 'items' in metadata else {} + + def _is_block_project_ssh_keys_enabled(self, instance): + metadata = self._metadata_to_dict(instance['metadata']) + return metadata.get('block-project-ssh-keys') == 'true' + + def _get_common_instance_metadata_dict(self, project_id): + computeengine_client = gcp_connect_service(service='computeengine') + request = computeengine_client.projects().get(project=project_id) + project = request.execute() + return self._metadata_to_dict(project['commonInstanceMetadata']) + + def _is_oslogin_enabled(self, instance, project_id): + instance_metadata = self._metadata_to_dict(instance['metadata']) + if instance_metadata.get('enable-oslogin') == 'FALSE': + return False + elif instance_metadata.get('enable-oslogin') == 'TRUE': + return True + project_metadata = self._get_common_instance_metadata_dict(project_id) + return project_metadata.get('enable-oslogin') == 'TRUE' + + def _is_serial_port_enabled(self, instance): + metadata = self._metadata_to_dict(instance['metadata']) + return metadata.get('serial-port-enable') == 'true' + + def _has_full_access_to_all_cloud_apis(self, instance): + full_access_scope = 'https://www.googleapis.com/auth/cloud-platform' + return any(full_access_scope in service_account['scopes'] for service_account in instance['serviceAccounts']) + + def _is_encrypted_with_csek(self, disk): + return 'diskEncryptionKey' in disk and 'sha256' in disk['diskEncryptionKey'] and disk['diskEncryptionKey']['sha256'] != '' From 132edb870982adb7eaed27b03d561e783970e506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pelletier?= Date: Sat, 23 Feb 2019 17:59:09 -0500 Subject: [PATCH 066/154] Added new GCE findings details to partial --- .../partials/gcp/services.computeengine.instances.html | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/output/data/html/partials/gcp/services.computeengine.instances.html b/ScoutSuite/output/data/html/partials/gcp/services.computeengine.instances.html index a82d9094f..7a0a36c50 100644 --- a/ScoutSuite/output/data/html/partials/gcp/services.computeengine.instances.html +++ b/ScoutSuite/output/data/html/partials/gcp/services.computeengine.instances.html @@ -12,6 +12,10 @@

Information

Creation Date: {{creation_timestamp}}
Status: {{status}}
Deletion Protection: {{convert_bool_to_enabled deletion_protection_enabled}}
+
Block Project SSH Keys: {{convert_bool_to_enabled block_project_ssh_keys_enabled}}
+
IP Forwarding: {{convert_bool_to_enabled ip_forwarding_enabled}}
+
OS Login: {{convert_bool_to_enabled oslogin_enabled}}
+
Serial Port Connection: {{convert_bool_to_enabled serial_port_enabled}}
{{#if tags}}
Tags:
    @@ -37,7 +41,7 @@
    Disks
    {{#each disks}}
  • {{source_device_name}}
      • -
    • Bootable: {{bootable}}
      • +
    • Bootable: {{bootable}}
    • Type: {{type}}
    • Mode: {{mode}}
      • {{#if latest_snapshot}} @@ -45,6 +49,7 @@
      Disks
      {{else}}
    • Latest snapshot: None
      • {{/if}} +
    • Customer Supplied Encryption: {{convert_bool_to_enabled encrypted_with_csek}}
    {{else}}
  • None
    • From 62254cda0d44f9470c39e66f0ffb73c5a8d6f6f4 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sun, 24 Feb 2019 13:02:25 -0500 Subject: [PATCH 067/154] [Opinel] Fixed broken regex --- ScoutSuite/core/conditions.py | 2 +- tests/test-rules-processingengine.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/core/conditions.py b/ScoutSuite/core/conditions.py index 782572c46..801bc3ba8 100644 --- a/ScoutSuite/core/conditions.py +++ b/ScoutSuite/core/conditions.py @@ -187,7 +187,7 @@ def _pass_condition(b, test, a): a = [a] b = str(b) for c in a: - if not re.match(c, b): + if re.match(c, b): result = True break elif test == 'notMatch': diff --git a/tests/test-rules-processingengine.py b/tests/test-rules-processingengine.py index e8d3eab12..44efaa68c 100644 --- a/tests/test-rules-processingengine.py +++ b/tests/test-rules-processingengine.py @@ -74,8 +74,7 @@ def _test_rule(self, ruleset_file_name, rule_file_name, rule): try: assert (set(sorted(findings)) == set(sorted(items))) - except Exception as e: - printError(e) + except Exception: printError('Expected items:\n %s' % json.dumps(sorted(items))) printError('Reported items:\n %s' % json.dumps(sorted(findings))) assert (False) From 68870351d3e4ad8462f80c294c88f29d5cd00227 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sun, 24 Feb 2019 13:14:09 -0500 Subject: [PATCH 068/154] [Opinel] Renamed console.py functions --- ScoutSuite/__listall__.py | 12 ++-- ScoutSuite/__main__.py | 10 +-- ScoutSuite/core/conditions.py | 28 +++----- ScoutSuite/core/console.py | 64 +++++++++---------- ScoutSuite/core/exceptions.py | 4 +- ScoutSuite/core/fs.py | 6 +- ScoutSuite/core/processingengine.py | 10 +-- ScoutSuite/core/rule.py | 4 +- ScoutSuite/core/rule_definition.py | 12 ++-- ScoutSuite/core/ruleset.py | 10 +-- ScoutSuite/core/threads.py | 4 +- ScoutSuite/core/utils.py | 10 +-- ScoutSuite/output/html.py | 6 +- ScoutSuite/output/js.py | 8 +-- ScoutSuite/output/utils.py | 6 +- ScoutSuite/providers/aws/aws.py | 6 +- ScoutSuite/providers/aws/configs/regions.py | 14 ++-- ScoutSuite/providers/aws/credentials.py | 20 +++--- ScoutSuite/providers/aws/profiles.py | 4 +- ScoutSuite/providers/aws/provider.py | 20 +++--- ScoutSuite/providers/aws/services/ec2.py | 12 ++-- ScoutSuite/providers/aws/services/iam.py | 14 ++-- ScoutSuite/providers/aws/services/rds.py | 6 +- ScoutSuite/providers/aws/services/s3.py | 20 +++--- ScoutSuite/providers/azure/provider.py | 6 +- ScoutSuite/providers/azure/utils.py | 6 +- ScoutSuite/providers/base/configs/base.py | 16 ++--- ScoutSuite/providers/base/configs/browser.py | 10 +-- ScoutSuite/providers/base/configs/services.py | 8 +-- ScoutSuite/providers/base/provider.py | 24 +++---- ScoutSuite/providers/gcp/configs/base.py | 16 ++--- ScoutSuite/providers/gcp/provider.py | 20 +++--- .../gcp/services/cloudresourcemanager.py | 4 +- ScoutSuite/providers/gcp/services/cloudsql.py | 8 +-- .../providers/gcp/services/cloudstorage.py | 6 +- ScoutSuite/providers/gcp/services/iam.py | 6 +- ScoutSuite/providers/gcp/utils.py | 6 +- tests/test-rules-processingengine.py | 12 ++-- tests/test-rules-ruleset.py | 10 +-- tests/test-scoutsuite.py | 2 +- tests/test-utils_cloudwatch.py | 4 +- tests/test-utils_sns.py | 4 +- 42 files changed, 233 insertions(+), 245 deletions(-) diff --git a/ScoutSuite/__listall__.py b/ScoutSuite/__listall__.py index 917027fce..58dbd444a 100644 --- a/ScoutSuite/__listall__.py +++ b/ScoutSuite/__listall__.py @@ -7,7 +7,7 @@ try: from opinel.utils.globals import check_requirements - from ScoutSuite.core.console import configPrintException, printError, printException, printInfo + from ScoutSuite.core.console import config_debug_level, print_error, print_exception, print_info except Exception as e: print('Error: Scout2 depends on the opinel package. Install all the requirements with the following command:') print(' $ pip install -r requirements.txt') @@ -28,7 +28,7 @@ def main(args): # Configure the debug level - configPrintException(args.debug) + config_debug_level(args.debug) # FIXME check that all requirements are installed # # Check version of opinel @@ -46,8 +46,8 @@ def main(args): aws_config = report.jsrw.load_from_file(AWSCONFIG) services = aws_config['service_list'] except Exception as e: - printException(e) - printError('Error, failed to load the configuration for profile %s' % profile_name) + print_exception(e) + print_error('Error, failed to load the configuration for profile %s' % profile_name) continue # Create a ruleset with only whatever rules were specified... @@ -68,7 +68,7 @@ def main(args): f.write(json.dumps(rule_dict)) ruleset = TmpRuleset(rule_dirs=[os.getcwd()], rule_filename=rule_filename, rule_args=[]) else: - printError( + print_error( 'Error, you must provide either a rule configuration file or the path to the resources targeted.') continue @@ -115,4 +115,4 @@ def main(args): (lines, template) = format_listall_output(args.format_file[0], None, args.format, rule) # Print the output - printInfo(generate_listall_output(lines, resources, aws_config, template, [])) + print_info(generate_listall_output(lines, resources, aws_config, template, [])) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index b75d6e950..70d869e24 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -6,7 +6,7 @@ import os import webbrowser -from ScoutSuite.core.console import configPrintException, printInfo, printDebug +from ScoutSuite.core.console import config_debug_level, print_info, print_debug from ScoutSuite.providers.aws.profiles import AWSProfiles from ScoutSuite.core.cli_parser import ScoutSuiteArgumentParser @@ -32,7 +32,7 @@ def main(args=None): args = args.__dict__ # Configure the debug level - configPrintException(args.get('debug')) + config_debug_level(args.get('debug')) # Create a cloud provider object cloud_provider = get_provider(provider=args.get('provider'), @@ -83,7 +83,7 @@ def main(args=None): try: cloud_provider.fetch(regions=args.get('regions')) except KeyboardInterrupt: - printInfo('\nCancelled by user') + print_info('\nCancelled by user') return 130 # Update means we reload the whole config and overwrite part of it @@ -127,7 +127,7 @@ def main(args=None): exceptions.process(cloud_provider) exceptions = exceptions.exceptions except Exception as e: - printDebug('Warning, failed to load exceptions. The file may not exist or may have an invalid format.') + print_debug('Warning, failed to load exceptions. The file may not exist or may have an invalid format.') print(e) exceptions = {} @@ -157,7 +157,7 @@ def main(args=None): # Open the report by default if not args.get('no_browser'): - printInfo('Opening the HTML report...') + print_info('Opening the HTML report...') url = 'file://%s' % os.path.abspath(html_report_path) webbrowser.open(url, new=2) diff --git a/ScoutSuite/core/conditions.py b/ScoutSuite/core/conditions.py index 801bc3ba8..c8fc36ecf 100644 --- a/ScoutSuite/core/conditions.py +++ b/ScoutSuite/core/conditions.py @@ -8,8 +8,9 @@ from iampoliciesgonewild import get_actions_from_statement, _expand_wildcard_action -from ScoutSuite.core.console import printError, printException +from ScoutSuite.core.console import print_error, print_exception from ScoutSuite.core import condition_operators +from ScoutSuite.providers.base.configs.browser import get_value_at re_get_value_at = re.compile(r'_GET_VALUE_AT_\((.*?)\)') re_nested_get_value_at = re.compile(r'_GET_VALUE_AT_\(.*') @@ -26,9 +27,6 @@ def pass_conditions(all_info, current_path, conditions, unknown_as_pass_conditio :return: """ - # Imported here temporarly to fix cyclic dependency - from ScoutSuite.providers.base.configs.browser import get_value_at - if len(conditions) == 0: return True condition_operator = conditions.pop(0) @@ -48,32 +46,26 @@ def pass_conditions(all_info, current_path, conditions, unknown_as_pass_conditio res = _pass_condition(target_obj, test_name, test_values) except Exception as e: res = True if unknown_as_pass_condition else False - printError('Unable to process testcase \'%s\' on value \'%s\', interpreted as %s.' % ( + print_error('Unable to process testcase \'%s\' on value \'%s\', interpreted as %s.' % ( test_name, str(target_obj), res)) - printException(e, True) + print_exception(e, True) # Quick exit and + false if condition_operator == 'and' and not res: return False # Quick exit or + true if condition_operator == 'or' and res: return True - # Still here ? - # or -> false - # and -> true - if condition_operator == 'or': - return False - else: - return True + return not condition_operator == 'or' def __prepare_age_test(a, b): if type(a) != list: - printError('Error: olderThan requires a list such as [ N , \'days\' ] or [ M, \'hours\'].') + print_error('Error: olderThan requires a list such as [ N , \'days\' ] or [ M, \'hours\'].') raise Exception number = int(a[0]) unit = a[1] if unit not in ['days', 'hours', 'minutes', 'seconds']: - printError('Error: only days, hours, minutes, and seconds are supported.') + print_error('Error: only days, hours, minutes, and seconds are supported.') raise Exception if unit == 'hours': number *= 3600 @@ -264,7 +256,7 @@ def _pass_condition(b, test, a): # Unknown test case else: - printError('Error: unknown test case %s' % test) + print_error('Error: unknown test case %s' % test) raise Exception return result @@ -272,10 +264,6 @@ def _pass_condition(b, test, a): def fix_path_string(all_info, current_path, path_to_value): # handle nested _GET_VALUE_AT_... - - # Imported here temporarly to fix cyclic dependency - from ScoutSuite.providers.base.configs.browser import get_value_at - while True: dynamic_path = re_get_value_at.findall(path_to_value) if len(dynamic_path) == 0: diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index c20a36b35..3ed568574 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -20,7 +20,7 @@ # Print configuration functions ######################################## -def configPrintException(enable): +def config_debug_level(enable): """ Configure whether full stacktraces should be dumped in the console output @@ -36,32 +36,32 @@ def configPrintException(enable): # Print functions ######################################## -def printDebug(msg): +def print_debug(msg): if verbose_exceptions: - printGeneric(sys.stderr, msg) + print_generic(sys.stderr, msg) -def printError(msg, newLine=True): - printGeneric(sys.stderr, msg, newLine) +def print_error(msg, newLine=True): + print_generic(sys.stderr, msg, newLine) -def printException(e, debug_only=False): +def print_exception(e, debug_only=False): global verbose_exceptions if verbose_exceptions: - printError(str(traceback.format_exc())) + print_error(str(traceback.format_exc())) elif not debug_only: - printError(str(e)) + print_error(str(e)) -def printGeneric(out, msg, newLine=True): +def print_generic(out, msg, newLine=True): out.write(msg) out.flush() if newLine == True: out.write('\n') -def printInfo(msg, newLine=True): - printGeneric(sys.stdout, msg, newLine) +def print_info(msg, newLine=True): + print_generic(sys.stdout, msg, newLine) ######################################## @@ -92,7 +92,7 @@ def prompt(test_input=None): return choice -def prompt_4_mfa_code(activate=False, input=None): +def prompt_mfa_code(activate=False, input=None): """ Prompt for an MFA code @@ -106,16 +106,16 @@ def prompt_4_mfa_code(activate=False, input=None): prompt_string = 'Enter the next value: ' else: prompt_string = 'Enter your MFA code (or \'q\' to abort): ' - mfa_code = prompt_4_value(prompt_string, no_confirm=True, input=input) + mfa_code = prompt_value(prompt_string, no_confirm=True, input=input) if mfa_code == 'q': return mfa_code if not re_mfa_code.match(): - printError('Error: your MFA code must only consist of digits and be at least 6 characters long.') + print_error('Error: your MFA code must only consist of digits and be at least 6 characters long.') break return mfa_code -def prompt_4_overwrite(filename, force_write, input=None): +def prompt_overwrite(filename, force_write, input=None): """ Prompt whether the file should be overwritten @@ -127,12 +127,12 @@ def prompt_4_overwrite(filename, force_write, input=None): """ if not os.path.exists(filename) or force_write: return True - return prompt_4_yes_no('File \'{}\' already exists. Do you want to overwrite it'.format(filename), input=input) + return prompt_yes_no('File \'{}\' already exists. Do you want to overwrite it'.format(filename), input=input) -def prompt_4_value(question, choices=None, default=None, display_choices=True, display_indices=False, - authorize_list=False, is_question=False, no_confirm=False, required=True, regex=None, - regex_format='', max_laps=5, input=None, return_index=False): +def prompt_value(question, choices=None, default=None, display_choices=True, display_indices=False, + authorize_list=False, is_question=False, no_confirm=False, required=True, regex=None, + regex_format='', max_laps=5, input=None, return_index=False): """ Prompt for a value . . @@ -157,35 +157,35 @@ def prompt_4_value(question, choices=None, default=None, display_choices=True, d lap_n = 0 while True: if lap_n >= max_laps: - printError('Automatically abording prompt loop after 5 failures') + print_error('Automatically abording prompt loop after 5 failures') return None lap_n += 1 can_return = False # Display the question, choices, and prompt for the answer if is_question: question = question + '? ' - printError(question) + print_error(question) if choices and display_indices: for c in choices: - printError('%3d. %s' % (choices.index(c), c)) - printError('Enter the number corresponding to your choice: ', False) + print_error('%3d. %s' % (choices.index(c), c)) + print_error('Enter the number corresponding to your choice: ', False) choice = prompt(input) # Set the default value if empty choice if not choice or choice == '': if default: - if no_confirm or prompt_4_yes_no('Use the default value (' + default + ')'): + if no_confirm or prompt_yes_no('Use the default value (' + default + ')'): # return default choice = default can_return = True elif not required: can_return = True else: - printError('Error: you cannot leave this parameter empty.') + print_error('Error: you cannot leave this parameter empty.') # Validate the value against a whitelist of choices elif choices: user_choices = [item.strip() for item in choice.split(',')] if not authorize_list and len(user_choices) > 1: - printError('Error: multiple values are not supported; please enter a single value.') + print_error('Error: multiple values are not supported; please enter a single value.') else: choice_valid = True if display_indices and int(choice) < len(choices): @@ -194,7 +194,7 @@ def prompt_4_value(question, choices=None, default=None, display_choices=True, d else: for c in user_choices: if not c in choices: - printError('Invalid value (%s).' % c) + print_error('Invalid value (%s).' % c) choice_valid = False break if choice_valid: @@ -205,17 +205,17 @@ def prompt_4_value(question, choices=None, default=None, display_choices=True, d # return choice can_return = True else: - printError('Error: expected format is: %s' % regex_format) + print_error('Error: expected format is: %s' % regex_format) else: # No automated validation, can attempt to return can_return = True if can_return: # Manually onfirm that the entered value is correct if needed - if no_confirm or prompt_4_yes_no('You entered "' + choice + '". Is that correct', input=input): + if no_confirm or prompt_yes_no('You entered "' + choice + '". Is that correct', input=input): return int(int_choice) if return_index else choice -def prompt_4_yes_no(question, input=None): +def prompt_yes_no(question, input=None): """ Prompt for a yes/no or y/n answer . @@ -226,7 +226,7 @@ def prompt_4_yes_no(question, input=None): """ count = 0 while True: - printError(question + ' (y/n)? ') + print_error(question + ' (y/n)? ') choice = prompt(input).lower() if choice == 'yes' or choice == 'y': return True @@ -234,6 +234,6 @@ def prompt_4_yes_no(question, input=None): return False else: count += 1 - printError('\'%s\' is not a valid answer. Enter \'yes\'(y) or \'no\'(n).' % choice) + print_error('\'%s\' is not a valid answer. Enter \'yes\'(y) or \'no\'(n).' % choice) if count > 3: return None diff --git a/ScoutSuite/core/exceptions.py b/ScoutSuite/core/exceptions.py index a6e5dd11b..b821623a0 100644 --- a/ScoutSuite/core/exceptions.py +++ b/ScoutSuite/core/exceptions.py @@ -3,7 +3,7 @@ Exceptions handling """ -from ScoutSuite.core.console import printDebug +from ScoutSuite.core.console import print_debug from ScoutSuite import EXCEPTIONS from ScoutSuite.output.js import JavaScriptReaderWriter @@ -24,7 +24,7 @@ def process(self, cloud_provider): for rule in self.exceptions[service]: filtered_items = [] if rule not in cloud_provider.services[service]['findings']: - printDebug('Warning:: key error should not be happening') + print_debug('Warning:: key error should not be happening') continue for item in cloud_provider.services[service]['findings'][rule]['items']: if item not in self.exceptions[service][rule]: diff --git a/ScoutSuite/core/fs.py b/ScoutSuite/core/fs.py index e4eb2c333..6cfcdced8 100644 --- a/ScoutSuite/core/fs.py +++ b/ScoutSuite/core/fs.py @@ -6,7 +6,7 @@ import os import yaml -from ScoutSuite.core.console import printError, printException, prompt_4_overwrite +from ScoutSuite.core.console import print_error, print_exception, prompt_overwrite from ScoutSuite.core.conditions import _pass_condition @@ -111,12 +111,12 @@ def save_blob_as_json(filename, blob, force_write, debug): :return: """ try: - if prompt_4_overwrite(filename, force_write): + if prompt_overwrite(filename, force_write): with open(filename, 'wt') as f: print('%s' % json.dumps(blob, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=CustomJSONEncoder), file=f) except Exception as e: - printException(e) + print_exception(e) pass diff --git a/ScoutSuite/core/processingengine.py b/ScoutSuite/core/processingengine.py index 1ceb66008..d90a6028f 100644 --- a/ScoutSuite/core/processingengine.py +++ b/ScoutSuite/core/processingengine.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ScoutSuite.core.console import printDebug, printError, printException +from ScoutSuite.core.console import print_debug, print_error, print_exception from ScoutSuite.utils import manage_dictionary from ScoutSuite.core.utils import recurse @@ -22,7 +22,7 @@ def __init__(self, ruleset): manage_dictionary(self.rules, rule.path, []) self.rules[rule.path].append(rule) except Exception as e: - printError('Failed to create rule %s: %s' % (rule.path, e)) + print_error('Failed to create rule %s: %s' % (rule.path, e)) def run(self, cloud_provider, skip_dashboard = False): @@ -37,7 +37,7 @@ def run(self, cloud_provider, skip_dashboard = False): if not rule.enabled: # or rule.service not in []: # TODO: handle this... continue - printDebug('Processing %s rule[%s]: "%s"' % (rule.service, rule.filename, rule.description)) + print_debug('Processing %s rule[%s]: "%s"' % (rule.service, rule.filename, rule.description)) finding_path = rule.path path = finding_path.split('.') service = path[0] @@ -59,8 +59,8 @@ def run(self, cloud_provider, skip_dashboard = False): cloud_provider.services[service][self.ruleset.rule_type][rule.key]['service'] = rule.service cloud_provider.services[service][self.ruleset.rule_type][rule.key]['rationale'] = rule.rationale if hasattr(rule, 'rationale') else 'No description available.' except Exception as e: - printException(e) - printError('Failed to process rule defined in %s' % rule.filename) + print_exception(e) + print_error('Failed to process rule defined in %s' % rule.filename) # Fallback if process rule failed to ensure report creation and data dump still happen cloud_provider.services[service][self.ruleset.rule_type][rule.key]['checked_items'] = 0 cloud_provider.services[service][self.ruleset.rule_type][rule.key]['flagged_items'] = 0 diff --git a/ScoutSuite/core/rule.py b/ScoutSuite/core/rule.py index 3717f6cb0..f5eecd603 100644 --- a/ScoutSuite/core/rule.py +++ b/ScoutSuite/core/rule.py @@ -4,7 +4,7 @@ import re from ScoutSuite.core.fs import read_ip_ranges -from ScoutSuite.core.console import printError +from ScoutSuite.core.console import print_error from ScoutSuite.utils import format_service_name @@ -139,4 +139,4 @@ def set_definition(self, rule_definitions, attributes = None, ip_ranges = None, setattr(self, 'key', '%s-%s' % (self.key, self.key_suffix)) except Exception as e: # printException(e) - printError('Failed to set definition %s: %s' % (self.filename, e)) + print_error('Failed to set definition %s: %s' % (self.filename, e)) diff --git a/ScoutSuite/core/rule_definition.py b/ScoutSuite/core/rule_definition.py index a071c6d0e..53fcf2776 100644 --- a/ScoutSuite/core/rule_definition.py +++ b/ScoutSuite/core/rule_definition.py @@ -3,7 +3,7 @@ import json import os -from ScoutSuite.core.console import printError +from ScoutSuite.core.console import print_error class RuleDefinition(object): @@ -21,7 +21,7 @@ def __init__(self, data_path, file_name = None, rule_dirs = None, string_definit self.string_definition = string_definition self.load_from_string_definition() else: - printError('Error') + print_error('Error') def __str__(self): @@ -49,7 +49,7 @@ def load(self): try: file_path = os.path.join(rule_dir, self.file_name) if rule_dir else self.file_name except Exception as e: - printError('Failed to load file %s: %e' % (self.file_name, e)) + print_error('Failed to load file %s: %e' % (self.file_name, e)) if os.path.isfile(file_path): self.file_path = file_path file_name_valid = True @@ -72,7 +72,7 @@ def load(self): if os.path.isfile(self.file_path): file_name_valid = True if not file_name_valid: - printError('Error: could not find %s' % self.file_name) + print_error('Error: could not find %s' % self.file_name) else: try: with open(self.file_path, 'rt') as f: @@ -80,7 +80,7 @@ def load(self): self.load_from_string_definition() except Exception as e: # printException(e) - printError('Failed to load rule defined in %s: %s' % (self.file_name, e)) + print_error('Failed to load rule defined in %s: %s' % (self.file_name, e)) def load_from_string_definition(self): @@ -89,4 +89,4 @@ def load_from_string_definition(self): for attr in definition: setattr(self, attr, definition[attr]) except Exception as e: - printError('Failed to load string definition %s: %e' % (self.string_definition, e)) + print_error('Failed to load string definition %s: %e' % (self.string_definition, e)) diff --git a/ScoutSuite/core/ruleset.py b/ScoutSuite/core/ruleset.py index 791be7e5e..f0487fa2c 100644 --- a/ScoutSuite/core/ruleset.py +++ b/ScoutSuite/core/ruleset.py @@ -4,7 +4,7 @@ import os import tempfile -from ScoutSuite.core.console import printDebug, printError, prompt_4_yes_no +from ScoutSuite.core.console import print_debug, print_error, prompt_yes_no from ScoutSuite.core.rule import Rule from ScoutSuite.core.rule_definition import RuleDefinition @@ -44,7 +44,7 @@ def __init__(self, self.filename = self.find_file(filename, provider=cloud_provider) if not self.filename: self.search_ruleset(environment_name) - printDebug('Loading ruleset %s' % self.filename) + print_debug('Loading ruleset %s' % self.filename) self.name = os.path.basename(self.filename).replace('.json', '') if not name else name self.load(self.rule_type) self.shared_init(ruleset_generator, rules_dir, aws_account_id, ip_ranges) @@ -85,13 +85,13 @@ def load(self, rule_type, quiet=False): self.handle_rule_versions(filename, rule_type, rule) except Exception as e: # printException(e) - printError('Error: ruleset file %s contains malformed JSON.' % self.filename) + print_error('Error: ruleset file %s contains malformed JSON.' % self.filename) self.rules = [] self.about = '' else: self.rules = [] if not quiet: - printError('Error: the file %s does not exist.' % self.filename) + print_error('Error: the file %s does not exist.' % self.filename) def load_rules(self, file, rule_type, quiet=False): file.seek(0) @@ -179,7 +179,7 @@ def search_ruleset(self, environment_name, no_prompt=False): ruleset_file_name = 'ruleset-%s.json' % environment_name ruleset_file_path = os.path.join(self.rules_data_path, 'rulesets/%s' % ruleset_file_name) if os.path.exists(ruleset_file_path): - if no_prompt or prompt_4_yes_no( + if no_prompt or prompt_yes_no( "A ruleset whose name matches your environment name was found in %s. Would you like to use it instead of the default one" % ruleset_file_name): ruleset_found = True self.filename = ruleset_file_path diff --git a/ScoutSuite/core/threads.py b/ScoutSuite/core/threads.py index 5bee3fa8f..3826ae406 100644 --- a/ScoutSuite/core/threads.py +++ b/ScoutSuite/core/threads.py @@ -3,7 +3,7 @@ from threading import Thread from six.moves.queue import Queue -from ScoutSuite.core.console import printException +from ScoutSuite.core.console import print_exception def thread_work(targets, function, params={}, num_threads=0): @@ -44,6 +44,6 @@ def threaded_per_region(q, params): method = params['method'] method(params) except Exception as e: - printException(e) + print_exception(e) finally: q.task_done() diff --git a/ScoutSuite/core/utils.py b/ScoutSuite/core/utils.py index f0e8de441..74a516d14 100644 --- a/ScoutSuite/core/utils.py +++ b/ScoutSuite/core/utils.py @@ -6,7 +6,7 @@ from six import string_types import copy -from ScoutSuite.core.console import printError +from ScoutSuite.core.console import print_error from ScoutSuite.core.conditions import pass_conditions, fix_path_string @@ -75,9 +75,9 @@ def recurse(all_info, current_info, target_path, current_path, config, add_suffi results = results + recurse(all_info, current_info, [], split_current_path, config, add_suffix) else: - printError('Error: unhandled case, typeof(current_info) = %s' % type(current_info)) - printError('Path: %s' % current_path) - printError('Object: %s' % str(current_info)) - printError('Entry target path: %s' % str(dbg_target_path)) + print_error('Error: unhandled case, typeof(current_info) = %s' % type(current_info)) + print_error('Path: %s' % current_path) + print_error('Object: %s' % str(current_info)) + print_error('Entry target path: %s' % str(dbg_target_path)) raise Exception return results diff --git a/ScoutSuite/output/html.py b/ScoutSuite/output/html.py index f1c285cec..381d1eff7 100644 --- a/ScoutSuite/output/html.py +++ b/ScoutSuite/output/html.py @@ -7,7 +7,7 @@ import zipfile import dateutil.tz -from ScoutSuite.core.console import printInfo, printException +from ScoutSuite.core.console import print_info, print_exception from ScoutSuite import AWSCONFIG, EXCEPTIONS, HTMLREPORT, AWSRULESET, AWSCONFIG_FILE, EXCEPTIONS_FILE, HTMLREPORT_FILE, \ GENERATOR_FILE @@ -43,7 +43,7 @@ def get_content_from(self, templates_type): with open('%s' % filename, 'rt') as f: contents = contents + f.read() except Exception as e: - printException('Error reading filename %s: %s' % (filename, e)) + print_exception('Error reading filename %s: %s' % (filename, e)) return contents def prepare_html_report_dir(self): @@ -91,7 +91,7 @@ def create_html_report(self, force_write): contents += self.get_content_from('summaries') contents += self.get_content_from('summaries/%s' % self.provider) new_file, first_line = get_filename(HTMLREPORT, self.profile, self.report_dir) - printInfo('Creating %s ...' % new_file) + print_info('Creating %s ...' % new_file) if prompt_4_overwrite(new_file, force_write): if os.path.exists(new_file): os.remove(new_file) diff --git a/ScoutSuite/output/js.py b/ScoutSuite/output/js.py index 37b4e4488..d0b56fee3 100644 --- a/ScoutSuite/output/js.py +++ b/ScoutSuite/output/js.py @@ -6,7 +6,7 @@ import json import os -from ScoutSuite.core.console import printException, printInfo +from ScoutSuite.core.console import print_exception, print_info from ScoutSuite import DEFAULT_REPORT_DIR from ScoutSuite.output.utils import get_filename, prompt_4_overwrite @@ -70,7 +70,7 @@ def save_to_file(self, config, config_type, force_write, debug): print('%s' % first_line, file=f) print('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, cls=Scout2Encoder), file=f) except Exception as e: - printException(e) + print_exception(e) def to_dict(self, config): @@ -86,7 +86,7 @@ def __open_file(self, config_filename, force_write, quiet=False): :return: """ if not quiet: - printInfo('Saving config...') + print_info('Saving config...') if prompt_4_overwrite(config_filename, force_write): try: config_dirname = os.path.dirname(config_filename) @@ -94,6 +94,6 @@ def __open_file(self, config_filename, force_write, quiet=False): os.makedirs(config_dirname) return open(config_filename, 'wt') except Exception as e: - printException(e) + print_exception(e) else: return None diff --git a/ScoutSuite/output/utils.py b/ScoutSuite/output/utils.py index 7027fafbd..322c51d12 100644 --- a/ScoutSuite/output/utils.py +++ b/ScoutSuite/output/utils.py @@ -4,7 +4,7 @@ import os import sys -from ScoutSuite.core.console import printError +from ScoutSuite.core.console import print_error from six.moves import input @@ -27,7 +27,7 @@ def prompt_4_yes_no(question): elif choice == 'no' or choice == 'n': return False else: - printError('\'%s\' is not a valid answer. Enter \'yes\'(y) or \'no\'(n).' % choice) + print_error('\'%s\' is not a valid answer. Enter \'yes\'(y) or \'no\'(n).' % choice) def prompt_4_overwrite(filename, force_write): @@ -58,7 +58,7 @@ def get_filename(config_type, profile, report_dir): filename = AWSRULESET_FILE first_line = 'scoutsuite_results =' else: - printError('invalid config type provided (%s)' % config_type) + print_error('invalid config type provided (%s)' % config_type) raise Exception # Append profile name if necessary if profile != 'default' and config_type != AWSRULESET: diff --git a/ScoutSuite/providers/aws/aws.py b/ScoutSuite/providers/aws/aws.py index 4355f8d9f..684a7c365 100644 --- a/ScoutSuite/providers/aws/aws.py +++ b/ScoutSuite/providers/aws/aws.py @@ -6,7 +6,7 @@ import boto3 from botocore.session import Session -from ScoutSuite.core.console import printInfo, printException +from ScoutSuite.core.console import print_info, print_exception def build_region_list(service, chosen_regions=None, partition_name='aws'): @@ -58,10 +58,10 @@ def connect_service(service, credentials, region_name=None, config=None, silent= info_message = 'Connecting to AWS %s' % service if region_name: info_message = info_message + ' in %s' % region_name - printInfo('%s...' % info_message) + print_info('%s...' % info_message) api_client = aws_session.client(**client_params) except Exception as e: - printException(e) + print_exception(e) return api_client diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 3b141af3c..26dced7e9 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -15,7 +15,7 @@ from ScoutSuite.providers.aws.aws import build_region_list, connect_service, get_aws_account_id, get_name, \ handle_truncated_response -from ScoutSuite.core.console import printException, printInfo +from ScoutSuite.core.console import print_exception, print_info from ScoutSuite.providers.base.configs import resource_id_map from ScoutSuite.providers.base.configs.threads import thread_configs @@ -123,7 +123,7 @@ def fetch_all(self, credentials, regions=None, partition_name='aws', targets=Non realtargets = realtargets + ((target[0], target[1], target[2], params, target[4]),) targets['other_regions'] = realtargets - printInfo('Fetching %s config...' % format_service_name(self.service)) + print_info('Fetching %s config...' % format_service_name(self.service)) self.fetchstatuslogger = FetchStatusLogger(targets['first_region'], True) api_service = 'ec2' if self.service.lower() == 'vpc' else self.service.lower() @@ -192,11 +192,11 @@ def _fetch_region(self, q, params): targets) # params['targets']) self.fetchstatuslogger.counts['regions']['fetched'] += 1 except Exception as e: - printException(e) + print_exception(e) finally: q.task_done() except Exception as e: - printException(e) + print_exception(e) pass def _fetch_target(self, q, params): @@ -218,11 +218,11 @@ def _fetch_target(self, q, params): if is_throttled(e): q.put((method, region, backup)) else: - printException(e) + print_exception(e) finally: q.task_done() except Exception as e: - printException(e) + print_exception(e) pass def finalize(self): @@ -310,7 +310,7 @@ def _fetch_targets(self, api_client, q, target): targets = handle_truncated_response(list_method, list_params, [response_attribute])[response_attribute] except Exception as e: if not ignore_list_error: - printException(e) + print_exception(e) targets = [] setattr(self, '%s_count' % target_type, len(targets)) self.fetchstatuslogger.counts[target_type]['discovered'] += len(targets) diff --git a/ScoutSuite/providers/aws/credentials.py b/ScoutSuite/providers/aws/credentials.py index 5f70e0f71..e4b828661 100644 --- a/ScoutSuite/providers/aws/credentials.py +++ b/ScoutSuite/providers/aws/credentials.py @@ -12,8 +12,8 @@ import dateutil.parser import requests # TODO: get rid of that and make sure urllib2 validates certs ? -from ScoutSuite.core.console import printException, printError, printInfo -from ScoutSuite.core.console import prompt_4_mfa_code +from ScoutSuite.core.console import print_exception, print_error, print_info +from ScoutSuite.core.console import prompt_mfa_code from ScoutSuite.providers.aws.aws import connect_service from ScoutSuite.core.fs import save_blob_as_json @@ -169,7 +169,7 @@ def init_sts_session(profile_name, credentials, duration=28800, save_creds=True) # Prompt for MFA code if MFA serial present if 'SerialNumber' in credentials and credentials['SerialNumber']: if not credentials['TokenCode']: - credentials['TokenCode'] = prompt_4_mfa_code() + credentials['TokenCode'] = prompt_mfa_code() if credentials['TokenCode'] == 'q': credentials['SerialNumber'] = None sts_args['TokenCode'] = credentials['TokenCode'] @@ -223,7 +223,7 @@ def read_creds_from_aws_credentials_file(profile_name, credentials_file=aws_cred except Exception as e: # Silent if error is due to no ~/.aws/credentials file if not hasattr(e, 'errno') or e.errno != 2: - printException(e) + print_exception(e) return credentials @@ -356,7 +356,7 @@ def read_profile_from_aws_config_file(profile_name, config_file=aws_config_file) except Exception as e: # Silent if error is due to no .aws/config file if not hasattr(e, 'errno') or e.errno != 2: - printException(e) + print_exception(e) return role_arn, source_profile, mfa_serial, external_id @@ -371,7 +371,7 @@ def show_profiles_from_aws_credentials_file(credentials_files=None): credentials_files = [aws_credentials_file, aws_config_file] profiles = get_profiles_from_aws_credentials_file(credentials_files) for profile in set(profiles): - printInfo(' * %s' % profile) + print_info(' * %s' % profile) def write_creds_to_aws_credentials_file(profile_name, credentials, credentials_file=aws_credentials_file): @@ -530,7 +530,7 @@ def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, credentials['SerialNumber'] = role_mfa_serial # Auto prompt for a code... if not mfa_code: - credentials['TokenCode'] = prompt_4_mfa_code() + credentials['TokenCode'] = prompt_mfa_code() if external_id: credentials['ExternalId'] = external_id credentials = assume_role(profile_name, credentials, role_arn, role_session_name) @@ -543,7 +543,7 @@ def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, expiration = expiration.replace(tzinfo=None) current = datetime.datetime.utcnow() if expiration < current: - printInfo('Saved STS credentials expired on %s' % credentials['Expiration']) + print_info('Saved STS credentials expired on %s' % credentials['Expiration']) force_init = True else: force_init = True @@ -554,7 +554,7 @@ def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, credentials = read_creds_from_aws_credentials_file(profile_name if first_sts_session else '%s-nomfa' % profile_name) if not credentials['AccessKeyId']: - printInfo('Warning: Unable to determine STS token expiration; later API calls may fail.') + print_info('Warning: Unable to determine STS token expiration; later API calls may fail.') credentials = sts_credentials else: if mfa_serial_arg: @@ -566,7 +566,7 @@ def read_creds(profile_name, csv_file=None, mfa_serial_arg=None, mfa_code=None, # If we don't have valid creds by now, print an error message if 'AccessKeyId' not in credentials or credentials['AccessKeyId'] is None or \ 'SecretAccessKey' not in credentials or credentials['SecretAccessKey'] is None: - printError('Error: could not find AWS credentials. Use the --help option for more information.') + print_error('Error: could not find AWS credentials. Use the --help option for more information.') if 'AccessKeyId' not in credentials: credentials = {'AccessKeyId': None} return credentials diff --git a/ScoutSuite/providers/aws/profiles.py b/ScoutSuite/providers/aws/profiles.py index 057e4b98b..98f776f70 100644 --- a/ScoutSuite/providers/aws/profiles.py +++ b/ScoutSuite/providers/aws/profiles.py @@ -3,7 +3,7 @@ import os import re -from ScoutSuite.core.console import printDebug +from ScoutSuite.core.console import print_debug from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.providers.aws.credentials import read_creds @@ -115,7 +115,7 @@ def find_profiles_in_file(filename, names=None, quiet=True): if type(names) != list: names = [names] if not quiet: - printDebug('Searching for profiles matching %s in %s ... ' % (str(names), filename)) + print_debug('Searching for profiles matching %s in %s ... ' % (str(names), filename)) name_filters = [] for name in names: name_filters.append(re.compile('^%s$' % name)) diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index eb50bc268..cf7c7993a 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -3,7 +3,7 @@ import copy import os -from ScoutSuite.core.console import printDebug, printError, printException, printInfo +from ScoutSuite.core.console import print_debug, print_error, print_exception, print_info from ScoutSuite.providers.aws.aws import get_aws_account_id from ScoutSuite.providers.aws.configs.services import AWSServicesConfig from ScoutSuite.providers.aws.credentials import read_creds @@ -136,7 +136,7 @@ def add_security_group_name_to_ec2_grants_callback(self, current_config, path, c @staticmethod def _process_cloudtrail_trails(cloudtrail_config): - printInfo('Processing CloudTrail config...') + print_info('Processing CloudTrail config...') global_events_logging = [] data_logging_trails_count = 0 for region in cloudtrail_config['regions']: @@ -282,7 +282,7 @@ def _update_bucket_permissions(self, s3_info, iam_info, action, iam_entity, allo elif policy_type == 'ManagedPolicies': policy = iam_info['ManagedPolicies'][policy_name]['PolicyDocument'] else: - printError('Error, found unknown policy type.') + print_error('Error, found unknown policy type.') for statement in policy['Statement']: for target_path in statement['NotResource']: parts = target_path.split('/') @@ -340,7 +340,7 @@ def match_instances_and_subnets_callback(self, current_config, path, current_pat subnet['instances'].append(instance_id) def _match_instances_and_roles(self): - printInfo('Matching EC2 instances and IAM roles...') + print_info('Matching EC2 instances and IAM roles...') ec2_config = self.services['ec2'] iam_config = self.services['iam'] role_instances = {} @@ -471,8 +471,8 @@ def match_security_groups_and_resources_callback(self, current_config, path, cur if vpc_id == ec2_classic and resource_type == 'elbs': pass else: - printError('Failed to parse %s in %s in %s' % (resource_type, vpc_id, region)) - printException(e) + print_error('Failed to parse %s in %s in %s' % (resource_type, vpc_id, region)) + print_exception(e) def _merge_route53_and_route53domains(self): if 'route53domains' not in self.services: @@ -504,7 +504,7 @@ def set_emr_vpc_ids_callback(self, current_config, path, current_path, vpc_id, c elif 'RequestedEc2SubnetIds' in cluster['Ec2InstanceAttributes']: subnet_id = cluster['Ec2InstanceAttributes']['RequestedEc2SubnetIds'] else: - printError('Unable to determine VPC id for EMR cluster %s' % str(cluster_id)) + print_error('Unable to determine VPC id for EMR cluster %s' % str(cluster_id)) continue if sg_id in self.sg_map: vpc_id = self.sg_map[sg_id]['vpc_id'] @@ -518,7 +518,7 @@ def set_emr_vpc_ids_callback(self, current_config, path, current_path, vpc_id, c pop_list.append(cluster_id) sid_found = True if not sid_found: - printError('Unable to determine VPC id for %s' % (str(subnet_id) if subnet_id else str(sg_id))) + print_error('Unable to determine VPC id for %s' % (str(subnet_id) if subnet_id else str(sg_id))) continue if vpc_id: # noinspection PyArgumentList @@ -576,7 +576,7 @@ def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_l try: attached_vpc = get_object_at(self, vpc_path) except Exception: - printDebug( + print_debug( 'It appears that the flow log %s is attached to a resource that was previously deleted (%s).' % ( flow_log_id, attached_resource)) return @@ -596,7 +596,7 @@ def sort_vpc_flow_logs_callback(self, current_config, path, current_path, flow_l if flow_log_id not in subnet['flow_logs']: subnet['flow_logs'].append(flow_log_id) else: - printError('Resource %s attached to flow logs is not handled' % attached_resource) + print_error('Resource %s attached to flow logs is not handled' % attached_resource) def get_db_attack_surface(self, current_config, path, current_path, db_id, callback_args): service = current_path[1] diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index a2af3494c..11db0247b 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -8,7 +8,7 @@ # TODO: move a lot of this to VPCconfig, and use some sort of filter to only list SGs in EC2 classic import netaddr -from ScoutSuite.core.console import printException, printInfo +from ScoutSuite.core.console import print_exception, print_info from ScoutSuite.providers.aws.aws import get_name from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig @@ -210,17 +210,17 @@ def __init__(self, service_metadata, thread_config=4): def analyze_ec2_config(ec2_info, aws_account_id, force_write): try: - printInfo('Analyzing EC2 config... ', newLine=False) + print_info('Analyzing EC2 config... ', newLine=False) # Custom EC2 analysis # check_for_elastic_ip(ec2_info) # FIXME - commented for now as this method doesn't seem to be defined anywhere' # list_network_attack_surface(ec2_info, 'attack_surface', 'PublicIpAddress') # TODO: make this optional, commented out for now # list_network_attack_surface(ec2_info, 'private_attack_surface', 'PrivateIpAddress') - printInfo('Success') + print_info('Success') except Exception as e: - printInfo('Error') - printException(e) + print_info('Error') + print_exception(e) def add_security_group_name_to_ec2_grants_callback(ec2_config, current_config, path, current_path, ec2_grant, @@ -287,7 +287,7 @@ def link_elastic_ips_callback2(ec2_config, current_config, path, current_path, i if not 'PublicIpAddress' in current_config: current_config['PublicIpAddress'] = callback_args['elastic_ip'] elif current_config['PublicIpAddress'] != callback_args['elastic_ip']: - printInfo('Warning: public IP address exists (%s) for an instance associated with an elastic IP (%s)' % ( + print_info('Warning: public IP address exists (%s) for an instance associated with an elastic IP (%s)' % ( current_config['PublicIpAddress'], callback_args['elastic_ip'])) # This can happen... fix it diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index c113f5cd5..ca87b0290 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -2,7 +2,7 @@ from botocore.exceptions import ClientError -from ScoutSuite.core.console import printError, printException +from ScoutSuite.core.console import print_error, print_exception from ScoutSuite.providers.aws.aws import connect_service, handle_truncated_response from ScoutSuite.providers.aws.configs.base import AWSBaseConfig from ScoutSuite.utils import * @@ -78,7 +78,7 @@ def fetch_credential_reports(self, credentials, ignore_exception=False): response = api_client.generate_credential_report() if response['State'] != 'COMPLETE': if not ignore_exception: - printError('Failed to generate a credential report.') + print_error('Failed to generate a credential report.') return report = api_client.get_credential_report()['Content'] @@ -109,8 +109,8 @@ def fetch_credential_reports(self, credentials, ignore_exception=False): except Exception as e: if ignore_exception: return - printError('Failed to download a credential report.') - printException(e) + print_error('Failed to download a credential report.') + print_exception(e) def _compute_last_used(self, credential_report): dates = [credential_report['password_last_used'], @@ -215,7 +215,7 @@ def fetch_password_policy(self, credentials): else: raise e except Exception as e: - printError(str(e)) + print_error(str(e)) ######################################## # Roles @@ -341,7 +341,7 @@ def __get_inline_policies(self, api_client, iam_resource_type, resource_id, reso if is_throttled(e): raise e else: - printException(e) + print_exception(e) return fetched_policies try: for policy_name in policy_names: @@ -357,7 +357,7 @@ def __get_inline_policies(self, api_client, iam_resource_type, resource_id, reso if is_throttled(e): raise e else: - printException(e) + print_exception(e) return fetched_policies def __parse_permissions(self, policy_name, policy_document, policy_type, iam_resource_type, resource_name): diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index 96a0f75d7..702e05248 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ScoutSuite.core.console import printError, printException +from ScoutSuite.core.console import print_error, print_exception from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig @@ -85,8 +85,8 @@ def parse_parameter_group(self, global_params, region, parameter_group): parameter_name = parameter.pop('ParameterName') parameter_group['parameters'][parameter_name] = parameter except Exception as e: - printException(e) - printError('Failed fetching DB parameters for %s' % parameter_group['name']) + print_exception(e) + print_error('Failed fetching DB parameters for %s' % parameter_group['name']) # Save parameter_group_id = self.get_non_provider_id(parameter_group['name']) self.parameter_groups[parameter_group_id] = parameter_group diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 0642cae1b..0d474004f 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -7,7 +7,7 @@ from botocore.exceptions import ClientError -from ScoutSuite.core.console import printError, printException, printInfo +from ScoutSuite.core.console import print_error, print_exception, print_info from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.base import AWSBaseConfig from ScoutSuite.utils import manage_dictionary @@ -57,7 +57,7 @@ def parse_buckets(self, bucket, params): bucket['region'] = 'eu-west-1' # h4ck :: S3 is global but region-aware... if bucket['region'] not in params['api_clients']: - printInfo('Skipping bucket %s (region %s outside of scope)' % (bucket['name'], bucket['region'])) + print_info('Skipping bucket %s (region %s outside of scope)' % (bucket['name'], bucket['region'])) self.buckets_count -= 1 return @@ -142,7 +142,7 @@ def update_bucket_permissions(s3_info, iam_info, action, iam_entity, allowed_iam elif policy_type == 'ManagedPolicies': policy = iam_info['ManagedPolicies'][policy_name]['PolicyDocument'] else: - printError('Error, found unknown policy type.') + print_error('Error, found unknown policy type.') for statement in policy['Statement']: for target_path in statement['NotResource']: parts = target_path.split('/') @@ -218,7 +218,7 @@ def get_s3_acls(api_client, bucket_name, bucket, key_name=None): set_s3_permissions(grantees[grantee]['permissions'], permission) return grantees except Exception as e: - printError('Failed to get ACL configuration for %s: %s' % (bucket_name, e)) + print_error('Failed to get ACL configuration for %s: %s' % (bucket_name, e)) return {} @@ -228,7 +228,7 @@ def get_s3_bucket_policy(api_client, bucket_name, bucket_info): return True except Exception as e: if not (type(e) == ClientError and e.response['Error']['Code'] == 'NoSuchBucketPolicy'): - printError('Failed to get bucket policy for %s: %s' % (bucket_name, e)) + print_error('Failed to get bucket policy for %s: %s' % (bucket_name, e)) return False @@ -252,7 +252,7 @@ def get_s3_bucket_secure_transport(api_client, bucket_name, bucket_info): bucket_info['secure_transport_enabled'] = False return True except Exception as e: - printError('Failed to get evaluate bucket policy for %s: %s' % (bucket_name, e)) + print_error('Failed to get evaluate bucket policy for %s: %s' % (bucket_name, e)) bucket_info['secure_transport'] = None return False @@ -286,7 +286,7 @@ def get_s3_bucket_logging(api_client, bucket_name, bucket_info): bucket_info['logging'] = 'Disabled' return True except Exception as e: - printError('Failed to get logging configuration for %s: %s' % (bucket_name, e)) + print_error('Failed to get logging configuration for %s: %s' % (bucket_name, e)) bucket_info['logging'] = 'Unknown' return False @@ -313,11 +313,11 @@ def get_s3_bucket_default_encryption(api_client, bucket_name, bucket_info): bucket_info['default_encryption_enabled'] = False return True else: - printError('Failed to get encryption configuration for %s: %s' % (bucket_name, e)) + print_error('Failed to get encryption configuration for %s: %s' % (bucket_name, e)) bucket_info['default_encryption_enabled'] = None return False except Exception as e: - printError('Failed to get encryption configuration for %s: %s' % (bucket_name, e)) + print_error('Failed to get encryption configuration for %s: %s' % (bucket_name, e)) bucket_info['default_encryption'] = 'Unknown' return False @@ -379,7 +379,7 @@ def get_s3_bucket_keys(api_client, bucket_name, bucket, check_encryption, check_ key['ServerSideEncryption'] = k['ServerSideEncryption'] if 'ServerSideEncryption' in k else None key['SSEKMSKeyId'] = k['SSEKMSKeyId'] if 'SSEKMSKeyId' in k else None except Exception as e: - printException(e) + print_exception(e) continue if check_acls: # noinspection PyBroadException diff --git a/ScoutSuite/providers/azure/provider.py b/ScoutSuite/providers/azure/provider.py index 7875e2369..295d0aeac 100644 --- a/ScoutSuite/providers/azure/provider.py +++ b/ScoutSuite/providers/azure/provider.py @@ -5,7 +5,7 @@ from getpass import getpass -from ScoutSuite.core.console import printError, printException +from ScoutSuite.core.console import print_error, print_exception from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.azure.configs.services import AzureServicesConfig @@ -130,8 +130,8 @@ def authenticate(self, cli=None, msi=None, service_principal=None, file_auth=Non self.credentials = AzureCredentials(credentials, self.aws_account_id) return True except Exception as e: - printError('Failed to authenticate to Azure') - printException(e) + print_error('Failed to authenticate to Azure') + print_exception(e) return False def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): diff --git a/ScoutSuite/providers/azure/utils.py b/ScoutSuite/providers/azure/utils.py index 8e3dae2e6..3eadb44e0 100644 --- a/ScoutSuite/providers/azure/utils.py +++ b/ScoutSuite/providers/azure/utils.py @@ -2,7 +2,7 @@ import re -from ScoutSuite.core.console import printException +from ScoutSuite.core.console import print_exception from azure.mgmt.storage import StorageManagementClient from azure.mgmt.monitor import MonitorManagementClient @@ -38,11 +38,11 @@ def azure_connect_service(service, credentials, region_name=None): elif service == 'loadbalancer': return NetworkManagementClient(credentials.credentials, credentials.subscription_id) else: - printException('Service %s not supported' % service) + print_exception('Service %s not supported' % service) return None except Exception as e: - printException(e) + print_exception(e) return None diff --git a/ScoutSuite/providers/base/configs/base.py b/ScoutSuite/providers/base/configs/base.py index 99dd70a3b..237fefc19 100644 --- a/ScoutSuite/providers/base/configs/base.py +++ b/ScoutSuite/providers/base/configs/base.py @@ -21,7 +21,7 @@ from ScoutSuite.providers.azure.utils import azure_connect_service from ScoutSuite.providers.aws.aws import build_region_list -from ScoutSuite.core.console import printException, printInfo +from ScoutSuite.core.console import print_exception, print_info from ScoutSuite.output.console import FetchStatusLogger from ScoutSuite.utils import format_service_name @@ -78,7 +78,7 @@ def fetch_all(self, credentials, regions=None, partition_name='aws', targets=Non # Initialize targets if not targets: targets = type(self).targets - printInfo('Fetching %s config...' % format_service_name(self.service)) + print_info('Fetching %s config...' % format_service_name(self.service)) formatted_string = None # FIXME the below should be in moved to each provider's code @@ -169,11 +169,11 @@ def __fetch_target(self, q, params): e.response['Error']['Code'] in ['Throttling']: q.put((target_type, backup), ) else: - printException(e) + print_exception(e) finally: q.task_done() except Exception as e: - printException(e) + print_exception(e) pass def __fetch_service(self, q, params): @@ -191,14 +191,14 @@ def __fetch_service(self, q, params): try: method = self._get_method(api_client, target_type, list_method_name) except Exception as e: - printException(e) + print_exception(e) continue try: targets = self._get_targets(response_attribute, api_client, method, list_params, ignore_list_error) except Exception as e: if not ignore_list_error: - printException(e) + print_exception(e) targets = [] self.fetchstatuslogger.counts[target_type]['discovered'] += len(targets) @@ -207,11 +207,11 @@ def __fetch_service(self, q, params): params['q'].put((target_type, target), ) except Exception as e: - printException(e) + print_exception(e) finally: q.task_done() except Exception as e: - printException(e) + print_exception(e) pass def _get_method(self, api_client, target_type, list_method_name): diff --git a/ScoutSuite/providers/base/configs/browser.py b/ScoutSuite/providers/base/configs/browser.py index 5a5e9e171..fd18deb99 100644 --- a/ScoutSuite/providers/base/configs/browser.py +++ b/ScoutSuite/providers/base/configs/browser.py @@ -2,7 +2,7 @@ import copy -from ScoutSuite.core.console import printError, printException +from ScoutSuite.core.console import print_error, print_exception ######################################## @@ -111,12 +111,12 @@ def get_value_at(all_info, current_path, key, to_string=False): try: target_obj = target_obj[p] except Exception as e: - printError('Current path: %s' % str(current_path)) - printException(e) + print_error('Current path: %s' % str(current_path)) + print_exception(e) raise Exception except Exception as e: - printError('Current path: %s' % str(current_path)) - printException(e) + print_error('Current path: %s' % str(current_path)) + print_exception(e) raise Exception if to_string: return str(target_obj) diff --git a/ScoutSuite/providers/base/configs/services.py b/ScoutSuite/providers/base/configs/services.py index 5d19b7317..f7fc27948 100644 --- a/ScoutSuite/providers/base/configs/services.py +++ b/ScoutSuite/providers/base/configs/services.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ScoutSuite.core.console import printError, printException, printDebug +from ScoutSuite.core.console import print_error, print_exception, print_debug from ScoutSuite.providers.aws.aws import get_partition_name class BaseServicesConfig(object): @@ -34,10 +34,10 @@ def fetch(self, credentials, services=None, regions=None): if hasattr(service_config, 'finalize'): service_config.finalize() else: - printDebug('No method to fetch service %s.' % service) + print_debug('No method to fetch service %s.' % service) except Exception as e: - printError('Error: could not fetch %s configuration.' % service) - printException(e) + print_error('Error: could not fetch %s configuration.' % service) + print_exception(e) # TODO is this ever called? # def postprocessing(self): diff --git a/ScoutSuite/providers/base/provider.py b/ScoutSuite/providers/base/provider.py index ea375383a..4d0672a4d 100644 --- a/ScoutSuite/providers/base/provider.py +++ b/ScoutSuite/providers/base/provider.py @@ -7,7 +7,7 @@ import sys import copy -from ScoutSuite.core.console import printException, printInfo +from ScoutSuite.core.console import print_exception, print_info from ScoutSuite import __version__ as scout2_version from ScoutSuite.providers.base.configs.browser import get_object_at @@ -185,7 +185,7 @@ def _update_metadata(self): self.metadata[service_group][service]['resources'][resource]['count'] = \ service_config[count] except Exception as e: - printException(e) + print_exception(e) else: self.metadata[service_group][service]['resources'][resource]['count'] = len(service_config['regions']) @@ -333,11 +333,11 @@ def _go_to_and_do(self, current_config, path, current_path, callback, callback_a callback_args) except Exception as e: - printException(e) - printInfo('Path: %s' % str(current_path)) - printInfo('Key = %s' % str(key) if 'key' in locals() else 'not defined') - printInfo('Value = %s' % str(value) if 'value' in locals() else 'not defined') - printInfo('Path = %s' % path) + print_exception(e) + print_info('Path: %s' % str(current_path)) + print_info('Key = %s' % str(key) if 'key' in locals() else 'not defined') + print_info('Value = %s' % str(value) if 'value' in locals() else 'not defined') + print_info('Path = %s' % path) def _new_go_to_and_do(self, current_config, path, current_path, callbacks): """ @@ -388,8 +388,8 @@ def _new_go_to_and_do(self, current_config, path, current_path, callbacks): tmp.append(i) self._new_go_to_and_do(current_config[key][i], copy.deepcopy(path), tmp, callbacks) except Exception as e: - printException(e) - printInfo('Path: %s' % str(current_path)) - printInfo('Key = %s' % str(key) if 'key' in locals() else 'not defined') - printInfo('Value = %s' % str(value) if 'value' in locals() else 'not defined') - printInfo('Path = %s' % path) + print_exception(e) + print_info('Path: %s' % str(current_path)) + print_info('Key = %s' % str(key) if 'key' in locals() else 'not defined') + print_info('Value = %s' % str(value) if 'value' in locals() else 'not defined') + print_info('Path = %s' % path) diff --git a/ScoutSuite/providers/gcp/configs/base.py b/ScoutSuite/providers/gcp/configs/base.py index f9df42d81..32ba08274 100644 --- a/ScoutSuite/providers/gcp/configs/base.py +++ b/ScoutSuite/providers/gcp/configs/base.py @@ -14,7 +14,7 @@ from google.cloud import container_v1 from googleapiclient.errors import HttpError -from ScoutSuite.core.console import printException, printError +from ScoutSuite.core.console import print_exception, print_error from ScoutSuite.providers.base.configs.base import BaseConfig from ScoutSuite.providers.gcp.utils import gcp_connect_service @@ -58,7 +58,7 @@ def get_regions(self, **kwargs): except HttpError as e: pass except Exception as e: - printException(e) + print_exception(e) if self.regions: break @@ -86,7 +86,7 @@ def get_zones(self, **kwargs): except HttpError as e: pass except Exception as e: - printException(e) + print_exception(e) if self.zones: break @@ -204,22 +204,22 @@ def _get_targets(self, response_attribute, api_client, method, list_params, igno error_json = json.loads(e.content) if error_json['error']['message'] not in self.error_list: self.error_list.append(error_json['error']['message']) - printError(error_json['error']['message']) + print_error(error_json['error']['message']) except PermissionDenied as e: - printError("%s: %s - %s" % (e.message, self.service, self.targets)) + print_error("%s: %s - %s" % (e.message, self.service, self.targets)) except Exception as e: - printException(e) + print_exception(e) except HttpError as e: error_json = json.loads(e.content) if error_json['error']['message'] not in self.error_list: self.error_list.append(error_json['error']['message']) - printError(error_json['error']['message']) + print_error(error_json['error']['message']) except Exception as e: - printException(e) + print_exception(e) finally: return targets diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py index 456ded1d8..212a3adf4 100644 --- a/ScoutSuite/providers/gcp/provider.py +++ b/ScoutSuite/providers/gcp/provider.py @@ -5,7 +5,7 @@ import google.auth import googleapiclient -from ScoutSuite.core.console import printError, printException, printInfo +from ScoutSuite.core.console import print_error, print_exception, print_info from ScoutSuite.providers.base.provider import BaseProvider from ScoutSuite.providers.gcp.configs.services import GCPServicesConfig @@ -62,7 +62,7 @@ def authenticate(self, user_account=None, service_account=None, **kargs): client_secrets_path = os.path.abspath(service_account) os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = client_secrets_path else: - printError('Failed to authenticate to GCP - no supported account type') + print_error('Failed to authenticate to GCP - no supported account type') return False try: @@ -109,7 +109,7 @@ def authenticate(self, user_account=None, service_account=None, **kargs): # Raise exception if none of the above else: - printInfo("Could not infer the Projects to scan and no default Project ID was found.") + print_info("Could not infer the Projects to scan and no default Project ID was found.") return False # TODO this shouldn't be done here? but it has to in order to init with projects... @@ -120,13 +120,13 @@ def authenticate(self, user_account=None, service_account=None, **kargs): return False except google.auth.exceptions.DefaultCredentialsError as e: - printError('Failed to authenticate to GCP') - printException(e) + print_error('Failed to authenticate to GCP') + print_exception(e) return False except googleapiclient.errors.HttpError as e: - printError('Failed to authenticate to GCP') - printException(e) + print_error('Failed to authenticate to GCP') + print_exception(e) return False def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None): @@ -208,11 +208,11 @@ def _get_projects(self, parent_type, parent_id): for folder in folder_response['folders']: projects.extend(self._get_projects("folder", folder['name'].strip(u'folders/'))) - printInfo("Found {} project(s) to scan.".format(len(projects))) + print_info("Found {} project(s) to scan.".format(len(projects))) except Exception as e: - printError('Unable to list accessible Projects') - printException(e) + print_error('Unable to list accessible Projects') + print_exception(e) finally: return projects diff --git a/ScoutSuite/providers/gcp/services/cloudresourcemanager.py b/ScoutSuite/providers/gcp/services/cloudresourcemanager.py index f44056b93..83e21defe 100644 --- a/ScoutSuite/providers/gcp/services/cloudresourcemanager.py +++ b/ScoutSuite/providers/gcp/services/cloudresourcemanager.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from ScoutSuite.core.console import printException +from ScoutSuite.core.console import print_exception class CloudResourceManager(GCPBaseConfig): @@ -44,7 +44,7 @@ def parse_projects(self, binding, params): elif type == 'serviceAccount': binding_dict['members']['service_accounts'].append(entity) else: - printException('Type %s not handled' % type) + print_exception('Type %s not handled' % type) self.bindings[binding_dict['id']] = binding_dict diff --git a/ScoutSuite/providers/gcp/services/cloudsql.py b/ScoutSuite/providers/gcp/services/cloudsql.py index 7ffc2ce27..a34a3f444 100644 --- a/ScoutSuite/providers/gcp/services/cloudsql.py +++ b/ScoutSuite/providers/gcp/services/cloudsql.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ScoutSuite.core.console import printError +from ScoutSuite.core.console import print_error from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig @@ -56,7 +56,7 @@ def _get_users(self, instance, params): users_dict[user['name']] = self._parse_user(user) except Exception as e: - printError('Failed to fetch users for SQL instance %s: %s' % (instance['name'], e)) + print_error('Failed to fetch users for SQL instance %s: %s' % (instance['name'], e)) return users_dict @@ -77,7 +77,7 @@ def _get_instance_backups(self, instance, params): except AttributeError as e: return backups_dict except Exception as e: - printError('Failed to fetch backups for SQL instance %s: %s' % (instance['name'], e)) + print_error('Failed to fetch backups for SQL instance %s: %s' % (instance['name'], e)) return backups_dict try: @@ -94,7 +94,7 @@ def _get_instance_backups(self, instance, params): return backups_dict except Exception as e: - printError('Failed to parse backups for SQL instance %s: %s' % (instance['name'], e)) + print_error('Failed to parse backups for SQL instance %s: %s' % (instance['name'], e)) return None def _is_log_enabled(self, instance) : diff --git a/ScoutSuite/providers/gcp/services/cloudstorage.py b/ScoutSuite/providers/gcp/services/cloudstorage.py index b558dffed..2bb24db6c 100644 --- a/ScoutSuite/providers/gcp/services/cloudstorage.py +++ b/ScoutSuite/providers/gcp/services/cloudstorage.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ScoutSuite.core.console import printError +from ScoutSuite.core.console import print_error from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig @@ -56,7 +56,7 @@ def get_cloudstorage_bucket_logging(bucket, bucket_dict): bucket_dict['logging_enabled'] = bucket.get_logging() is not None return True except Exception as e: - printError('Failed to get bucket logging configuration for %s: %s' % (bucket.name, e)) + print_error('Failed to get bucket logging configuration for %s: %s' % (bucket.name, e)) bucket_dict['logging_enabled'] = None return False @@ -77,6 +77,6 @@ def get_cloudstorage_bucket_acl(bucket, bucket_dict): return True except Exception as e: - printError('Failed to get bucket ACL configuration for %s: %s' % (bucket.name, e)) + print_error('Failed to get bucket ACL configuration for %s: %s' % (bucket.name, e)) bucket_dict['acls'] = 'Unknown' return False diff --git a/ScoutSuite/providers/gcp/services/iam.py b/ScoutSuite/providers/gcp/services/iam.py index be9d91f78..4a2859087 100644 --- a/ScoutSuite/providers/gcp/services/iam.py +++ b/ScoutSuite/providers/gcp/services/iam.py @@ -2,7 +2,7 @@ from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from ScoutSuite.core.console import printError +from ScoutSuite.core.console import print_error from googleapiclient import discovery @@ -72,7 +72,7 @@ def _get_service_account_keys(self, api_client, project_id, service_account_emai else: return None except Exception as e: - printError('Failed to get keys for service account %s: %s' % (service_account_email, e)) + print_error('Failed to get keys for service account %s: %s' % (service_account_email, e)) return None def _get_service_account_iam_policy(self, api_client, project_id, service_account_email): @@ -89,5 +89,5 @@ def _get_service_account_iam_policy(self, api_client, project_id, service_accoun else: return None except Exception as e: - printError('Failed to get IAM policy for service account %s: %s' % (service_account_email, e)) + print_error('Failed to get IAM policy for service account %s: %s' % (service_account_email, e)) return None diff --git a/ScoutSuite/providers/gcp/utils.py b/ScoutSuite/providers/gcp/utils.py index 0c18a3d92..96186dfb0 100644 --- a/ScoutSuite/providers/gcp/utils.py +++ b/ScoutSuite/providers/gcp/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ScoutSuite.core.console import printException +from ScoutSuite.core.console import print_exception from google.cloud import storage from google.cloud import logging as stackdriver_logging @@ -51,11 +51,11 @@ def gcp_connect_service(service, credentials=None, region_name=None): return container_v1.ClusterManagerClient() else: - printException('Service %s not supported' % service) + print_exception('Service %s not supported' % service) return None except Exception as e: - printException(e) + print_exception(e) return None diff --git a/tests/test-rules-processingengine.py b/tests/test-rules-processingengine.py index 44efaa68c..e6cff5799 100644 --- a/tests/test-rules-processingengine.py +++ b/tests/test-rules-processingengine.py @@ -4,7 +4,7 @@ import os import tempfile -from ScoutSuite.core.console import configPrintException, printError +from ScoutSuite.core.console import config_debug_level, print_error from ScoutSuite.core.processingengine import ProcessingEngine from ScoutSuite.core.ruleset import Ruleset @@ -17,7 +17,7 @@ class DummyObject(object): class TestAWSScout2RulesProcessingEngine: def setup(self): - configPrintException(True) + config_debug_level(True) self.rule_counters = {'found': 0, 'tested': 0, 'verified': 0} self.test_dir = os.path.dirname(os.path.realpath(__file__)) @@ -64,8 +64,8 @@ def _test_rule(self, ruleset_file_name, rule_file_name, rule): test_result_file_name = os.path.join(self.test_dir, 'data/rule-results/%s' % rule_file_name) if not os.path.isfile(test_result_file_name): - printError('Expected findings:: ') - printError(json.dumps(findings, indent=4)) + print_error('Expected findings:: ') + print_error(json.dumps(findings, indent=4)) return self.rule_counters['verified'] += 1 @@ -75,8 +75,8 @@ def _test_rule(self, ruleset_file_name, rule_file_name, rule): try: assert (set(sorted(findings)) == set(sorted(items))) except Exception: - printError('Expected items:\n %s' % json.dumps(sorted(items))) - printError('Reported items:\n %s' % json.dumps(sorted(findings))) + print_error('Expected items:\n %s' % json.dumps(sorted(items))) + print_error('Reported items:\n %s' % json.dumps(sorted(findings))) assert (False) def _generate_ruleset(self, rule_file_name, rule): diff --git a/tests/test-rules-ruleset.py b/tests/test-rules-ruleset.py index f336c3074..1d89c404f 100644 --- a/tests/test-rules-ruleset.py +++ b/tests/test-rules-ruleset.py @@ -3,7 +3,7 @@ import os from mock import patch -from ScoutSuite.core.console import configPrintException, printDebug +from ScoutSuite.core.console import config_debug_level, print_debug from ScoutSuite.core.rule import Rule from ScoutSuite.core.ruleset import Ruleset @@ -12,7 +12,7 @@ class TestAWSScout2RulesRuleset: def setup(self): - configPrintException(True) + config_debug_level(True) self.test_dir = os.path.dirname(os.path.realpath(__file__)) self.test_ruleset_001 = os.path.join(self.test_dir, 'data/test-ruleset.json') @@ -32,17 +32,17 @@ def test_ruleset_class(self, printError): assert (type(test001.rules[test_file_key][0] == Rule)) assert (hasattr(test001.rules[test_file_key][0], 'path')) for rule in test001.rules: - printDebug(test001.rules[rule][0].to_string()) + print_debug(test001.rules[rule][0].to_string()) assert (test_file_key in test001.rule_definitions) assert (test001.rule_definitions[test_file_key].description == "Password expiration disabled") for rule_def in test001.rule_definitions: - printDebug(str(test001.rule_definitions[rule_def])) + print_debug(str(test001.rule_definitions[rule_def])) assert (printError.call_count == 0) test002 = Ruleset(filename=self.test_ruleset_002) for rule in test002.rules: - printDebug(test002.rules[rule][0].to_string()) + print_debug(test002.rules[rule][0].to_string()) assert (printError.call_count == 1) # is this expected ?? assert ("test-ruleset-absolute-path.json does not exist." in printError.call_args_list[0][0][0]) diff --git a/tests/test-scoutsuite.py b/tests/test-scoutsuite.py index 37b40b81b..5caf2c5c7 100644 --- a/tests/test-scoutsuite.py +++ b/tests/test-scoutsuite.py @@ -15,7 +15,7 @@ class TestScoutSuiteClass: @classmethod def setUpClass(cls): - configPrintException(True) + config_debug_level(True) creds = read_creds_from_environment_variables() cls.profile_name = 'travislike' if creds['AccessKeyId'] == None else None cls.has_run_scout_suite = False diff --git a/tests/test-utils_cloudwatch.py b/tests/test-utils_cloudwatch.py index 1bde42713..43e420212 100644 --- a/tests/test-utils_cloudwatch.py +++ b/tests/test-utils_cloudwatch.py @@ -1,12 +1,12 @@ from ScoutSuite.utils_cloudwatch import * -from ScoutSuite.core.console import configPrintException +from ScoutSuite.core.console import config_debug_level # # Test for Scout2 CloudWatch functions # class TestScout2CloudWatchUtilsClass: - configPrintException(True) + config_debug_level(True) # # Test get_cloudwatch_region in us-east-1 diff --git a/tests/test-utils_sns.py b/tests/test-utils_sns.py index 881027b85..5cdba2356 100644 --- a/tests/test-utils_sns.py +++ b/tests/test-utils_sns.py @@ -1,12 +1,12 @@ from ScoutSuite.utils_sns import * -from ScoutSuite.core.console import configPrintException +from ScoutSuite.core.console import config_debug_level # # Test for Scout2 SNS functions # class TestScout2SNSUtilsClass: - configPrintException(True) + config_debug_level(True) # # Test get_sns_region in us-east-1 From e4581a5b130a76d9fc1044276be62440c08dbad1 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sun, 24 Feb 2019 13:19:36 -0500 Subject: [PATCH 069/154] [Opinel] Removed dead code --- ScoutSuite/core/conditions.py | 16 ++++++------ ScoutSuite/core/fs.py | 46 ++++------------------------------- 2 files changed, 13 insertions(+), 49 deletions(-) diff --git a/ScoutSuite/core/conditions.py b/ScoutSuite/core/conditions.py index c8fc36ecf..e8ace2658 100644 --- a/ScoutSuite/core/conditions.py +++ b/ScoutSuite/core/conditions.py @@ -43,7 +43,7 @@ def pass_conditions(all_info, current_path, conditions, unknown_as_pass_conditio if dynamic_value: test_values = get_value_at(all_info, current_path, dynamic_value.groups()[0], True) try: - res = _pass_condition(target_obj, test_name, test_values) + res = pass_condition(target_obj, test_name, test_values) except Exception as e: res = True if unknown_as_pass_condition else False print_error('Unable to process testcase \'%s\' on value \'%s\', interpreted as %s.' % ( @@ -77,7 +77,7 @@ def __prepare_age_test(a, b): return age, number -def _pass_condition(b, test, a): +def pass_condition(b, test, a): """ Generic test function used by Scout2 / AWS recipes . @@ -97,7 +97,7 @@ def _pass_condition(b, test, a): b = str(b) result = (a == b) elif test == 'notEqual': - result = (not _pass_condition(b, 'equal', a)) + result = (not pass_condition(b, 'equal', a)) # More/Less tests elif test == 'lessThan': @@ -113,11 +113,11 @@ def _pass_condition(b, test, a): elif test == 'empty': result = ((type(b) == dict and b == {}) or (type(b) == list and b == []) or (type(b) == list and b == [None])) elif test == 'notEmpty': - result = (not _pass_condition(b, 'empty', 'a')) + result = (not pass_condition(b, 'empty', 'a')) elif test == 'null': result = ((b is None) or (type(b) == str and b == 'None')) elif test == 'notNull': - result = (not _pass_condition(b, 'null', a)) + result = (not pass_condition(b, 'null', a)) # Boolean tests elif test == 'true': @@ -183,7 +183,7 @@ def _pass_condition(b, test, a): result = True break elif test == 'notMatch': - result = (not _pass_condition(b, 'match', a)) + result = (not pass_condition(b, 'match', a)) # Date tests elif test == 'priorToDate': @@ -209,7 +209,7 @@ def _pass_condition(b, test, a): result = True break elif test == 'notInSubnets': - result = (not _pass_condition(b, 'inSubnets', a)) + result = (not pass_condition(b, 'inSubnets', a)) # Policy statement tests elif test == 'containAction': @@ -223,7 +223,7 @@ def _pass_condition(b, test, a): result = True break elif test == 'notContainAction': - result = (not _pass_condition(b, 'containAction', a)) + result = (not pass_condition(b, 'containAction', a)) elif test == 'containAtLeastOneAction': result = False if type(b) != dict: diff --git a/ScoutSuite/core/fs.py b/ScoutSuite/core/fs.py index 6cfcdced8..9a2ce123a 100644 --- a/ScoutSuite/core/fs.py +++ b/ScoutSuite/core/fs.py @@ -4,10 +4,9 @@ import datetime import json import os -import yaml -from ScoutSuite.core.console import print_error, print_exception, prompt_overwrite -from ScoutSuite.core.conditions import _pass_condition +from ScoutSuite.core.console import print_exception, prompt_overwrite +from ScoutSuite.core.conditions import pass_condition class CustomJSONEncoder(json.JSONEncoder): @@ -72,7 +71,7 @@ def read_ip_ranges(filename, local_file=True, ip_only=False, conditions=[]): for condition in conditions: if type(condition) != list or len(condition) < 3: continue - condition_passed = _pass_condition(d[condition[0]], condition[1], condition[2]) + condition_passed = pass_condition(d[condition[0]], condition[1], condition[2]) if not condition_passed: break if condition_passed: @@ -86,7 +85,7 @@ def read_ip_ranges(filename, local_file=True, ip_only=False, conditions=[]): return targets -def read_file(file_path, mode='rt'): +def read_file(file_path): """ Read the contents of a file @@ -94,7 +93,7 @@ def read_file(file_path, mode='rt'): :return: Contents of the file """ - with open(file_path, mode) as f: + with open(file_path, 'rt') as f: contents = f.read() return contents @@ -118,38 +117,3 @@ def save_blob_as_json(filename, blob, force_write, debug): except Exception as e: print_exception(e) pass - - -def save_ip_ranges(profile_name, prefixes, force_write, debug, output_format='json'): - """ - Creates/Modifies an ip-range-XXX.json file - - :param profile_name: - :param prefixes: - :param force_write: - :param debug: - - :return: - """ - filename = 'ip-ranges-%s.json' % profile_name - ip_ranges = {} - ip_ranges['createDate'] = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') - # Unique prefixes - unique_prefixes = {} - for prefix in prefixes: - if type(prefix) == dict: - unique_prefixes[prefix['ip_prefix']] = prefix - else: - unique_prefixes[prefix] = {'ip_prefix': prefix} - unique_prefixes = list(unique_prefixes.values()) - ip_ranges['prefixes'] = unique_prefixes - if output_format == 'json': - save_blob_as_json(filename, ip_ranges, force_write, debug) - else: - # Write as CSV - output = 'account_id, region, ip, instance_id, instance_name\n' - for prefix in unique_prefixes: - output += '%s, %s, %s, %s, %s\n' % ( - prefix['account_id'], prefix['region'], prefix['ip_prefix'], prefix['instance_id'], prefix['name']) - with open('ip-ranges-%s.csv' % profile_name, 'wt') as f: - f.write(output) From 2d84eeb3c5aebc7f52c49b3ff8d0ee376ba708f2 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sun, 24 Feb 2019 13:26:55 -0500 Subject: [PATCH 070/154] [Opinel] Removed dead code --- ScoutSuite/core/fs.py | 4 ++-- ScoutSuite/providers/aws/credentials.py | 2 +- ScoutSuite/utils.py | 11 ++++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/core/fs.py b/ScoutSuite/core/fs.py index 9a2ce123a..4899e87fa 100644 --- a/ScoutSuite/core/fs.py +++ b/ScoutSuite/core/fs.py @@ -98,7 +98,7 @@ def read_file(file_path): return contents -def save_blob_as_json(filename, blob, force_write, debug): +def save_blob_as_json(filename, blob, force_write): """ Creates/Modifies file and saves python object as JSON @@ -112,7 +112,7 @@ def save_blob_as_json(filename, blob, force_write, debug): try: if prompt_overwrite(filename, force_write): with open(filename, 'wt') as f: - print('%s' % json.dumps(blob, indent=4 if debug else None, separators=(',', ': '), sort_keys=True, + print('%s' % json.dumps(blob, indent=4, separators=(',', ': '), sort_keys=True, cls=CustomJSONEncoder), file=f) except Exception as e: print_exception(e) diff --git a/ScoutSuite/providers/aws/credentials.py b/ScoutSuite/providers/aws/credentials.py index e4b828661..b6fd7197d 100644 --- a/ScoutSuite/providers/aws/credentials.py +++ b/ScoutSuite/providers/aws/credentials.py @@ -88,7 +88,7 @@ def assume_role(role_name, credentials, role_arn, role_session_name, silent=Fals cached_credentials_path = os.path.dirname(cached_credentials_filename) if not os.path.isdir(cached_credentials_path): os.makedirs(cached_credentials_path) - save_blob_as_json(cached_credentials_filename, sts_response, True, False) # blob, force_write, debug): + save_blob_as_json(cached_credentials_filename, sts_response, True) return credentials diff --git a/ScoutSuite/utils.py b/ScoutSuite/utils.py index 9f15da47e..18b2b9f47 100644 --- a/ScoutSuite/utils.py +++ b/ScoutSuite/utils.py @@ -3,8 +3,6 @@ import re - - ######################################## # Globals ######################################## @@ -68,7 +66,6 @@ def get_keys(src, dst, keys): :return: """ for key in keys: - #dst[no_camel(key)] = src[key] if key in src else None dst[key] = src[key] if key in src else None @@ -90,7 +87,11 @@ def is_throttled(e): :param e: Exception raised :return: True if it's a throttling exception else False """ - return True if (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in [ 'Throttling', 'RequestLimitExceeded', 'ThrottlingException' ]) else False + return True if (hasattr(e, 'response') + and 'Error' in e.response + and e.response['Error']['Code'] in ['Throttling', + 'RequestLimitExceeded', + 'ThrottlingException']) else False def manage_dictionary(dictionary, key, init, callback=None): @@ -108,4 +109,4 @@ def manage_dictionary(dictionary, key, init, callback=None): manage_dictionary(dictionary, key, init) if callback: callback(dictionary[key]) - return dictionary \ No newline at end of file + return dictionary From 8e056de3fe464eb932cd685c372e3a64ded73924 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Sun, 24 Feb 2019 13:32:46 -0500 Subject: [PATCH 071/154] [Opinel] Fixed tests --- tests/test-main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-main.py b/tests/test-main.py index 209ec0988..5d752d8d6 100644 --- a/tests/test-main.py +++ b/tests/test-main.py @@ -19,7 +19,7 @@ def setUp(self): self.mocked_printInfo = MagicMock() - for import_name, mocked_object in [("printInfo", self.mocked_printInfo), + for import_name, mocked_object in [("print_info", self.mocked_printInfo), ("get_provider", self.mocked_provider), ("Ruleset", self.mocked_ruleset), ("ProcessingEngine", self.mocked_engine), From 6d85284d72ee036eb7a5b2db2b85a7f9a9d6a780 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Sun, 24 Feb 2019 16:53:06 -0500 Subject: [PATCH 072/154] Reverted exception --- ScoutSuite/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/__main__.py b/ScoutSuite/__main__.py index 70d869e24..adfe572ca 100644 --- a/ScoutSuite/__main__.py +++ b/ScoutSuite/__main__.py @@ -18,6 +18,7 @@ from ScoutSuite.providers import get_provider +# noinspection PyBroadException def main(args=None): """ Main method that runs a scan @@ -128,7 +129,6 @@ def main(args=None): exceptions = exceptions.exceptions except Exception as e: print_debug('Warning, failed to load exceptions. The file may not exist or may have an invalid format.') - print(e) exceptions = {} # Finalize From 6486d8b004c18308fc55b70a7923546ab3455fdf Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Sun, 24 Feb 2019 17:18:26 -0500 Subject: [PATCH 073/154] Removed dead code, initialized class variables --- ScoutSuite/providers/aws/profiles.py | 3 --- ScoutSuite/providers/aws/provider.py | 3 --- ScoutSuite/providers/aws/services/awslambda.py | 1 + ScoutSuite/providers/aws/services/cloudformation.py | 3 ++- ScoutSuite/providers/aws/services/cloudtrail.py | 1 + ScoutSuite/providers/aws/services/cloudwatch.py | 1 + ScoutSuite/providers/aws/services/directconnect.py | 1 + ScoutSuite/providers/aws/services/ec2.py | 3 ++- ScoutSuite/providers/aws/services/efs.py | 3 ++- ScoutSuite/providers/aws/services/elasticache.py | 1 + ScoutSuite/providers/aws/services/elb.py | 3 ++- ScoutSuite/providers/aws/services/elbv2.py | 3 ++- ScoutSuite/providers/aws/services/iam.py | 11 +++++++---- ScoutSuite/providers/aws/services/redshift.py | 1 + ScoutSuite/providers/aws/services/s3.py | 9 +++------ ScoutSuite/providers/aws/services/ses.py | 1 + ScoutSuite/providers/aws/services/sns.py | 3 +++ ScoutSuite/providers/aws/services/sqs.py | 1 + ScoutSuite/providers/aws/services/vpc.py | 7 ++++++- 19 files changed, 37 insertions(+), 22 deletions(-) diff --git a/ScoutSuite/providers/aws/profiles.py b/ScoutSuite/providers/aws/profiles.py index 98f776f70..e630a1983 100644 --- a/ScoutSuite/providers/aws/profiles.py +++ b/ScoutSuite/providers/aws/profiles.py @@ -36,9 +36,6 @@ def get_credentials(self): pass return self.credentials - def set_attribute(self, attribute, value): - self.attributes[attribute] = value - def parse_raw_profile(self): for line in self.raw_profile.split('\n')[1:]: line = line.strip() diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index cf7c7993a..d0e7c2b14 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -538,9 +538,6 @@ def _parse_elb_policies(self): self.parse_elb_policies_callback, {}) - # if 'elbv2' in self.config['services']: - # Do something too here... - # noinspection PyArgumentList @staticmethod def parse_elb_policies_callback(current_config, path, current_path, region_id, callback_args): diff --git a/ScoutSuite/providers/aws/services/awslambda.py b/ScoutSuite/providers/aws/services/awslambda.py index 02ac22568..6e4553b67 100644 --- a/ScoutSuite/providers/aws/services/awslambda.py +++ b/ScoutSuite/providers/aws/services/awslambda.py @@ -11,6 +11,7 @@ ######################################## class LambdaRegionConfig(RegionConfig): + functions = {} def parse_function(self, global_params, region, function): function['name'] = function.pop('FunctionName') diff --git a/ScoutSuite/providers/aws/services/cloudformation.py b/ScoutSuite/providers/aws/services/cloudformation.py index 2b683c937..300ba5fd1 100644 --- a/ScoutSuite/providers/aws/services/cloudformation.py +++ b/ScoutSuite/providers/aws/services/cloudformation.py @@ -13,14 +13,15 @@ class CloudFormationRegionConfig(RegionConfig): """ CloudFormation configuration for a single AWS region """ + stacks = {} def parse_stack(self, global_params, region, stack): """ Parse a single stack and fetch additional attributes + :param stack: :param global_params: Parameters shared for all regions :param region: Name of the AWS region - :param stack_url: URL of the AWS stack """ stack['id'] = stack.pop('StackId') stack['name'] = stack.pop('StackName') diff --git a/ScoutSuite/providers/aws/services/cloudtrail.py b/ScoutSuite/providers/aws/services/cloudtrail.py index c6dcecd71..05cca58bf 100644 --- a/ScoutSuite/providers/aws/services/cloudtrail.py +++ b/ScoutSuite/providers/aws/services/cloudtrail.py @@ -15,6 +15,7 @@ class CloudTrailRegionConfig(RegionConfig): """ CloudTrail configuration for a single AWS region """ + trails = {} def parse_trail(self, global_params, region, trail): """ diff --git a/ScoutSuite/providers/aws/services/cloudwatch.py b/ScoutSuite/providers/aws/services/cloudwatch.py index d9bf11acb..c1cf21c98 100644 --- a/ScoutSuite/providers/aws/services/cloudwatch.py +++ b/ScoutSuite/providers/aws/services/cloudwatch.py @@ -14,6 +14,7 @@ class CloudWatchRegionConfig(RegionConfig): """ CloudWatch configuration for a single AWS region """ + alarms = {} def parse_alarm(self, global_params, region, alarm): """ diff --git a/ScoutSuite/providers/aws/services/directconnect.py b/ScoutSuite/providers/aws/services/directconnect.py index 3320c50af..d9b46ac93 100644 --- a/ScoutSuite/providers/aws/services/directconnect.py +++ b/ScoutSuite/providers/aws/services/directconnect.py @@ -11,6 +11,7 @@ class DirectConnectRegionConfig(RegionConfig): """ DirectConnect configuration for a single AWS region """ + connections = {} def parse_connection(self, global_params, region, connection): """ diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 11db0247b..36926cb9f 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -32,12 +32,13 @@ class EC2RegionConfig(RegionConfig): """ EC2 configuration for a single AWS region """ + elastic_ips = {} def parse_elastic_ip(self, global_params, region, eip): """ :param global_params: - :param region: + :param region: Name of the AWS region :param eip: :return: """ diff --git a/ScoutSuite/providers/aws/services/efs.py b/ScoutSuite/providers/aws/services/efs.py index 2c6568f6b..ab4e6186a 100644 --- a/ScoutSuite/providers/aws/services/efs.py +++ b/ScoutSuite/providers/aws/services/efs.py @@ -15,12 +15,13 @@ class EFSRegionConfig(RegionConfig): """ EFS configuration for a single AWS region """ + file_systems = {} def parse_file_system(self, global_params, region, file_system): """ :param global_params: - :param region: + :param region: Name of the AWS region :param file_system: :return: """ diff --git a/ScoutSuite/providers/aws/services/elasticache.py b/ScoutSuite/providers/aws/services/elasticache.py index 3c642de0c..c025285b7 100644 --- a/ScoutSuite/providers/aws/services/elasticache.py +++ b/ScoutSuite/providers/aws/services/elasticache.py @@ -13,6 +13,7 @@ class ElastiCacheRegionConfig(RegionConfig): """ ElastiCache configuration for a single AWS region """ + security_groups = {} def parse_cluster(self, global_params, region, cluster): """ diff --git a/ScoutSuite/providers/aws/services/elb.py b/ScoutSuite/providers/aws/services/elb.py index ab81f48bf..811e51405 100644 --- a/ScoutSuite/providers/aws/services/elb.py +++ b/ScoutSuite/providers/aws/services/elb.py @@ -16,13 +16,14 @@ class ELBRegionConfig(RegionConfig): """ ELB configuration for a single AWS region """ + elb_policies = {} def parse_elb(self, global_params, region, lb): """ :param lb: :param global_params: - :param region: + :param region: Name of the AWS region :return: """ elb = {'name': lb.pop('LoadBalancerName')} diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index de26cca3b..04079d0cf 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -21,13 +21,14 @@ class ELBv2RegionConfig(RegionConfig): :ivar vpcs: Dictionary of VPCs [id] """ + ssl_policies = {} def parse_lb(self, global_params, region, lb): """ :param lb: :param global_params: - :param region: + :param region: Name of the AWS region :return: """ lb['arn'] = lb.pop('LoadBalancerArn') diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index ca87b0290..238937b26 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -112,7 +112,8 @@ def fetch_credential_reports(self, credentials, ignore_exception=False): print_error('Failed to download a credential report.') print_exception(e) - def _compute_last_used(self, credential_report): + @staticmethod + def _compute_last_used(credential_report): dates = [credential_report['password_last_used'], credential_report['access_key_1_last_used_date'], credential_report['access_key_2_last_used_date']] @@ -120,7 +121,8 @@ def _compute_last_used(self, credential_report): dates = [date for date in dates if date is not None] return max(dates) if len(dates) > 0 else None - def _sanitize_date(self, date): + @staticmethod + def _sanitize_date(date): """ Returns the date if it is not equal to 'N/A' or 'no_information', else returns None """ @@ -143,7 +145,7 @@ def parse_groups(self, group, params): group['name'] = group.pop('GroupName') group['arn'] = group.pop('Arn') # Get group's members - group['users'] = self.__fetch_group_users(api_client, group['name']); + group['users'] = self.__fetch_group_users(api_client, group['name']) # Get inline policies policies = self.__get_inline_policies(api_client, 'group', group['id'], group['name']) if len(policies): @@ -319,7 +321,8 @@ def get_id_for_resource(self, iam_resource_type, resource_name): if getattr(self, iam_resource_type)[resource_id]['name'] == resource_name: return resource_id - def __fetch_group_users(self, api_client, group_name): + @staticmethod + def __fetch_group_users(api_client, group_name): users = [] fetched_users = api_client.get_group(GroupName=group_name)['Users'] for user in fetched_users: diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index 29b8576a9..aac05e8f5 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -18,6 +18,7 @@ class RedshiftRegionConfig(RegionConfig): """ Redshift configuration for a single AWS region """ + parameter_groups = {} def parse_cluster(self, global_params, region, cluster): """ diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 0d474004f..5a3f8c4b1 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -112,7 +112,7 @@ def update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, bucket = s3_info['buckets'][bucket_name] manage_dictionary(bucket, iam_entity, {}) manage_dictionary(bucket, iam_entity + '_count', 0) - if not allowed_iam_entity in bucket[iam_entity]: + if allowed_iam_entity not in bucket[iam_entity]: bucket[iam_entity][allowed_iam_entity] = {} bucket[iam_entity + '_count'] = bucket[iam_entity + '_count'] + 1 @@ -133,6 +133,7 @@ def update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, def update_bucket_permissions(s3_info, iam_info, action, iam_entity, allowed_iam_entity, full_path, policy_type, policy_name): + global policy allowed_buckets = [] # By default, all buckets are allowed for bucket_name in s3_info['buckets']: @@ -162,11 +163,7 @@ def update_bucket_permissions(s3_info, iam_info, action, iam_entity, allowed_iam def init_s3_permissions(): - permissions = {} - permissions['read'] = False - permissions['write'] = False - permissions['read_acp'] = False - permissions['write_acp'] = False + permissions = {'read': False, 'write': False, 'read_acp': False, 'write_acp': False} return permissions diff --git a/ScoutSuite/providers/aws/services/ses.py b/ScoutSuite/providers/aws/services/ses.py index f0af048a5..474da005a 100644 --- a/ScoutSuite/providers/aws/services/ses.py +++ b/ScoutSuite/providers/aws/services/ses.py @@ -13,6 +13,7 @@ class SESRegionConfig(RegionConfig): """ SES configuration for a single AWS region """ + identities = {} def parse_identitie(self, global_params, region, identity_name): """ diff --git a/ScoutSuite/providers/aws/services/sns.py b/ScoutSuite/providers/aws/services/sns.py index 8dce5e096..a8413e939 100644 --- a/ScoutSuite/providers/aws/services/sns.py +++ b/ScoutSuite/providers/aws/services/sns.py @@ -17,11 +17,13 @@ class SNSRegionConfig(RegionConfig): """ SNS configuration for a single AWS region """ + topics = {} def parse_subscription(self, params, region, subscription): """ Parse a single subscription and reference it in its corresponding topic + :param region: :param params: Global parameters (defaults to {}) :param subscription: SNS Subscription """ @@ -39,6 +41,7 @@ def parse_topic(self, params, region, topic): """ Parse a single topic and fetch additional attributes + :param region: Name of the AWS region :param params: Global parameters (defaults to {}) :param topic: SNS Topic """ diff --git a/ScoutSuite/providers/aws/services/sqs.py b/ScoutSuite/providers/aws/services/sqs.py index d4d512d8a..11f7f1fde 100644 --- a/ScoutSuite/providers/aws/services/sqs.py +++ b/ScoutSuite/providers/aws/services/sqs.py @@ -13,6 +13,7 @@ class SQSRegionConfig(RegionConfig): """ SQS configuration for a single AWS region """ + queues = {} def parse_queue(self, global_params, region, queue_url): """ diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index febdbd4df..edccd8c40 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -27,6 +27,10 @@ class VPCRegionConfig(RegionConfig): """ VPC configuration for a single AWS region """ + customer_gateways = {} + flow_logs = {} + vpn_connections = {} + vpn_gateways = {} def parse_customer_gateway(self, global_params, region, cgw): cgw['id'] = cgw.pop('CustomerGatewayId') @@ -63,7 +67,8 @@ def parse_network_acl(self, global_params, region, network_acl): manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) self.vpcs[vpc_id].network_acls[network_acl['id']] = network_acl - def __parse_network_acl_entries(self, entries, egress): + @staticmethod + def __parse_network_acl_entries(entries, egress): """ :param entries: From e7a3d54fe4f074749aecfe0913136647f6df1ec3 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Sun, 24 Feb 2019 17:25:00 -0500 Subject: [PATCH 074/154] More warning fixes for AWS --- ScoutSuite/providers/aws/services/ec2.py | 21 ++++++++++++-------- ScoutSuite/providers/aws/services/rds.py | 2 ++ ScoutSuite/providers/base/configs/browser.py | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index 36926cb9f..ad0b1959d 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -33,6 +33,9 @@ class EC2RegionConfig(RegionConfig): EC2 configuration for a single AWS region """ elastic_ips = {} + images = {} + snapshots = {} + volumes = {} def parse_elastic_ip(self, global_params, region, eip): """ @@ -70,7 +73,8 @@ def parse_instance(self, global_params, region, reservation): instance['network_interfaces'][eni['NetworkInterfaceId']] = nic self.vpcs[vpc_id].instances[i['InstanceId']] = instance - def _get_user_data(self, region, instance_id): + @staticmethod + def _get_user_data(region, instance_id): user_data_response = api_clients[region].describe_instance_attribute(Attribute='userData', InstanceId=instance_id) @@ -99,9 +103,9 @@ def parse_security_group(self, global_params, region, group): """ Parse a single Redsfhit security group + :param group: :param global_params: Parameters shared for all regions :param region: Name of the AWS region - :param security)_group: Security group """ vpc_id = group['VpcId'] if 'VpcId' in group and group['VpcId'] else ec2_classic manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) @@ -113,10 +117,10 @@ def parse_security_group(self, global_params, region, group): 'count'] = self.__parse_security_group_rules(group['IpPermissionsEgress']) self.vpcs[vpc_id].security_groups[group['GroupId']] = security_group - def __parse_security_group_rules(self, rules): + @staticmethod + def __parse_security_group_rules(rules): """ - :param self: :param rules: :return: """ @@ -174,7 +178,8 @@ def parse_snapshot(self, global_params, region, snapshot): 'CreateVolumePermissions'] snapshot['public'] = self._is_public(snapshot) - def _is_public(self, snapshot): + @staticmethod + def _is_public(snapshot): return any([permission.get('Group') == 'all' for permission in snapshot['createVolumePermission']]) def parse_volume(self, global_params, region, volume): @@ -285,7 +290,7 @@ def check_for_elastic_ip(ec2_info): def link_elastic_ips_callback2(ec2_config, current_config, path, current_path, instance_id, callback_args): if instance_id == callback_args['instance_id']: - if not 'PublicIpAddress' in current_config: + if 'PublicIpAddress' not in current_config: current_config['PublicIpAddress'] = callback_args['elastic_ip'] elif current_config['PublicIpAddress'] != callback_args['elastic_ip']: print_info('Warning: public IP address exists (%s) for an instance associated with an elastic IP (%s)' % ( @@ -302,7 +307,7 @@ def list_instances_in_security_groups(region_info): :return: """ for vpc in region_info['vpcs']: - if not 'instances' in region_info['vpcs'][vpc]: + if 'instances' not in region_info['vpcs'][vpc]: return for instance in region_info['vpcs'][vpc]['instances']: state = region_info['vpcs'][vpc]['instances'][instance]['State']['Name'] @@ -323,5 +328,5 @@ def manage_vpc(vpc_info, vpc_id): """ manage_dictionary(vpc_info, vpc_id, {}) vpc_info[vpc_id]['id'] = vpc_id - if not 'name' in vpc_info[vpc_id]: + if 'name' not in vpc_info[vpc_id]: vpc_info[vpc_id]['name'] = vpc_id diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index 702e05248..36a68e3e0 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -15,6 +15,8 @@ class RDSRegionConfig(RegionConfig): """ RDS configuration for a single AWS region """ + parameter_groups = {} + security_groups = {} def parse_instance(self, global_params, region, dbi): """ diff --git a/ScoutSuite/providers/base/configs/browser.py b/ScoutSuite/providers/base/configs/browser.py index fd18deb99..344077022 100644 --- a/ScoutSuite/providers/base/configs/browser.py +++ b/ScoutSuite/providers/base/configs/browser.py @@ -38,7 +38,7 @@ def get_object_at(object, path, attribute_name=None): """ Get arbitrary object given a dictionary and path (list of keys). - :param dictionary: + :param object: :param path: :param attribute_name: :return: From 1b035d4658cccf6429d7e3a1c3509645d55dc4fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pelletier?= Date: Sun, 24 Feb 2019 17:51:19 -0500 Subject: [PATCH 075/154] Switched to using GCE client passed in params --- ScoutSuite/providers/gcp/services/computeengine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/providers/gcp/services/computeengine.py b/ScoutSuite/providers/gcp/services/computeengine.py index a478107f5..e4e5e2956 100644 --- a/ScoutSuite/providers/gcp/services/computeengine.py +++ b/ScoutSuite/providers/gcp/services/computeengine.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from ScoutSuite.providers.gcp.configs.base import GCPBaseConfig -from ScoutSuite.providers.gcp.utils import gcp_connect_service from googleapiclient.errors import HttpError import json @@ -82,6 +81,8 @@ def __init__(self, thread_config): def parse_instances(self, instance, params): project_id = self._get_project_id(instance) + self.api_client = params['api_client'] + instance_dict = {} instance_dict['id'] = self.get_non_provider_id(instance['name']) instance_dict['project_id'] = project_id @@ -209,8 +210,7 @@ def _is_block_project_ssh_keys_enabled(self, instance): return metadata.get('block-project-ssh-keys') == 'true' def _get_common_instance_metadata_dict(self, project_id): - computeengine_client = gcp_connect_service(service='computeengine') - request = computeengine_client.projects().get(project=project_id) + request = self.api_client.projects().get(project=project_id) project = request.execute() return self._metadata_to_dict(project['commonInstanceMetadata']) From 255e176560f7fd15194323d91fac73e2d4959205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pelletier?= Date: Sun, 24 Feb 2019 17:53:58 -0500 Subject: [PATCH 076/154] Cleaned up project fetching in GCE --- ScoutSuite/providers/gcp/services/computeengine.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ScoutSuite/providers/gcp/services/computeengine.py b/ScoutSuite/providers/gcp/services/computeengine.py index e4e5e2956..45d90d1d2 100644 --- a/ScoutSuite/providers/gcp/services/computeengine.py +++ b/ScoutSuite/providers/gcp/services/computeengine.py @@ -210,8 +210,7 @@ def _is_block_project_ssh_keys_enabled(self, instance): return metadata.get('block-project-ssh-keys') == 'true' def _get_common_instance_metadata_dict(self, project_id): - request = self.api_client.projects().get(project=project_id) - project = request.execute() + project = self.api_client.projects().get(project=project_id).execute() return self._metadata_to_dict(project['commonInstanceMetadata']) def _is_oslogin_enabled(self, instance, project_id): From ea81f0fb07ad27c306e9e676d167c3602cc192e7 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 24 Feb 2019 19:11:05 -0500 Subject: [PATCH 077/154] Added rationale and cis reference number --- .../providers/gcp/rules/findings/cloudstorage-bucket-member.json | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-member.json b/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-member.json index a76f8f02f..a1a3832f9 100644 --- a/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-member.json +++ b/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-member.json @@ -2,6 +2,7 @@ "arg_names": [ "Member", "Description"], "key": "cloudstorage-bucket-_ARG_0_", "description": "_ARG_1_", + "rationale": "Allowing anonymous and/or public access grants permissions to anyone to access bucket content. Such access might not be desired if you are storing any sensitive data. Hence, ensure that anonymous and/or public access to a bucket is not allowed (CIS 5.1).", "dashboard_name": "Buckets", "display_path": "cloudstorage.buckets.id", "path": "cloudstorage.buckets.id.acl_configuration", From 934f5f678e3131416ec371d7952111a5d16a4b5b Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 24 Feb 2019 19:27:16 -0500 Subject: [PATCH 078/154] Added rationale and CIS number. --- .../gcp/rules/findings/cloudstorage-bucket-no-logging.json | 1 + 1 file changed, 1 insertion(+) diff --git a/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-no-logging.json b/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-no-logging.json index ca728cf99..e53941e96 100644 --- a/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-no-logging.json +++ b/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-no-logging.json @@ -1,6 +1,7 @@ { "dashboard_name": "Buckets", "description": "Bucket without logging", + "rationale": "By enabling access and storage logs on target Storage buckets, it is possible to capture all events which may affect objects within target buckets (CIS 5.3).", "path": "cloudstorage.buckets.id", "conditions": [ "and", [ "cloudstorage.buckets.id.logging_enabled", "false", "" ] From c5c721aeb0d21f22032801850b5459b935606b5b Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Sun, 24 Feb 2019 19:31:32 -0500 Subject: [PATCH 079/154] Improved description --- .../gcp/rules/findings/cloudstorage-bucket-no-logging.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-no-logging.json b/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-no-logging.json index e53941e96..1a6b442ec 100644 --- a/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-no-logging.json +++ b/ScoutSuite/providers/gcp/rules/findings/cloudstorage-bucket-no-logging.json @@ -1,6 +1,6 @@ { "dashboard_name": "Buckets", - "description": "Bucket without logging", + "description": "Buckets should have logging enabled", "rationale": "By enabling access and storage logs on target Storage buckets, it is possible to capture all events which may affect objects within target buckets (CIS 5.3).", "path": "cloudstorage.buckets.id", "conditions": [ "and", From f1ccbfe5707f070957c14f10d91db08ac0adaf23 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 14:15:46 -0500 Subject: [PATCH 080/154] [Opinel] Fixed tests --- ScoutSuite/core/conditions.py | 6 +++++- tests/test-rules-ruleset.py | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ScoutSuite/core/conditions.py b/ScoutSuite/core/conditions.py index e8ace2658..e1da14c29 100644 --- a/ScoutSuite/core/conditions.py +++ b/ScoutSuite/core/conditions.py @@ -10,7 +10,6 @@ from ScoutSuite.core.console import print_error, print_exception from ScoutSuite.core import condition_operators -from ScoutSuite.providers.base.configs.browser import get_value_at re_get_value_at = re.compile(r'_GET_VALUE_AT_\((.*?)\)') re_nested_get_value_at = re.compile(r'_GET_VALUE_AT_\(.*') @@ -27,6 +26,9 @@ def pass_conditions(all_info, current_path, conditions, unknown_as_pass_conditio :return: """ + # Fixes circular dependency + from ScoutSuite.providers.base.configs.browser import get_value_at + if len(conditions) == 0: return True condition_operator = conditions.pop(0) @@ -263,6 +265,8 @@ def pass_condition(b, test, a): def fix_path_string(all_info, current_path, path_to_value): + # Fixes circulare dependency + from ScoutSuite.providers.base.configs.browser import get_value_at # handle nested _GET_VALUE_AT_... while True: dynamic_path = re_get_value_at.findall(path_to_value) diff --git a/tests/test-rules-ruleset.py b/tests/test-rules-ruleset.py index 1d89c404f..7f3c4be16 100644 --- a/tests/test-rules-ruleset.py +++ b/tests/test-rules-ruleset.py @@ -18,7 +18,7 @@ def setup(self): self.test_ruleset_001 = os.path.join(self.test_dir, 'data/test-ruleset.json') self.test_ruleset_002 = os.path.join(self.test_dir, 'data/test-ruleset-absolute-path.json') - @patch("ScoutSuite.core.ruleset.printError") + @patch("ScoutSuite.core.ruleset.print_error") def test_ruleset_class(self, printError): test001 = Ruleset(filename=self.test_ruleset_001) assert (os.path.isdir(test001.rules_data_path)) @@ -48,14 +48,14 @@ def test_ruleset_class(self, printError): test005 = Ruleset(filename=self.test_ruleset_001, ruleset_generator=True) - @patch("ScoutSuite.core.ruleset.printError") + @patch("ScoutSuite.core.ruleset.print_error") def test_ruleset_file_not_exist(self, printError): test003 = Ruleset(cloud_provider='aws', filename='tests/data/no-such-file.json') assert (test003.rules == []) assert (printError.call_count == 1) assert ("no-such-file.json does not exist" in printError.call_args_list[0][0][0]) - @patch("ScoutSuite.core.ruleset.printError") + @patch("ScoutSuite.core.ruleset.print_error") def test_ruleset_invalid(self, printError): test004 = Ruleset(cloud_provider='aws', filename='tests/data/invalid-file.json') assert (test004.rules == []) @@ -88,7 +88,7 @@ def test_path_for_ruletypes(self): target = Ruleset(filename='filters') assert (os.path.samefile(target.filename, rpath + 'rulesets/filters.json')) - @patch("ScoutSuite.core.ruleset.prompt_4_yes_no") + @patch("ScoutSuite.core.ruleset.prompt_yes_no") def test_file_search(self, prompt_yes_no): prompt_yes_no.return_value = False From 912add81dc600366069409857d2f90ccc74510b9 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 14:30:20 -0500 Subject: [PATCH 081/154] [Opinel] Code cleanup based on suggestions --- ScoutSuite/providers/aws/configs/regions.py | 3 +-- ScoutSuite/providers/aws/services/cloudwatch.py | 3 ++- ScoutSuite/utils.py | 9 ++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index 26dced7e9..c9b5fe50a 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -75,8 +75,7 @@ def __init__(self, service_metadata=None, thread_config=4): if 'api_call' not in resource_metadata: continue params = resource_metadata['params'] if 'params' in resource_metadata else {} - ignore_exceptions = True if 'no_exceptions' in resource_metadata and \ - resource_metadata['no_exceptions'] else False + ignore_exceptions = resource_metadata.get('no_exceptions', False) if not only_first_region: self.targets['other_regions'] += ((resource, resource_metadata['response'], diff --git a/ScoutSuite/providers/aws/services/cloudwatch.py b/ScoutSuite/providers/aws/services/cloudwatch.py index c1cf21c98..af3b528d8 100644 --- a/ScoutSuite/providers/aws/services/cloudwatch.py +++ b/ScoutSuite/providers/aws/services/cloudwatch.py @@ -28,7 +28,8 @@ def parse_alarm(self, global_params, region, alarm): alarm['name'] = alarm.pop('AlarmName') # Drop some data for k in ['AlarmConfigurationUpdatedTimestamp', 'StateReason', 'StateReasonData', 'StateUpdatedTimestamp']: - alarm.pop(k) if k in alarm else None + if k in alarm: + alarm.pop(k) alarm_id = self.get_non_provider_id(alarm['arn']) self.alarms[alarm_id] = alarm diff --git a/ScoutSuite/utils.py b/ScoutSuite/utils.py index 18b2b9f47..11e3b0738 100644 --- a/ScoutSuite/utils.py +++ b/ScoutSuite/utils.py @@ -87,11 +87,10 @@ def is_throttled(e): :param e: Exception raised :return: True if it's a throttling exception else False """ - return True if (hasattr(e, 'response') - and 'Error' in e.response - and e.response['Error']['Code'] in ['Throttling', - 'RequestLimitExceeded', - 'ThrottlingException']) else False + return (hasattr(e, 'response') and 'Error' in e.response + and e.response['Error']['Code'] in ['Throttling', + 'RequestLimitExceeded', + 'ThrottlingException']) def manage_dictionary(dictionary, key, init, callback=None): From 38cc6117be4d649eaa041392d733031231a654e2 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 14:38:48 -0500 Subject: [PATCH 082/154] [Opinel] Removed useless global --- ScoutSuite/providers/aws/services/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/services/s3.py b/ScoutSuite/providers/aws/services/s3.py index 5a3f8c4b1..c44e207a3 100644 --- a/ScoutSuite/providers/aws/services/s3.py +++ b/ScoutSuite/providers/aws/services/s3.py @@ -133,7 +133,7 @@ def update_iam_permissions(s3_info, bucket_name, iam_entity, allowed_iam_entity, def update_bucket_permissions(s3_info, iam_info, action, iam_entity, allowed_iam_entity, full_path, policy_type, policy_name): - global policy + policy = {} allowed_buckets = [] # By default, all buckets are allowed for bucket_name in s3_info['buckets']: From d4fcb003f19962a947c5d49bb758415dfa1577c5 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 14:48:37 -0500 Subject: [PATCH 083/154] [Opinel] Reordered functions in conditions.py --- ScoutSuite/core/conditions.py | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ScoutSuite/core/conditions.py b/ScoutSuite/core/conditions.py index e1da14c29..08d14f15b 100644 --- a/ScoutSuite/core/conditions.py +++ b/ScoutSuite/core/conditions.py @@ -60,25 +60,6 @@ def pass_conditions(all_info, current_path, conditions, unknown_as_pass_conditio return not condition_operator == 'or' -def __prepare_age_test(a, b): - if type(a) != list: - print_error('Error: olderThan requires a list such as [ N , \'days\' ] or [ M, \'hours\'].') - raise Exception - number = int(a[0]) - unit = a[1] - if unit not in ['days', 'hours', 'minutes', 'seconds']: - print_error('Error: only days, hours, minutes, and seconds are supported.') - raise Exception - if unit == 'hours': - number *= 3600 - unit = 'seconds' - elif unit == 'minutes': - number *= 60 - unit = 'seconds' - age = getattr((datetime.datetime.today() - dateutil.parser.parse(str(b)).replace(tzinfo=None)), unit) - return age, number - - def pass_condition(b, test, a): """ Generic test function used by Scout2 / AWS recipes @@ -282,3 +263,22 @@ def fix_path_string(all_info, current_path, path_to_value): dv = get_value_at(all_info, current_path, tmp) path_to_value = path_to_value.replace('_GET_VALUE_AT_(%s)' % tmp, dv) return path_to_value + + +def __prepare_age_test(a, b): + if type(a) != list: + print_error('Error: olderThan requires a list such as [ N , \'days\' ] or [ M, \'hours\'].') + raise Exception + number = int(a[0]) + unit = a[1] + if unit not in ['days', 'hours', 'minutes', 'seconds']: + print_error('Error: only days, hours, minutes, and seconds are supported.') + raise Exception + if unit == 'hours': + number *= 3600 + unit = 'seconds' + elif unit == 'minutes': + number *= 60 + unit = 'seconds' + age = getattr((datetime.datetime.today() - dateutil.parser.parse(str(b)).replace(tzinfo=None)), unit) + return age, number From d238a3af5c5305a9bb8f03431d1659181f7a7c98 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 14:54:02 -0500 Subject: [PATCH 084/154] [Opinel] Renamed newLine to new_line for PEP8 compliance --- ScoutSuite/core/console.py | 12 ++++++------ ScoutSuite/providers/aws/services/ec2.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 3ed568574..09852cd47 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -41,8 +41,8 @@ def print_debug(msg): print_generic(sys.stderr, msg) -def print_error(msg, newLine=True): - print_generic(sys.stderr, msg, newLine) +def print_error(msg, new_line=True): + print_generic(sys.stderr, msg, new_line) def print_exception(e, debug_only=False): @@ -53,15 +53,15 @@ def print_exception(e, debug_only=False): print_error(str(e)) -def print_generic(out, msg, newLine=True): +def print_generic(out, msg, new_line=True): out.write(msg) out.flush() - if newLine == True: + if new_line: out.write('\n') -def print_info(msg, newLine=True): - print_generic(sys.stdout, msg, newLine) +def print_info(msg, new_line=True): + print_generic(sys.stdout, msg, new_line) ######################################## diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index ad0b1959d..c9b089231 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -216,7 +216,7 @@ def __init__(self, service_metadata, thread_config=4): def analyze_ec2_config(ec2_info, aws_account_id, force_write): try: - print_info('Analyzing EC2 config... ', newLine=False) + print_info('Analyzing EC2 config... ', new_line=False) # Custom EC2 analysis # check_for_elastic_ip(ec2_info) # FIXME - commented for now as this method doesn't seem to be defined anywhere' From ec9c0e2de16f8ef08188ed1cb204370c6bf5623b Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 15:08:00 -0500 Subject: [PATCH 085/154] [Opinel] Fixed all warnings in console.py --- ScoutSuite/core/console.py | 42 ++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 09852cd47..038e736f3 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -76,7 +76,7 @@ def prompt(test_input=None): :return: Value typed by user (or passed in argument when testing) """ - if test_input != None: + if not test_input: if type(test_input) == list and len(test_input): choice = test_input.pop(0) elif type(test_input) == list: @@ -84,20 +84,16 @@ def prompt(test_input=None): else: choice = test_input else: - # Coverage: 4 missed statements - try: - choice = input() - except: - choice = input() + choice = input() return choice -def prompt_mfa_code(activate=False, input=None): +def prompt_mfa_code(activate=False, test_input=None): """ Prompt for an MFA code :param activate: Set to true when prompting for the 2nd code when activating a new MFA device - :param input: Used for unit testing + :param test_input: Used for unit testing :return: The MFA code """ @@ -106,7 +102,7 @@ def prompt_mfa_code(activate=False, input=None): prompt_string = 'Enter the next value: ' else: prompt_string = 'Enter your MFA code (or \'q\' to abort): ' - mfa_code = prompt_value(prompt_string, no_confirm=True, input=input) + mfa_code = prompt_value(prompt_string, no_confirm=True, test_input=test_input) if mfa_code == 'q': return mfa_code if not re_mfa_code.match(): @@ -115,24 +111,24 @@ def prompt_mfa_code(activate=False, input=None): return mfa_code -def prompt_overwrite(filename, force_write, input=None): +def prompt_overwrite(filename, force_write, test_input=None): """ Prompt whether the file should be overwritten :param filename: Name of the file about to be written :param force_write: Skip confirmation prompt if this flag is set - :param input: Used for unit testing + :param test_input: Used for unit testing :return: Boolean whether file write operation is allowed """ if not os.path.exists(filename) or force_write: return True - return prompt_yes_no('File \'{}\' already exists. Do you want to overwrite it'.format(filename), input=input) + return prompt_yes_no('File \'{}\' already exists. Do you want to overwrite it'.format(filename), test_input=test_input) def prompt_value(question, choices=None, default=None, display_choices=True, display_indices=False, authorize_list=False, is_question=False, no_confirm=False, required=True, regex=None, - regex_format='', max_laps=5, input=None, return_index=False): + regex_format='', max_laps=5, test_input=None, return_index=False): """ Prompt for a value . . @@ -148,16 +144,18 @@ def prompt_value(question, choices=None, default=None, display_choices=True, dis :param regex: TODO :param regex_format TODO :param max_laps: Exit after N laps - :param input: Used for unit testing + :param test_input: Used for unit testing :return: """ + int_choice = 0 + if choices and display_choices and not display_indices: question = question + ' (' + '/'.join(choices) + ')' lap_n = 0 while True: if lap_n >= max_laps: - print_error('Automatically abording prompt loop after 5 failures') + print_error('Automatically aborting prompt loop after 5 failures') return None lap_n += 1 can_return = False @@ -169,7 +167,7 @@ def prompt_value(question, choices=None, default=None, display_choices=True, dis for c in choices: print_error('%3d. %s' % (choices.index(c), c)) print_error('Enter the number corresponding to your choice: ', False) - choice = prompt(input) + choice = prompt(test_input) # Set the default value if empty choice if not choice or choice == '': if default: @@ -193,7 +191,7 @@ def prompt_value(question, choices=None, default=None, display_choices=True, dis choice = choices[int(choice)] else: for c in user_choices: - if not c in choices: + if c not in choices: print_error('Invalid value (%s).' % c) choice_valid = False break @@ -210,24 +208,24 @@ def prompt_value(question, choices=None, default=None, display_choices=True, dis # No automated validation, can attempt to return can_return = True if can_return: - # Manually onfirm that the entered value is correct if needed - if no_confirm or prompt_yes_no('You entered "' + choice + '". Is that correct', input=input): + # Manually confirm that the entered value is correct if needed + if no_confirm or prompt_yes_no('You entered "' + choice + '". Is that correct', test_input=test_input): return int(int_choice) if return_index else choice -def prompt_yes_no(question, input=None): +def prompt_yes_no(question, test_input=None): """ Prompt for a yes/no or y/n answer . :param question: Question to be asked - :param input: Used for unit testing + :param test_input: Used for unit testing :return: True for yes/y, False for no/n """ count = 0 while True: print_error(question + ' (y/n)? ') - choice = prompt(input).lower() + choice = prompt(test_input).lower() if choice == 'yes' or choice == 'y': return True elif choice == 'no' or choice == 'n': From c6c92980d5fc2957bd09a3bcdf93a5127958f699 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 15:08:29 -0500 Subject: [PATCH 086/154] [Opinel] Removed unused parameter --- ScoutSuite/core/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/core/exceptions.py b/ScoutSuite/core/exceptions.py index b821623a0..d2b477068 100644 --- a/ScoutSuite/core/exceptions.py +++ b/ScoutSuite/core/exceptions.py @@ -11,7 +11,7 @@ class RuleExceptions(object): - def __init__(self, profile, file_path=None, foobar=None): + def __init__(self, profile, file_path=None): self.profile = profile self.file_path = file_path self.jsrw = JavaScriptReaderWriter(self.profile) From c16ad61b5962ac8f798d40bc43698d07dd8cf6f2 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 15:09:31 -0500 Subject: [PATCH 087/154] [Opinel] Fixed all warnings in fs.py --- ScoutSuite/core/fs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ScoutSuite/core/fs.py b/ScoutSuite/core/fs.py index 4899e87fa..a55b50ee9 100644 --- a/ScoutSuite/core/fs.py +++ b/ScoutSuite/core/fs.py @@ -46,7 +46,7 @@ def load_data(data_file, key_name=None, local_file=False): return data -def read_ip_ranges(filename, local_file=True, ip_only=False, conditions=[]): +def read_ip_ranges(filename, local_file=True, ip_only=False, conditions=None): """ Returns the list of IP prefixes from an ip-ranges file @@ -56,6 +56,9 @@ def read_ip_ranges(filename, local_file=True, ip_only=False, conditions=[]): :param ip_only: :return: """ + if not conditions: + conditions = [] + targets = [] data = load_data(filename, local_file=local_file) if 'source' in data: @@ -105,7 +108,6 @@ def save_blob_as_json(filename, blob, force_write): :param filename: :param blob: :param force_write: - :param debug: :return: """ From 749efa9e8989f02d9e4ecbc034d73d82c305ad0c Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 15:13:37 -0500 Subject: [PATCH 088/154] [Opinel] Removed opinel threading as it is dead code --- ScoutSuite/core/fs.py | 2 +- ScoutSuite/core/threads.py | 49 -------------------------------------- 2 files changed, 1 insertion(+), 50 deletions(-) delete mode 100644 ScoutSuite/core/threads.py diff --git a/ScoutSuite/core/fs.py b/ScoutSuite/core/fs.py index a55b50ee9..fe2fed76c 100644 --- a/ScoutSuite/core/fs.py +++ b/ScoutSuite/core/fs.py @@ -58,7 +58,7 @@ def read_ip_ranges(filename, local_file=True, ip_only=False, conditions=None): """ if not conditions: conditions = [] - + targets = [] data = load_data(filename, local_file=local_file) if 'source' in data: diff --git a/ScoutSuite/core/threads.py b/ScoutSuite/core/threads.py deleted file mode 100644 index 3826ae406..000000000 --- a/ScoutSuite/core/threads.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -from threading import Thread -from six.moves.queue import Queue - -from ScoutSuite.core.console import print_exception - - -def thread_work(targets, function, params={}, num_threads=0): - """ - Generic multithreading helper - - :param targets: - :param function: - :param params: - :param num_threads: - - :return: - """ - q = Queue(maxsize=0) - if not num_threads: - num_threads = len(targets) - for i in range(num_threads): - worker = Thread(target=function, args=(q, params)) - worker.setDaemon(True) - worker.start() - for target in targets: - q.put(target) - q.join() - - -def threaded_per_region(q, params): - """ - Helper for multithreading on a per-region basis - - :param q: - :param params: - - :return: - """ - while True: - try: - params['region'] = q.get() - method = params['method'] - method(params) - except Exception as e: - print_exception(e) - finally: - q.task_done() From 083d9e5740cb6c9d462f0c9a0f19899d64d97fe3 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 15:23:31 -0500 Subject: [PATCH 089/154] [Opinel] Factored some aws utils into aws --- ScoutSuite/providers/aws/configs/regions.py | 3 +- ScoutSuite/providers/aws/provider.py | 3 +- ScoutSuite/providers/aws/services/ec2.py | 3 +- .../providers/aws/services/elasticache.py | 3 +- ScoutSuite/providers/aws/services/elb.py | 3 +- ScoutSuite/providers/aws/services/elbv2.py | 3 +- ScoutSuite/providers/aws/services/iam.py | 1 + ScoutSuite/providers/aws/services/rds.py | 3 +- ScoutSuite/providers/aws/services/redshift.py | 3 +- ScoutSuite/providers/aws/services/vpc.py | 3 +- ScoutSuite/providers/aws/utils.py | 74 +++++++++++++ ScoutSuite/utils.py | 100 ++---------------- tests/test-utils.py | 1 + 13 files changed, 104 insertions(+), 99 deletions(-) create mode 100644 ScoutSuite/providers/aws/utils.py diff --git a/ScoutSuite/providers/aws/configs/regions.py b/ScoutSuite/providers/aws/configs/regions.py index c9b5fe50a..5024e8af9 100644 --- a/ScoutSuite/providers/aws/configs/regions.py +++ b/ScoutSuite/providers/aws/configs/regions.py @@ -20,7 +20,8 @@ from ScoutSuite.providers.base.configs import resource_id_map from ScoutSuite.providers.base.configs.threads import thread_configs from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import format_service_name, is_throttled, manage_dictionary +from ScoutSuite.utils import format_service_name, manage_dictionary +from ScoutSuite.providers.aws.utils import is_throttled from ScoutSuite.providers.aws.configs.base import BaseConfig from ScoutSuite.output.console import FetchStatusLogger diff --git a/ScoutSuite/providers/aws/provider.py b/ScoutSuite/providers/aws/provider.py index d0e7c2b14..9394b85e9 100644 --- a/ScoutSuite/providers/aws/provider.py +++ b/ScoutSuite/providers/aws/provider.py @@ -10,7 +10,8 @@ from ScoutSuite.providers.aws.services.vpc import put_cidr_name from ScoutSuite.providers.base.configs.browser import combine_paths, get_object_at, get_value_at from ScoutSuite.providers.base.provider import BaseProvider -from ScoutSuite.utils import ec2_classic, manage_dictionary +from ScoutSuite.utils import manage_dictionary +from ScoutSuite.providers.aws.utils import ec2_classic # noinspection PyBroadException diff --git a/ScoutSuite/providers/aws/services/ec2.py b/ScoutSuite/providers/aws/services/ec2.py index c9b089231..a4970881b 100644 --- a/ScoutSuite/providers/aws/services/ec2.py +++ b/ScoutSuite/providers/aws/services/ec2.py @@ -13,7 +13,8 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig from ScoutSuite.providers.base.configs.browser import get_attribute_at -from ScoutSuite.utils import get_keys, ec2_classic, manage_dictionary +from ScoutSuite.utils import manage_dictionary +from ScoutSuite.providers.aws.utils import ec2_classic, get_keys from ScoutSuite.core.fs import load_data ######################################## diff --git a/ScoutSuite/providers/aws/services/elasticache.py b/ScoutSuite/providers/aws/services/elasticache.py index c025285b7..092da241c 100644 --- a/ScoutSuite/providers/aws/services/elasticache.py +++ b/ScoutSuite/providers/aws/services/elasticache.py @@ -2,7 +2,8 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic, manage_dictionary +from ScoutSuite.utils import manage_dictionary +from ScoutSuite.providers.aws.utils import ec2_classic ######################################## diff --git a/ScoutSuite/providers/aws/services/elb.py b/ScoutSuite/providers/aws/services/elb.py index 811e51405..8edb73c62 100644 --- a/ScoutSuite/providers/aws/services/elb.py +++ b/ScoutSuite/providers/aws/services/elb.py @@ -5,7 +5,8 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic, get_keys, manage_dictionary +from ScoutSuite.utils import manage_dictionary +from ScoutSuite.providers.aws.utils import ec2_classic, get_keys ######################################## diff --git a/ScoutSuite/providers/aws/services/elbv2.py b/ScoutSuite/providers/aws/services/elbv2.py index 04079d0cf..b8f93986b 100644 --- a/ScoutSuite/providers/aws/services/elbv2.py +++ b/ScoutSuite/providers/aws/services/elbv2.py @@ -7,7 +7,8 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic, manage_dictionary +from ScoutSuite.utils import manage_dictionary +from ScoutSuite.providers.aws.utils import ec2_classic ######################################## diff --git a/ScoutSuite/providers/aws/services/iam.py b/ScoutSuite/providers/aws/services/iam.py index 238937b26..48ee8d7af 100644 --- a/ScoutSuite/providers/aws/services/iam.py +++ b/ScoutSuite/providers/aws/services/iam.py @@ -5,6 +5,7 @@ from ScoutSuite.core.console import print_error, print_exception from ScoutSuite.providers.aws.aws import connect_service, handle_truncated_response from ScoutSuite.providers.aws.configs.base import AWSBaseConfig +from ScoutSuite.providers.aws.utils import get_keys, is_throttled from ScoutSuite.utils import * diff --git a/ScoutSuite/providers/aws/services/rds.py b/ScoutSuite/providers/aws/services/rds.py index 36a68e3e0..d563b1cf0 100644 --- a/ScoutSuite/providers/aws/services/rds.py +++ b/ScoutSuite/providers/aws/services/rds.py @@ -4,7 +4,8 @@ from ScoutSuite.providers.aws.aws import handle_truncated_response from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic, manage_dictionary +from ScoutSuite.utils import manage_dictionary +from ScoutSuite.providers.aws.utils import ec2_classic ######################################## diff --git a/ScoutSuite/providers/aws/services/redshift.py b/ScoutSuite/providers/aws/services/redshift.py index aac05e8f5..78e59482f 100644 --- a/ScoutSuite/providers/aws/services/redshift.py +++ b/ScoutSuite/providers/aws/services/redshift.py @@ -7,7 +7,8 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig, api_clients from ScoutSuite.providers.aws.configs.vpc import VPCConfig -from ScoutSuite.utils import ec2_classic, manage_dictionary +from ScoutSuite.utils import manage_dictionary +from ScoutSuite.providers.aws.utils import ec2_classic ######################################## diff --git a/ScoutSuite/providers/aws/services/vpc.py b/ScoutSuite/providers/aws/services/vpc.py index edccd8c40..a5f2c1c58 100644 --- a/ScoutSuite/providers/aws/services/vpc.py +++ b/ScoutSuite/providers/aws/services/vpc.py @@ -8,7 +8,8 @@ from ScoutSuite.providers.aws.configs.regions import RegionalServiceConfig, RegionConfig from ScoutSuite.providers.aws.configs.vpc import VPCConfig as SingleVPCConfig from ScoutSuite.providers.base.configs.browser import get_value_at -from ScoutSuite.utils import ec2_classic, get_keys, manage_dictionary +from ScoutSuite.utils import manage_dictionary +from ScoutSuite.providers.aws.utils import ec2_classic, get_keys from ScoutSuite.core.fs import load_data, read_ip_ranges ######################################## diff --git a/ScoutSuite/providers/aws/utils.py b/ScoutSuite/providers/aws/utils.py new file mode 100644 index 000000000..7a91d4df8 --- /dev/null +++ b/ScoutSuite/providers/aws/utils.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +import re + +ec2_classic = 'EC2-Classic' +formatted_service_name = { + 'cloudformation': 'CloudFormation', + 'cloudtrail': 'CloudTrail', + 'cloudwatch': 'CloudWatch', + 'config': 'Config', + 'directconnect': 'Direct Connect', + 'dynamodb': 'DynamoDB', + 'elasticache': 'ElastiCache', + 'lambda': 'Lambda', + 'redshift': 'RedShift', + 'route53': 'Route53', + 'route53domains': 'Route53Domains', + + 'cloudstorage': 'Cloud Storage', + 'cloudsql': 'Cloud SQL', + 'stackdriverlogging': 'Stackdriver Logging', + 'stackdrivermonitoring': 'Stackdriver Monitoring', + 'computeengine': 'Compute Engine', + 'kubernetesengine': 'Kubernetes Engine', + 'cloudresourcemanager': 'Cloud Resource Manager', + + 'monitor': 'Monitor', + 'storageaccounts': 'Storage Accounts', + 'sqldatabase': 'SQL Database', + 'securitycenter': 'Security Center', + 'keyvault': 'Key Vault', + 'appgateway': 'Application Gateway', + 'rediscache': 'Redis Cache', + 'network': 'Network', + 'appservice': 'App Service', + 'loadbalancer': 'Load Balancer' +} + + +def get_keys(src, dst, keys): + """ + Copies the value of keys from source object to dest object + + :param src: + :param dst: + :param keys: + :return: + """ + for key in keys: + dst[key] = src[key] if key in src else None + + +def no_camel(name): + """ + Converts CamelCase to camel_case + + :param name: + :return: + """ + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + + +def is_throttled(e): + """ + Determines whether the exception is due to API throttling. + + :param e: Exception raised + :return: True if it's a throttling exception else False + """ + return (hasattr(e, 'response') and 'Error' in e.response + and e.response['Error']['Code'] in ['Throttling', + 'RequestLimitExceeded', + 'ThrottlingException']) diff --git a/ScoutSuite/utils.py b/ScoutSuite/utils.py index 11e3b0738..0e0b9772f 100644 --- a/ScoutSuite/utils.py +++ b/ScoutSuite/utils.py @@ -1,96 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import print_function -import re - -######################################## -# Globals -######################################## - -ec2_classic = 'EC2-Classic' - -formatted_service_name = { - 'cloudformation': 'CloudFormation', - 'cloudtrail': 'CloudTrail', - 'cloudwatch': 'CloudWatch', - 'config': 'Config', - 'directconnect': 'Direct Connect', - 'dynamodb': 'DynamoDB', - 'elasticache': 'ElastiCache', - 'lambda': 'Lambda', - 'redshift': 'RedShift', - 'route53': 'Route53', - 'route53domains': 'Route53Domains', - - 'cloudstorage': 'Cloud Storage', - 'cloudsql': 'Cloud SQL', - 'stackdriverlogging': 'Stackdriver Logging', - 'stackdrivermonitoring': 'Stackdriver Monitoring', - 'computeengine': 'Compute Engine', - 'kubernetesengine': 'Kubernetes Engine', - 'cloudresourcemanager': 'Cloud Resource Manager', - - 'monitor': 'Monitor', - 'storageaccounts': 'Storage Accounts', - 'sqldatabase': 'SQL Database', - 'securitycenter': 'Security Center', - 'keyvault': 'Key Vault', - 'appgateway': 'Application Gateway', - 'rediscache': 'Redis Cache', - 'network': 'Network', - 'appservice': 'App Service', - 'loadbalancer': 'Load Balancer' -} - - -######################################## -# Functions -######################################## - -def format_service_name(service): - """ - - :param service: - :return: - """ - return formatted_service_name[service] if service in formatted_service_name else service.upper() - - -def get_keys(src, dst, keys): - """ - Copies the value of keys from source object to dest object - - :param src: - :param dst: - :param keys: - :return: - """ - for key in keys: - dst[key] = src[key] if key in src else None - - -def no_camel(name): - """ - Converts CamelCase to camel_case - - :param name: - :return: - """ - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) - return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() - - -def is_throttled(e): - """ - Determines whether the exception is due to API throttling. - - :param e: Exception raised - :return: True if it's a throttling exception else False - """ - return (hasattr(e, 'response') and 'Error' in e.response - and e.response['Error']['Code'] in ['Throttling', - 'RequestLimitExceeded', - 'ThrottlingException']) +from ScoutSuite.providers.aws.utils import formatted_service_name def manage_dictionary(dictionary, key, init, callback=None): @@ -109,3 +20,12 @@ def manage_dictionary(dictionary, key, init, callback=None): if callback: callback(dictionary[key]) return dictionary + + +def format_service_name(service): + """ + + :param service: + :return: + """ + return formatted_service_name[service] if service in formatted_service_name else service.upper() diff --git a/tests/test-utils.py b/tests/test-utils.py index 05fe11c46..b970e46c0 100644 --- a/tests/test-utils.py +++ b/tests/test-utils.py @@ -1,4 +1,5 @@ # Import AWS utils +from ScoutSuite.providers.aws.utils import get_keys, no_camel from ScoutSuite.utils import * From 3754e36a7bc8020fe8e0ee08162637f7eb849b52 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 15:30:38 -0500 Subject: [PATCH 090/154] [Opinel] Fixed another cyclic dependency --- ScoutSuite/providers/aws/utils.py | 32 ------------------------------ ScoutSuite/utils.py | 33 ++++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/ScoutSuite/providers/aws/utils.py b/ScoutSuite/providers/aws/utils.py index 7a91d4df8..fd4873a68 100644 --- a/ScoutSuite/providers/aws/utils.py +++ b/ScoutSuite/providers/aws/utils.py @@ -3,38 +3,6 @@ import re ec2_classic = 'EC2-Classic' -formatted_service_name = { - 'cloudformation': 'CloudFormation', - 'cloudtrail': 'CloudTrail', - 'cloudwatch': 'CloudWatch', - 'config': 'Config', - 'directconnect': 'Direct Connect', - 'dynamodb': 'DynamoDB', - 'elasticache': 'ElastiCache', - 'lambda': 'Lambda', - 'redshift': 'RedShift', - 'route53': 'Route53', - 'route53domains': 'Route53Domains', - - 'cloudstorage': 'Cloud Storage', - 'cloudsql': 'Cloud SQL', - 'stackdriverlogging': 'Stackdriver Logging', - 'stackdrivermonitoring': 'Stackdriver Monitoring', - 'computeengine': 'Compute Engine', - 'kubernetesengine': 'Kubernetes Engine', - 'cloudresourcemanager': 'Cloud Resource Manager', - - 'monitor': 'Monitor', - 'storageaccounts': 'Storage Accounts', - 'sqldatabase': 'SQL Database', - 'securitycenter': 'Security Center', - 'keyvault': 'Key Vault', - 'appgateway': 'Application Gateway', - 'rediscache': 'Redis Cache', - 'network': 'Network', - 'appservice': 'App Service', - 'loadbalancer': 'Load Balancer' -} def get_keys(src, dst, keys): diff --git a/ScoutSuite/utils.py b/ScoutSuite/utils.py index 0e0b9772f..08e74dcff 100644 --- a/ScoutSuite/utils.py +++ b/ScoutSuite/utils.py @@ -1,7 +1,38 @@ # -*- coding: utf-8 -*- from __future__ import print_function -from ScoutSuite.providers.aws.utils import formatted_service_name +formatted_service_name = { + 'cloudformation': 'CloudFormation', + 'cloudtrail': 'CloudTrail', + 'cloudwatch': 'CloudWatch', + 'config': 'Config', + 'directconnect': 'Direct Connect', + 'dynamodb': 'DynamoDB', + 'elasticache': 'ElastiCache', + 'lambda': 'Lambda', + 'redshift': 'RedShift', + 'route53': 'Route53', + 'route53domains': 'Route53Domains', + + 'cloudstorage': 'Cloud Storage', + 'cloudsql': 'Cloud SQL', + 'stackdriverlogging': 'Stackdriver Logging', + 'stackdrivermonitoring': 'Stackdriver Monitoring', + 'computeengine': 'Compute Engine', + 'kubernetesengine': 'Kubernetes Engine', + 'cloudresourcemanager': 'Cloud Resource Manager', + + 'monitor': 'Monitor', + 'storageaccounts': 'Storage Accounts', + 'sqldatabase': 'SQL Database', + 'securitycenter': 'Security Center', + 'keyvault': 'Key Vault', + 'appgateway': 'Application Gateway', + 'rediscache': 'Redis Cache', + 'network': 'Network', + 'appservice': 'App Service', + 'loadbalancer': 'Load Balancer' +} def manage_dictionary(dictionary, key, init, callback=None): From 73193e5bf22284ddc83bffdd282398196be249a7 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 16:16:49 -0500 Subject: [PATCH 091/154] [Opinel] Removed dead code --- ScoutSuite/core/fs.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/ScoutSuite/core/fs.py b/ScoutSuite/core/fs.py index fe2fed76c..bfba11aaa 100644 --- a/ScoutSuite/core/fs.py +++ b/ScoutSuite/core/fs.py @@ -88,19 +88,6 @@ def read_ip_ranges(filename, local_file=True, ip_only=False, conditions=None): return targets -def read_file(file_path): - """ - Read the contents of a file - - :param file_path: Path of the file to be read - - :return: Contents of the file - """ - with open(file_path, 'rt') as f: - contents = f.read() - return contents - - def save_blob_as_json(filename, blob, force_write): """ Creates/Modifies file and saves python object as JSON From dc34596257db6825406f6299abb17ee1a21802c4 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 16:18:54 -0500 Subject: [PATCH 092/154] [Opinel] Removed dead code --- ScoutSuite/providers/aws/aws.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ScoutSuite/providers/aws/aws.py b/ScoutSuite/providers/aws/aws.py index 684a7c365..69b70f684 100644 --- a/ScoutSuite/providers/aws/aws.py +++ b/ScoutSuite/providers/aws/aws.py @@ -90,11 +90,6 @@ def get_caller_identity(credentials): return api_client.get_caller_identity() -def get_username(credentials): - caller_identity = get_caller_identity(credentials) - return caller_identity['Arn'].split('/')[-1] - - def get_aws_account_id(credentials): caller_identity = get_caller_identity(credentials) return caller_identity['Arn'].split(':')[4] From e6520b607d6ea45e57d88d3b2efb545e2d9f9a33 Mon Sep 17 00:00:00 2001 From: Philippe Dugre Date: Mon, 25 Feb 2019 16:22:08 -0500 Subject: [PATCH 093/154] [Opinel] Removed awsdead code --- ScoutSuite/providers/aws/credentials.py | 29 ------------------------- 1 file changed, 29 deletions(-) diff --git a/ScoutSuite/providers/aws/credentials.py b/ScoutSuite/providers/aws/credentials.py index b6fd7197d..0bd2eb8ce 100644 --- a/ScoutSuite/providers/aws/credentials.py +++ b/ScoutSuite/providers/aws/credentials.py @@ -127,21 +127,6 @@ def get_profiles_from_aws_credentials_file(credentials_files=None): return sorted(profiles) -def generate_password(): - """ - Generate a password using random characters from uppercase, lowercase, digits, and symbols - - :return: The random password - """ - chars = string.ascii_letters + string.digits + '!@#$%^&*()_+-=[]{};:,<.>?|' - modulus = len(chars) - pchars = os.urandom(16) - if type(pchars) == str: - return ''.join(chars[i % modulus] for i in map(ord, pchars)) - else: - return ''.join(chars[i % modulus] for i in pchars) - - def init_creds(): """ Create a dictionary with all the necessary keys set to "None" @@ -360,20 +345,6 @@ def read_profile_from_aws_config_file(profile_name, config_file=aws_config_file) return role_arn, source_profile, mfa_serial, external_id -def show_profiles_from_aws_credentials_file(credentials_files=None): - """ - Show profile names from ~/.aws/credentials - - :param credentials_files: - :return: - """ - if credentials_files is None: - credentials_files = [aws_credentials_file, aws_config_file] - profiles = get_profiles_from_aws_credentials_file(credentials_files) - for profile in set(profiles): - print_info(' * %s' % profile) - - def write_creds_to_aws_credentials_file(profile_name, credentials, credentials_file=aws_credentials_file): """ Write credentials to AWS config file From d0a4e70d8d6961c5ef7011fad9ed1ac82e29f2f2 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Feb 2019 06:54:24 -0500 Subject: [PATCH 094/154] Update ScoutSuite/core/console.py Co-Authored-By: zer0x64 <17575242+zer0x64@users.noreply.github.com> --- ScoutSuite/core/console.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 038e736f3..3df245934 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -218,7 +218,7 @@ def prompt_yes_no(question, test_input=None): Prompt for a yes/no or y/n answer . :param question: Question to be asked - :param test_input: Used for unit testing + :param test_input: Used for unit testing :return: True for yes/y, False for no/n """ From e7c1b8ffd9726fda44cc244ffcee8934d4ca5d67 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Feb 2019 09:14:03 -0500 Subject: [PATCH 095/154] Update ScoutSuite/core/rule_definition.py Co-Authored-By: vifor2 --- ScoutSuite/core/rule_definition.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/core/rule_definition.py b/ScoutSuite/core/rule_definition.py index 53fcf2776..ef2714f46 100644 --- a/ScoutSuite/core/rule_definition.py +++ b/ScoutSuite/core/rule_definition.py @@ -79,7 +79,6 @@ def load(self): self.string_definition = f.read() self.load_from_string_definition() except Exception as e: - # printException(e) print_error('Failed to load rule defined in %s: %s' % (self.file_name, e)) From 8050f95d124d972698ec8575791c96197988d836 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Feb 2019 09:14:13 -0500 Subject: [PATCH 096/154] Update ScoutSuite/core/rule.py Co-Authored-By: vifor2 --- ScoutSuite/core/rule.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/core/rule.py b/ScoutSuite/core/rule.py index f5eecd603..a545f049b 100644 --- a/ScoutSuite/core/rule.py +++ b/ScoutSuite/core/rule.py @@ -138,5 +138,4 @@ def set_definition(self, rule_definitions, attributes = None, ip_ranges = None, if self.key_suffix: setattr(self, 'key', '%s-%s' % (self.key, self.key_suffix)) except Exception as e: - # printException(e) print_error('Failed to set definition %s: %s' % (self.filename, e)) From 0c5caff61a52af2180e7827f0dae5e035d5150ae Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Feb 2019 09:15:58 -0500 Subject: [PATCH 097/154] Update ScoutSuite/core/ruleset.py Co-Authored-By: vifor2 --- ScoutSuite/core/ruleset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/core/ruleset.py b/ScoutSuite/core/ruleset.py index f0487fa2c..0663cc188 100644 --- a/ScoutSuite/core/ruleset.py +++ b/ScoutSuite/core/ruleset.py @@ -84,7 +84,6 @@ def load(self, rule_type, quiet=False): for rule in ruleset['rules'][filename]: self.handle_rule_versions(filename, rule_type, rule) except Exception as e: - # printException(e) print_error('Error: ruleset file %s contains malformed JSON.' % self.filename) self.rules = [] self.about = '' From ba873395ebf6010c4f7b68cf0c9eaa68a5d1c7b2 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Feb 2019 09:16:45 -0500 Subject: [PATCH 098/154] Update ScoutSuite/providers/aws/aws.py Co-Authored-By: vifor2 --- ScoutSuite/providers/aws/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/aws.py b/ScoutSuite/providers/aws/aws.py index 69b70f684..e21480765 100644 --- a/ScoutSuite/providers/aws/aws.py +++ b/ScoutSuite/providers/aws/aws.py @@ -142,6 +142,6 @@ def is_throttled(e): :param e: Exception raised :return: True if it's a throttling exception else False """ - return True if (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in + return (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in ['Throttling', 'RequestLimitExceeded', 'ThrottlingException', 'TooManyRequestsException']) \ else False From ae3be6bcf5c39aa0165a9dce7c4f659c7dad45a7 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Feb 2019 09:21:09 -0500 Subject: [PATCH 099/154] Update ScoutSuite/providers/aws/aws.py Co-Authored-By: vifor2 --- ScoutSuite/providers/aws/aws.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ScoutSuite/providers/aws/aws.py b/ScoutSuite/providers/aws/aws.py index e21480765..6202521e4 100644 --- a/ScoutSuite/providers/aws/aws.py +++ b/ScoutSuite/providers/aws/aws.py @@ -144,4 +144,3 @@ def is_throttled(e): """ return (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in ['Throttling', 'RequestLimitExceeded', 'ThrottlingException', 'TooManyRequestsException']) \ - else False From 155d7ca43e7374115625124825904794eed676d3 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Feb 2019 09:22:15 -0500 Subject: [PATCH 100/154] Update ScoutSuite/providers/aws/aws.py Co-Authored-By: vifor2 --- ScoutSuite/providers/aws/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/providers/aws/aws.py b/ScoutSuite/providers/aws/aws.py index 6202521e4..88ee0efa8 100644 --- a/ScoutSuite/providers/aws/aws.py +++ b/ScoutSuite/providers/aws/aws.py @@ -143,4 +143,4 @@ def is_throttled(e): :return: True if it's a throttling exception else False """ return (hasattr(e, 'response') and 'Error' in e.response and e.response['Error']['Code'] in - ['Throttling', 'RequestLimitExceeded', 'ThrottlingException', 'TooManyRequestsException']) \ + ['Throttling', 'RequestLimitExceeded', 'ThrottlingException', 'TooManyRequestsException']) From c17442e70045350f929119f9b14cdfd47784adcb Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 26 Feb 2019 09:33:27 -0500 Subject: [PATCH 101/154] Fixed type in consoly.py --- ScoutSuite/core/console.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScoutSuite/core/console.py b/ScoutSuite/core/console.py index 3df245934..f51af0d19 100644 --- a/ScoutSuite/core/console.py +++ b/ScoutSuite/core/console.py @@ -76,7 +76,7 @@ def prompt(test_input=None): :return: Value typed by user (or passed in argument when testing) """ - if not test_input: + if test_input is not None: if type(test_input) == list and len(test_input): choice = test_input.pop(0) elif type(test_input) == list: From 30501150704beb44cbe265291242be6e375cc45e Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 26 Feb 2019 09:36:19 -0500 Subject: [PATCH 102/154] Removed dead code --- ScoutSuite/providers/aws/services/route53.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ScoutSuite/providers/aws/services/route53.py b/ScoutSuite/providers/aws/services/route53.py index 4f4cd6680..2cdbb1894 100644 --- a/ScoutSuite/providers/aws/services/route53.py +++ b/ScoutSuite/providers/aws/services/route53.py @@ -63,7 +63,4 @@ def parse_hosted_zones(self, hosted_zone, params): record_sets = handle_truncated_response(api_client.list_resource_record_sets, {'HostedZoneId': hosted_zone_id}, ['ResourceRecordSets']) hosted_zone.update(record_sets) - # print(str(record_sets)) - # record_sets = api_client.list_resource_record_sets() - # hosted_zone['RecordSets'] = record_sets['Resourc'] self.hosted_zones[hosted_zone_id] = hosted_zone From 76ba33fb298332f82e484e3c69cbfe41404cacd5 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 26 Feb 2019 09:41:27 -0500 Subject: [PATCH 103/154] Applied Antoine's last suggestion --- ScoutSuite/utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ScoutSuite/utils.py b/ScoutSuite/utils.py index 08e74dcff..fc0198ae2 100644 --- a/ScoutSuite/utils.py +++ b/ScoutSuite/utils.py @@ -37,19 +37,20 @@ def manage_dictionary(dictionary, key, init, callback=None): """ - :param dictionary: :param key: :param init: :param callback: - :return: """ - if not str(key) in dictionary: - dictionary[str(key)] = init - manage_dictionary(dictionary, key, init) - if callback: - callback(dictionary[key]) + + if str(key) in dictionary: + return dictionary + + dictionary[str(key)] = init + manage_dictionary(dictionary, key, init) + if callback: + callback(dictionary[key]) return dictionary From 9cd3951acfd17dd31dd9c7a6e340868f381f136e Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 26 Feb 2019 11:45:19 -0500 Subject: [PATCH 104/154] Added cloudformation_stack_drifted related info --- .../partials/aws/services.cloudformation.regions.id.stacks.html | 1 + ScoutSuite/providers/aws/services/cloudformation.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html b/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html index 7fcc95b32..04c820620 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html +++ b/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html @@ -20,6 +20,7 @@

    Information

    {{/if}}
    Termination protection enabled: {{termination_protection}}
    +
    Configuration has drifted: {{drifted}}

    Capabilities {{> count_badge count=Capabilities.length}}

    diff --git a/ScoutSuite/providers/aws/services/cloudformation.py b/ScoutSuite/providers/aws/services/cloudformation.py index 300ba5fd1..0cc37ccea 100644 --- a/ScoutSuite/providers/aws/services/cloudformation.py +++ b/ScoutSuite/providers/aws/services/cloudformation.py @@ -28,6 +28,7 @@ def parse_stack(self, global_params, region, stack): stack_description = api_clients[region].describe_stacks(StackName=stack['name']) stack['termination_protection'] = stack_description['Stacks'][0]['EnableTerminationProtection'] + stack['drifted'] = stack.pop('DriftInformation')['StackDriftStatus'] == 'DRIFTED' stack_policy = api_clients[region].get_stack_policy(StackName=stack['name']) if 'StackPolicyBody' in stack_policy: From eef65cce23fe0fe1ead58e8f62c925f7556593a7 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 26 Feb 2019 13:49:47 -0500 Subject: [PATCH 105/154] Can now tell if a stack has a deletion policy --- ScoutSuite/providers/aws/services/cloudformation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ScoutSuite/providers/aws/services/cloudformation.py b/ScoutSuite/providers/aws/services/cloudformation.py index 0cc37ccea..7ad6b3251 100644 --- a/ScoutSuite/providers/aws/services/cloudformation.py +++ b/ScoutSuite/providers/aws/services/cloudformation.py @@ -30,6 +30,12 @@ def parse_stack(self, global_params, region, stack): stack['termination_protection'] = stack_description['Stacks'][0]['EnableTerminationProtection'] stack['drifted'] = stack.pop('DriftInformation')['StackDriftStatus'] == 'DRIFTED' + template = api_clients[region].get_template(StackName=stack['name'])['TemplateBody']['Resources'] + for group in template.keys(): + stack['has_deletion_policy'] = 'DeletionPolicy' in template[group] + if stack['has_deletion_policy'] is True: + break + stack_policy = api_clients[region].get_stack_policy(StackName=stack['name']) if 'StackPolicyBody' in stack_policy: stack['policy'] = json.loads(stack_policy['StackPolicyBody']) From 8d778d1f46ada0ee6dd52a96e1d12dc2b0606162 Mon Sep 17 00:00:00 2001 From: Vincent Fortin Date: Tue, 26 Feb 2019 14:26:41 -0500 Subject: [PATCH 106/154] Now saving stack deletion policy type and it is viewable by the user --- .../aws/services.cloudformation.regions.id.stacks.html | 1 + .../aws/rules/findings/cloudformation-stack-with-role.json | 2 +- ScoutSuite/providers/aws/services/cloudformation.py | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html b/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html index 04c820620..ee4b33ff2 100644 --- a/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html +++ b/ScoutSuite/output/data/html/partials/aws/services.cloudformation.regions.id.stacks.html @@ -21,6 +21,7 @@

    Information

    Termination protection enabled: {{termination_protection}}
    Configuration has drifted: {{drifted}}
    +
    Deletion policy: {{deletion_policy}}

    Capabilities {{> count_badge count=Capabilities.length}}

    diff --git a/ScoutSuite/providers/aws/rules/findings/cloudformation-stack-with-role.json b/ScoutSuite/providers/aws/rules/findings/cloudformation-stack-with-role.json index 8f0546d17..1ff41e395 100644 --- a/ScoutSuite/providers/aws/rules/findings/cloudformation-stack-with-role.json +++ b/ScoutSuite/providers/aws/rules/findings/cloudformation-stack-with-role.json @@ -1,5 +1,5 @@ { - "description": "Role passed to stack", + "description": "Stack should not have a role", "rationale": "Passing a role to CloudFormation stacks may result in privilege escalation because IAM users with privileges within the CloudFormation scope implicitly inherit the stack's role's permissions.", "path": "cloudformation.regions.id.stacks.id", "dashboard_name": "Stacks", diff --git a/ScoutSuite/providers/aws/services/cloudformation.py b/ScoutSuite/providers/aws/services/cloudformation.py index 7ad6b3251..bef5b20f5 100644 --- a/ScoutSuite/providers/aws/services/cloudformation.py +++ b/ScoutSuite/providers/aws/services/cloudformation.py @@ -31,9 +31,10 @@ def parse_stack(self, global_params, region, stack): stack['drifted'] = stack.pop('DriftInformation')['StackDriftStatus'] == 'DRIFTED' template = api_clients[region].get_template(StackName=stack['name'])['TemplateBody']['Resources'] + stack['deletion_policy'] = 'Delete' for group in template.keys(): - stack['has_deletion_policy'] = 'DeletionPolicy' in template[group] - if stack['has_deletion_policy'] is True: + if 'DeletionPolicy' in template[group]: + stack['deletion_policy'] = template[group]['DeletionPolicy'] break stack_policy = api_clients[region].get_stack_policy(StackName=stack['name']) From 7ee43c354cbf01bc83dcce82769a234723fbc571 Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Feb 2019 16:37:52 -0500 Subject: [PATCH 107/154] Added navbar styling for active items --- .../data/inc-scoutsuite/css/scoutsuite-dark.css | 16 +++++++++++++++- .../data/inc-scoutsuite/css/scoutsuite-light.css | 13 +++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite-dark.css b/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite-dark.css index 6d7f30fab..3801d990e 100644 --- a/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite-dark.css +++ b/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite-dark.css @@ -4,4 +4,18 @@ .dropdown-item.disabled, .dropdown-item:disabled { color: #647273; -} \ No newline at end of file +} + +a.nav-link { + color: rgb(235, 235, 235) ; +} + +.dropdown-submenu a.nav-link { + color: rgb(235, 235, 235) ; +} + +.bg-primary .navbar-nav .active>.nav-link, .navbar-brand .active { + color: #25e8c2 !important; +} + + diff --git a/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite-light.css b/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite-light.css index e69de29bb..89a637d20 100644 --- a/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite-light.css +++ b/ScoutSuite/output/data/inc-scoutsuite/css/scoutsuite-light.css @@ -0,0 +1,13 @@ +a.nav-link { + color: rgb(235, 235, 235) ; +} + +.dropdown-submenu a.nav-link { + color: #7b8a8b; +} + +.bg-primary .navbar-nav .active>.nav-link, .navbar-brand .active { + color: #25e8c2 !important; +} + + From 020ddc4308ab1feadfff0bf19269c86e616cbf1c Mon Sep 17 00:00:00 2001 From: Antoine Boisier-Michaud Date: Tue, 26 Feb 2019 16:38:08 -0500 Subject: [PATCH 108/154] Integrated updateNavbar method --- .../output/data/html/partials/metadata.html | 26 ++++++++++--------- .../output/data/inc-scoutsuite/scoutsuite.js | 21 ++++++++------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/ScoutSuite/output/data/html/partials/metadata.html b/ScoutSuite/output/data/html/partials/metadata.html index 00018c31c..aa077a9ec 100644 --- a/ScoutSuite/output/data/html/partials/metadata.html +++ b/ScoutSuite/output/data/html/partials/metadata.html @@ -2,12 +2,12 @@