Merge pull request #245 from nccgroup/refactoring/aws/elasticache

Refactoring/aws/elasticache

ScoutSuite/providers/aws/configs/services.py

@@ -7,7 +7,7 @@
 from ScoutSuite.providers.aws.resources.directconnect.service import DirectConnect
 from ScoutSuite.providers.aws.resources.ec2.service import EC2
 from ScoutSuite.providers.aws.resources.efs.service import EFS
-from ScoutSuite.providers.aws.services.elasticache import ElastiCacheConfig
+from ScoutSuite.providers.aws.resources.elasticache.service import ElastiCache
 from ScoutSuite.providers.aws.services.elb import ELBConfig
 from ScoutSuite.providers.aws.services.elbv2 import ELBv2Config
 from ScoutSuite.providers.aws.services.emr import EMRConfig
@@ -60,7 +60,7 @@ def __init__(self, metadata=None, thread_config=4, **kwargs):
         self.directconnect = DirectConnect()
         self.ec2 = EC2()
         self.efs = EFS()
-        self.elasticache = ElastiCacheConfig(metadata['database']['elasticache'], thread_config)
+        self.elasticache = ElastiCache()
         self.elb = ELBConfig(metadata['compute']['elb'], thread_config)
         self.elbv2 = ELBv2Config(metadata['compute']['elbv2'], thread_config)
         self.emr = EMRConfig(metadata['analytics']['emr'], thread_config)

ScoutSuite/providers/aws/facade/elasticache.py

@@ -0,0 +1,67 @@
+import boto3
+
+from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils
+from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade
+from ScoutSuite.providers.aws.utils import ec2_classic
+from asyncio import Lock
+
+class ElastiCacheFacade(AWSBaseFacade):
+    regional_clusters_cache_locks = {}
+    regional_subnets_cache_locks = {}
+    clusters_cache = {}
+    subnets_cache = {}
+
+    async def get_clusters(self, region, vpc):
+        await self.cache_clusters(region)
+        return [cluster for cluster in self.clusters_cache[region] if cluster['VpcId'] == vpc]
+
+    async def cache_clusters(self, region):
+        async with self.regional_clusters_cache_locks.setdefault(region, Lock()):
+            if region in self.clusters_cache:
+                return
+
+            self.clusters_cache[region] = await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_clusters', 'CacheClusters')
+
+            for cluster in self.clusters_cache[region]:
+                if 'CacheSubnetGroupName' not in cluster:
+                    cluster['VpcId'] = ec2_classic
+                else:
+                    subnet_group = await self.get_subnet_group(region, cluster['CacheSubnetGroupName'])
+                    cluster['VpcId'] = subnet_group['VpcId']
+
+    async def get_security_groups(self, region):
+        client = AWSFacadeUtils.get_client('elasticache', region, self.session)
+
+        try:
+            return await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_security_groups', 'CacheSecurityGroups')
+
+        except client.exceptions.InvalidParameterValueException:
+            # Recent account are not allowed to use security groups at this level. Calling
+            # describe_cache_security_groups will throw an InvalidParameterValueException exception.
+            pass
+
+        return []
+
+    async def get_subnet_groups(self, region, vpc):
+        await self.cache_subnets(region)
+        return [subnet for subnet in self.subnets_cache[region] if subnet['VpcId'] == vpc]
+
+    async def get_subnet_group(self, region, subnet_name):
+        subnets = await AWSFacadeUtils.get_all_pages('elasticache', \
+                                                  region, \
+                                                  self.session, \
+                                                  'describe_cache_subnet_groups', \
+                                                  'CacheSubnetGroups', \
+                                                  CacheSubnetGroupName=subnet_name \
+                                                )
+        return subnets[0]
+
+    async def cache_subnets(self, region):
+        async with self.regional_subnets_cache_locks.setdefault(region, Lock()):
+            if region in self.subnets_cache:
+                return
+
+            self.subnets_cache[region] = await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_subnet_groups', 'CacheSubnetGroups')
+
+    async def get_parameter_groups(self, region):
+        return await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_parameter_groups', 'CacheParameterGroups')

ScoutSuite/providers/aws/facade/facade.py

@@ -6,15 +6,17 @@
 from ScoutSuite.providers.aws.facade.cloudformation import CloudFormation
 from ScoutSuite.providers.aws.facade.cloudtrail import CloudTrailFacade
 from ScoutSuite.providers.aws.facade.cloudwatch import CloudWatch
+from ScoutSuite.providers.aws.facade.directconnect import DirectConnectFacade
 from ScoutSuite.providers.aws.facade.ec2 import EC2Facade
 from ScoutSuite.providers.aws.facade.efs import EFSFacade
-from ScoutSuite.providers.aws.facade.directconnect import DirectConnectFacade
+from ScoutSuite.providers.aws.facade.elasticache import ElastiCacheFacade
 from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade
 from ScoutSuite.providers.utils import run_concurrently
 from ScoutSuite.core.console import print_error, print_debug
 
+
 class AWSFacade(AWSBaseFacade):
-    def __init__(self, credentials: dict=None):
+    def __init__(self, credentials: dict = None):
         self._set_session(credentials)
 
         self.ec2 = EC2Facade(self.session)
@@ -24,6 +26,7 @@ def __init__(self, credentials: dict=None):
         self.cloudwatch = CloudWatch(self.session)
         self.directconnect = DirectConnectFacade(self.session)
         self.efs = EFSFacade(self.session)
+        self.elasticache = ElastiCacheFacade(self.session)
 
     async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'):
         service = 'ec2containerservice' if service == 'ecs' else service
@@ -50,7 +53,7 @@ def _set_session(self, credentials: dict):
                           'aws_session_token': credentials.get('token')}
 
         self.session = boto3.session.Session(**session_params)
-        
+
         # TODO: This should only be done in the constructor. I put this here for now, because this method is currently
         # called from outside, but it should not happen.
         self.ec2 = EC2Facade(self.session)
@@ -60,3 +63,4 @@ def _set_session(self, credentials: dict):
         self.cloudwatch = CloudWatch(self.session)
         self.directconnect = DirectConnectFacade(self.session)
         self.efs = EFSFacade(self.session)
+        self.elasticache = ElastiCacheFacade(self.session)

ScoutSuite/providers/aws/resources/ec2/vpcs.py

@@ -1,37 +1,14 @@
 import asyncio
 
-from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources
+from ScoutSuite.providers.aws.resources.vpcs import Vpcs
 from ScoutSuite.providers.aws.resources.ec2.instances import EC2Instances
 from ScoutSuite.providers.aws.resources.ec2.securitygroups import SecurityGroups
 from ScoutSuite.providers.aws.resources.ec2.networkinterfaces import NetworkInterfaces
 
 
-class Vpcs(AWSCompositeResources):
+class Ec2Vpcs(Vpcs):
     _children = [
         (EC2Instances, 'instances'),
         (SecurityGroups, 'security_groups'),
         (NetworkInterfaces, 'network_interfaces')
-    ]
-
-    async def fetch_all(self, **kwargs):
-        vpcs = await self.facade.ec2.get_vpcs(self.scope['region'])
-        for vpc in vpcs:
-            name, resource = self._parse_vpc(vpc)
-            self[name] = resource
-
-        # TODO: make a refactoring of the following:
-        if len(self) == 0:
-            return
-        tasks = {
-            asyncio.ensure_future(
-                self._fetch_children(
-                    self[vpc],
-                    {'region': self.scope['region'], 'vpc': vpc}
-                )
-            ) for vpc in self
-        }
-        await asyncio.wait(tasks)
-
-    def _parse_vpc(self, vpc):
-        return vpc['VpcId'], {}
-
+    ]

ScoutSuite/providers/aws/resources/elasticache/cluster.py

@@ -0,0 +1,14 @@
+from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources
+from ScoutSuite.providers.aws.utils import ec2_classic
+
+
+class Clusters(AWSCompositeResources):
+    async def fetch_all(self, **kwargs):
+        raw_clusters = await self.facade.elasticache.get_clusters(self.scope['region'], self.scope['vpc'])
+        for raw_cluster in raw_clusters:
+            name, resource = self._parse_cluster(raw_cluster)
+            self[name] = resource
+
+    def _parse_cluster(self, raw_cluster):
+        raw_cluster['name'] = raw_cluster.pop('CacheClusterId')
+        return raw_cluster['name'], raw_cluster

ScoutSuite/providers/aws/resources/elasticache/parametergroups.py

@@ -0,0 +1,13 @@
+from ScoutSuite.providers.aws.resources.resources import AWSResources
+
+
+class ParameterGroups(AWSResources):
+    async def fetch_all(self, **kwargs):
+        raw_parameter_groups = await self.facade.elasticache.get_parameter_groups(self.scope['region'])
+        for raw_parameter_group in raw_parameter_groups:
+            name, resource = self._parse_parameter_group(raw_parameter_group)
+            self[name] = resource
+
+    def _parse_parameter_group(self, raw_parameter_group):
+        raw_parameter_group['name'] = raw_parameter_group.pop('CacheParameterGroupName')
+        return raw_parameter_group['name'], raw_parameter_group

ScoutSuite/providers/aws/resources/elasticache/securitygroups.py

@@ -0,0 +1,14 @@
+from ScoutSuite.providers.aws.resources.resources import AWSResources
+
+
+class SecurityGroups(AWSResources):
+    async def fetch_all(self, **kwargs):
+        raw_security_groups = await self.facade.elasticache.get_security_groups(self.scope['region'])
+
+        for raw_security_group in raw_security_groups:
+            name, resource = self._parse_security_group(raw_security_group)
+            self[name] = resource
+
+    def _parse_security_group(self, raw_security_group):
+        raw_security_group['name'] = raw_security_group.pop('CacheSecurityGroupName')
+        return raw_security_group['name'], raw_security_group

ScoutSuite/providers/aws/resources/elasticache/service.py

@@ -0,0 +1,29 @@
+from ScoutSuite.providers.aws.facade.facade import AWSFacade
+from ScoutSuite.providers.aws.resources.regions import Regions
+from ScoutSuite.providers.aws.resources.elasticache.vpcs import ElastiCacheVpcs
+from ScoutSuite.providers.aws.resources.elasticache.securitygroups import SecurityGroups
+from ScoutSuite.providers.aws.resources.elasticache.parametergroups import ParameterGroups
+
+
+class ElastiCache(Regions):
+    _children = [
+        (ElastiCacheVpcs, 'vpcs'),
+        (SecurityGroups, 'security_groups'),
+        (ParameterGroups, 'parameter_groups')
+    ]
+
+    def __init__(self):
+        super(ElastiCache, self).__init__('elasticache')
+
+    async def fetch_all(self, credentials=None, regions=None, partition_name='aws'):
+        await super(ElastiCache, self).fetch_all(credentials, regions, partition_name)
+
+        for region in self['regions']:
+            self['regions'][region]['clusters_count'] = sum([len(vpc['clusters']) for vpc in self['regions'][region]['vpcs'].values()])
+            self['regions'][region]['subnet_groups_count'] = sum([len(vpc['subnet_groups']) for vpc in self['regions'][region]['vpcs'].values()])
+
+        self['clusters_count'] = sum([region['clusters_count'] for region in self['regions'].values()])
+
+        # We do not want the parameter groups to be part of the resources count, as it is usually in 
+        # the three of four digits and would make the resources count confusing.
+        self.pop('parameter_groups_count')

ScoutSuite/providers/aws/resources/elasticache/subnetgroups.py

@@ -0,0 +1,14 @@
+from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources
+from ScoutSuite.providers.aws.utils import ec2_classic
+
+
+class SubnetGroups(AWSCompositeResources):
+    async def fetch_all(self, **kwargs):
+        raw_subnet_groups = await self.facade.elasticache.get_subnet_groups(self.scope['region'], self.scope['vpc'])
+        for raw_subnet_group in raw_subnet_groups:
+            name, resource = self._parse_subnet_group(raw_subnet_group)
+            self[name] = resource
+
+    def _parse_subnet_group(self, raw_subnet_group):
+        raw_subnet_group['name'] = raw_subnet_group.pop('CacheSubnetGroupName')
+        return raw_subnet_group['name'], raw_subnet_group

ScoutSuite/providers/aws/resources/elasticache/vpcs.py

@@ -0,0 +1,16 @@
+import asyncio
+
+from ScoutSuite.providers.aws.resources.vpcs import Vpcs
+from ScoutSuite.providers.aws.resources.elasticache.cluster import Clusters
+from ScoutSuite.providers.aws.resources.elasticache.subnetgroups import SubnetGroups
+from ScoutSuite.providers.aws.utils import ec2_classic
+
+
+class ElastiCacheVpcs(Vpcs):
+    _children = [
+        (Clusters, 'clusters'),
+        (SubnetGroups, 'subnet_groups')
+    ]
+
+    def __init__(self, facade, scope: dict):
+        super(ElastiCacheVpcs, self).__init__(facade, scope, add_ec2_classic=True)

ScoutSuite/providers/aws/resources/regions.py

@@ -43,4 +43,9 @@ async def fetch_all(self, credentials, regions=None, partition_name='aws'):
     def _set_counts(self):
         self['regions_count'] = len(self['regions'])
         for _, key in self._children:
+            # VPCs should not be counted as resources. They exist whether you have resources or not, so
+            # counting them would make the report confusing.
+            if key == 'vpcs':
+                continue
+
             self[key + '_count'] = sum([region[key + '_count'] for region in self['regions'].values()])

ScoutSuite/providers/aws/resources/vpcs.py

@@ -0,0 +1,35 @@
+import asyncio
+
+from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources
+
+
+class Vpcs(AWSCompositeResources):
+    """
+    Fetches resources inside the virtual private clouds (VPCs) defined in a region. 
+    :param add_ec2_classic: Setting this parameter to True will add 'EC2-Classic' to the list of VPCs.
+    """
+
+    def __init__(self, facade, scope: dict, add_ec2_classic=False):
+        super(Vpcs, self).__init__(facade, scope)
+        self.add_ec2_classic = add_ec2_classic
+
+    async def fetch_all(self, **kwargs):
+        vpcs = await self.facade.ec2.get_vpcs(self.scope['region'])
+        for vpc in vpcs:
+            name, resource = self._parse_vpc(vpc)
+            self[name] = resource
+
+        # TODO: make a refactoring of the following:
+        if len(self) == 0:
+            return
+
+        tasks = {
+            asyncio.ensure_future(
+                self._fetch_children(self[vpc], {'region': self.scope['region'], 'vpc': vpc})
+            ) for vpc in self
+        }
+
+        await asyncio.wait(tasks)
+
+    def _parse_vpc(self, vpc):
+        return vpc['VpcId'], {}