Merge branch 'refactoring/resource-configs' into refactoring/aws/efs

Scout.py

@@ -4,10 +4,14 @@
 import sys
 import asyncio
 
+from concurrent.futures import ThreadPoolExecutor
+
 from ScoutSuite.__main__ import main
 
 if __name__ == "__main__":
     loop = asyncio.get_event_loop()
+    # TODO: make max_workers parameterizable (through the thread_config cli argument)
+    loop.set_default_executor(ThreadPoolExecutor(max_workers=10))
     loop.run_until_complete(main())
     loop.close()
     sys.exit()

ScoutSuite/providers/aws/facade/awslambda.py

@@ -2,5 +2,5 @@
 
 
 class LambdaFacade:
-    def get_functions(self, region):
-        return AWSFacadeUtils.get_all_pages('lambda', region, 'list_functions', 'Functions')
+    async def get_functions(self, region):
+        return await AWSFacadeUtils.get_all_pages('lambda', region, 'list_functions', 'Functions')

ScoutSuite/providers/aws/facade/cloudtrail.py

@@ -1,13 +1,20 @@
 from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils
+from ScoutSuite.providers.utils import run_concurrently
 
 
 class CloudTrailFacade:
-    def get_trails(self, region):
+    async def get_trails(self, region):
         client = AWSFacadeUtils.get_client('cloudtrail', region)
-        trails = client.describe_trails()['trailList']
+        trails = await run_concurrently(
+            lambda: client.describe_trails()['trailList']
+        )
 
         for trail in trails:
-            trail.update(client.get_trail_status(Name=trail['TrailARN']))
-            trail['EventSelectors'] = client.get_event_selectors(TrailName=trail['TrailARN'])['EventSelectors']
+            trail.update(await run_concurrently(
+                lambda: client.get_trail_status(Name=trail['TrailARN'])
+            ))
+            trail['EventSelectors'] = await run_concurrently(
+                lambda: client.get_event_selectors(TrailName=trail['TrailARN'])['EventSelectors']
+            )
 
-        return trails
+        return trails

ScoutSuite/providers/aws/facade/ec2.py

@@ -1,23 +1,25 @@
 import boto3
 import base64
-import itertools
 
 from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils
+from ScoutSuite.providers.utils import run_concurrently
 
 
 class EC2Facade:
-    def get_instance_user_data(self, region: str, instance_id: str):
+    async def get_instance_user_data(self, region: str, instance_id: str):
         ec2_client = AWSFacadeUtils.get_client('ec2', region)
-        user_data_response = ec2_client.describe_instance_attribute(Attribute='userData', InstanceId=instance_id)
+        user_data_response = await run_concurrently(
+            lambda: ec2_client.describe_instance_attribute(Attribute='userData', InstanceId=instance_id))
 
         if 'Value' not in user_data_response['UserData'].keys():
             return None
 
         return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8')
 
-    def get_instances(self, region, vpc):
+    async def get_instances(self, region, vpc):
         filters = [{'Name': 'vpc-id', 'Values': [vpc]}]
-        reservations = AWSFacadeUtils.get_all_pages('ec2', region, 'describe_instances', 'Reservations', Filters=filters)
+        reservations =\
+            await AWSFacadeUtils.get_all_pages('ec2', region, 'describe_instances', 'Reservations', Filters=filters)
 
         instances = []
         for reservation in reservations:
@@ -27,36 +29,39 @@ def get_instances(self, region, vpc):
 
         return instances
 
-    def get_security_groups(self, region, vpc):
+    async def get_security_groups(self, region, vpc):
         filters = [{'Name': 'vpc-id', 'Values': [vpc]}]
-        return AWSFacadeUtils.get_all_pages('ec2', region, 'describe_security_groups', 'SecurityGroups', Filters=filters)
+        return await AWSFacadeUtils.get_all_pages(
+            'ec2', region, 'describe_security_groups', 'SecurityGroups', Filters=filters)
 
-    def get_vpcs(self, region):
-        ec2_client = boto3.client('ec2', region_name=region)
-        return ec2_client.describe_vpcs()['Vpcs']
+    async def get_vpcs(self, region):
+        ec2_client = await run_concurrently(lambda: boto3.client('ec2', region_name=region))
+        return await run_concurrently(lambda: ec2_client.describe_vpcs()['Vpcs'])
 
-    def get_images(self, region, owner_id):
+    async def get_images(self, region, owner_id):
         filters = [{'Name': 'owner-id', 'Values': [owner_id]}]
-        response = AWSFacadeUtils.get_client('ec2', region) \
-                                 .describe_images(Filters=filters)
+        client = AWSFacadeUtils.get_client('ec2', region)
+        response = await run_concurrently(lambda: client.describe_images(Filters=filters))
 
         return response['Images']
 
-    def get_network_interfaces(self, region, vpc):
+    async def get_network_interfaces(self, region, vpc):
         filters = [{'Name': 'vpc-id', 'Values': [vpc]}]
-        return AWSFacadeUtils.get_all_pages('ec2', region, 'describe_network_interfaces', 'NetworkInterfaces', Filters=filters)
+        return await AWSFacadeUtils.get_all_pages(
+            'ec2', region, 'describe_network_interfaces', 'NetworkInterfaces', Filters=filters)
 
-    def get_volumes(self, region):
-        return AWSFacadeUtils.get_all_pages('ec2', region, 'describe_volumes', 'Volumes')
+    async def get_volumes(self, region):
+        return await AWSFacadeUtils.get_all_pages('ec2', region, 'describe_volumes', 'Volumes')
 
-    def get_snapshots(self, region, owner_id):
+    async def get_snapshots(self, region, owner_id):
         filters = [{'Name': 'owner-id', 'Values': [owner_id]}]
-        snapshots = AWSFacadeUtils.get_all_pages('ec2', region, 'describe_snapshots', 'Snapshots', Filters=filters)
+        snapshots = await AWSFacadeUtils.get_all_pages(
+            'ec2', region, 'describe_snapshots', 'Snapshots', Filters=filters)
 
         ec2_client = AWSFacadeUtils.get_client('ec2', region)
         for snapshot in snapshots:
-            snapshot['CreateVolumePermissions'] = ec2_client.describe_snapshot_attribute(
+            snapshot['CreateVolumePermissions'] = await run_concurrently(lambda: ec2_client.describe_snapshot_attribute(
                 Attribute='createVolumePermission',
-                SnapshotId=snapshot['SnapshotId'])['CreateVolumePermissions']
+                SnapshotId=snapshot['SnapshotId'])['CreateVolumePermissions'])
 
         return snapshots

ScoutSuite/providers/aws/facade/facade.py

@@ -4,24 +4,25 @@
 from ScoutSuite.providers.aws.facade.cloudtrail import CloudTrailFacade
 from ScoutSuite.providers.aws.facade.ec2 import EC2Facade
 from ScoutSuite.providers.aws.facade.efs import EFSFacade
+from ScoutSuite.providers.utils import run_concurrently
 
 
 class AWSFacade(object):
+
     def __init__(self):
         self.ec2 = EC2Facade()
         self.awslambda = LambdaFacade()
         self.cloudtrail = CloudTrailFacade()
         self.efs = EFSFacade()
 
-
     async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'):
         service = 'ec2containerservice' if service == 'ecs' else service
-        available_services = Session().get_available_services()
+        available_services = await run_concurrently(lambda: Session().get_available_services())
 
         if service not in available_services:
             raise Exception('Service ' + service + ' is not available.')
 
-        regions = Session().get_available_regions(service, partition_name)
+        regions = await run_concurrently(lambda: Session().get_available_regions(service, partition_name))
 
         if chosen_regions:
             return list((Counter(regions) & Counter(chosen_regions)).elements())

ScoutSuite/providers/aws/facade/utils.py

@@ -1,26 +1,32 @@
-from typing import Callable
 import boto3
 
+from ScoutSuite.providers.utils import run_concurrently
+
+
 # TODO: Add docstrings
 class AWSFacadeUtils:
     _clients = {}
-    
+
     @staticmethod
-    def get_all_pages(service: str, region: str, paginator_name: str, response_key: str, **paginator_args):
-        pages = AWSFacadeUtils.get_client(service, region) \
-                              .get_paginator(paginator_name) \
-                              .paginate(**paginator_args)
-
-        return AWSFacadeUtils._get_from_all_pages(pages, response_key)
+    async def get_all_pages(service: str, region: str, paginator_name: str, response_key: str, **paginator_args):
+        client = AWSFacadeUtils.get_client(service, region)
+        # Building a paginator doesn't require any API call so no need to do it concurrently:
+        paginator = client.get_paginator(paginator_name).paginate(**paginator_args)
+
+        # Getting all pages from a paginator requires API calls so we need to do it concurrently:
+        return await run_concurrently(lambda: AWSFacadeUtils._get_all_pages_from_paginator(paginator, response_key))
 
     @staticmethod
-    def _get_from_all_pages(pages: [], key:str):
+    def _get_all_pages_from_paginator(paginator, key):
         resources = []
-        for page in pages:
+        # There's an API call hidden behind each iteration:
+        for page in paginator:
             resources.extend(page[key])
 
         return resources
 
     @staticmethod
     def get_client(service: str, region: str):
-        return AWSFacadeUtils._clients.setdefault((service, region), boto3.client(service, region_name=region))
+        # TODO: investigate the use of a mutex to avoid useless creation of a same type of client among threads:
+        client = boto3.client(service, region_name=region)
+        return AWSFacadeUtils._clients.setdefault((service, region), client)

ScoutSuite/providers/aws/resources/awslambda/service.py

@@ -1,18 +1,17 @@
 from ScoutSuite.providers.aws.resources.regions import Regions
 from ScoutSuite.providers.aws.resources.resources import AWSResources
-from ScoutSuite.providers.aws.facade.facade import AWSFacade
 
 
 class RegionalLambdas(AWSResources):
     async def fetch_all(self, **kwargs):
-        raw_functions  = self.facade.awslambda.get_functions(self.scope['region'])
+        raw_functions = await self.facade.awslambda.get_functions(self.scope['region'])
         for raw_function in raw_functions:
             name, resource = self._parse_function(raw_function)
             self[name] = resource
 
     def _parse_function(self, raw_function):
         raw_function['name'] = raw_function.pop('FunctionName')
-        return (raw_function['name'], raw_function)
+        return raw_function['name'], raw_function
 
 
 class Lambdas(Regions):

ScoutSuite/providers/aws/resources/cloudtrail/service.py

@@ -1,13 +1,13 @@
 from ScoutSuite.providers.aws.resources.regions import Regions
 from ScoutSuite.providers.aws.resources.resources import AWSResources
-from ScoutSuite.providers.aws.facade.facade import AWSFacade
 from ScoutSuite.providers.utils import get_non_provider_id
 
 import time
 
+
 class Trails(AWSResources):
     async def fetch_all(self, **kwargs):
-        raw_trails = self.facade.cloudtrail.get_trails(self.scope['region'])
+        raw_trails = await self.facade.cloudtrail.get_trails(self.scope['region'])
         for raw_trail in raw_trails:
             name, resource = self._parse_trail(raw_trail)
             self[name] = resource

ScoutSuite/providers/aws/resources/ec2/ami.py

@@ -1,10 +1,9 @@
 from ScoutSuite.providers.aws.resources.resources import AWSResources
-from ScoutSuite.providers.aws.facade.facade import AWSFacade
 
 
 class AmazonMachineImages(AWSResources):
     async def fetch_all(self, **kwargs):
-        raw_images  = self.facade.ec2.get_images(self.scope['region'], self.scope['owner_id'])
+        raw_images = await self.facade.ec2.get_images(self.scope['region'], self.scope['owner_id'])
         for raw_image in raw_images:
             name, resource = self._parse_image(raw_image)
             self[name] = resource

ScoutSuite/providers/aws/resources/ec2/instances.py

@@ -1,23 +1,22 @@
 from ScoutSuite.providers.aws.resources.resources import AWSResources
-from ScoutSuite.providers.aws.facade.facade import AWSFacade
 from ScoutSuite.providers.aws.aws import get_name
-from ScoutSuite.providers.aws.utils import ec2_classic, get_keys
+from ScoutSuite.providers.aws.utils import get_keys
 
 
 class EC2Instances(AWSResources):
     async def fetch_all(self, **kwargs):
-        raw_instances  = self.facade.ec2.get_instances(self.scope['region'], self.scope['vpc'])
+        raw_instances = await self.facade.ec2.get_instances(self.scope['region'], self.scope['vpc'])
         for raw_instance in raw_instances:
-            name, resource = self._parse_instance(raw_instance)
+            name, resource = await self._parse_instance(raw_instance)
             self[name] = resource
 
-    def _parse_instance(self, raw_instance):
+    async def _parse_instance(self, raw_instance):
         instance = {}
         id = raw_instance['InstanceId']
         instance['id'] = id
         instance['reservation_id'] = raw_instance['ReservationId']
         instance['monitoring_enabled'] = raw_instance['Monitoring']['State'] == 'enabled'
-        instance['user_data'] = self.facade.ec2.get_instance_user_data(self.scope['region'], id)
+        instance['user_data'] = await self.facade.ec2.get_instance_user_data(self.scope['region'], id)
 
         get_name(raw_instance, instance, 'InstanceId')
         get_keys(raw_instance, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId'])

ScoutSuite/providers/aws/resources/ec2/networkinterfaces.py

@@ -1,14 +1,13 @@
 from ScoutSuite.providers.aws.resources.resources import AWSResources
-from ScoutSuite.providers.aws.facade.facade import AWSFacade
 
 
 class NetworkInterfaces(AWSResources):
     async def fetch_all(self, **kwargs):
-        raw_security_groups = self.facade.ec2.get_network_interfaces(self.scope['region'], self.scope['vpc'])
+        raw_security_groups = await self.facade.ec2.get_network_interfaces(self.scope['region'], self.scope['vpc'])
         for raw_security_groups in raw_security_groups:
             name, resource = self._parse_network_interface(raw_security_groups)
             self[name] = resource
 
-    def _parse_network_interface(self, raw_network_interace):
-        raw_network_interace['name'] = raw_network_interace['NetworkInterfaceId']
-        return raw_network_interace['NetworkInterfaceId'], raw_network_interace
+    def _parse_network_interface(self, raw_network_interface):
+        raw_network_interface['name'] = raw_network_interface['NetworkInterfaceId']
+        return raw_network_interface['NetworkInterfaceId'], raw_network_interface

ScoutSuite/providers/aws/resources/ec2/securitygroups.py

@@ -1,7 +1,4 @@
 from ScoutSuite.providers.aws.resources.resources import AWSResources
-from ScoutSuite.providers.aws.facade.facade import AWSFacade
-from ScoutSuite.providers.aws.aws import get_name
-from ScoutSuite.providers.aws.utils import ec2_classic, get_keys
 from ScoutSuite.utils import manage_dictionary
 from ScoutSuite.core.fs import load_data
 
@@ -10,7 +7,7 @@ class SecurityGroups(AWSResources):
     icmp_message_types_dict = load_data('icmp_message_types.json', 'icmp_message_types')
 
     async def fetch_all(self, **kwargs):
-        raw_security_groups = self.facade.ec2.get_security_groups(self.scope['region'], self.scope['vpc'])
+        raw_security_groups = await self.facade.ec2.get_security_groups(self.scope['region'], self.scope['vpc'])
         for raw_security_groups in raw_security_groups:
             name, resource = self._parse_security_group(raw_security_groups)
             self[name] = resource

ScoutSuite/providers/aws/resources/ec2/snapshots.py

@@ -1,18 +1,14 @@
 from ScoutSuite.providers.aws.resources.resources import AWSResources
-from ScoutSuite.providers.aws.facade.facade import AWSFacade
 from ScoutSuite.providers.aws.aws import get_name
 
 
 class Snapshots(AWSResources):
     async def fetch_all(self, **kwargs):
-        raw_snapshots  = self.facade.ec2.get_snapshots(self.scope['region'], self.scope['owner_id'])
+        raw_snapshots = await self.facade.ec2.get_snapshots(self.scope['region'], self.scope['owner_id'])
         for raw_snapshot in raw_snapshots:
             name, resource = self._parse_snapshot(raw_snapshot)
             self[name] = resource
 
-    async def get_resources_from_api(self):
-        return self.facade.ec2.get_snapshots(self.scope['region'], self.scope['owner_id'])
-
     def _parse_snapshot(self, raw_snapshot):
         raw_snapshot['id'] = raw_snapshot.pop('SnapshotId')
         raw_snapshot['name'] = get_name(raw_snapshot, raw_snapshot, 'id')

ScoutSuite/providers/aws/resources/ec2/volumes.py

@@ -1,16 +1,15 @@
 from ScoutSuite.providers.aws.resources.resources import AWSResources
-from ScoutSuite.providers.aws.facade.facade import AWSFacade
 from ScoutSuite.providers.aws.aws import get_name
 
 
 class Volumes(AWSResources):
     async def fetch_all(self, **kwargs):
-        raw_volumes  = self.facade.ec2.get_volumes(self.scope['region'])
+        raw_volumes = await self.facade.ec2.get_volumes(self.scope['region'])
         for raw_volume in raw_volumes:
-            name, resource = self._parse_volumes(raw_volume)
+            name, resource = self._parse_volume(raw_volume)
             self[name] = resource
 
-    def _parse_volumes(self, raw_volume):
+    def _parse_volume(self, raw_volume):
         raw_volume['id'] = raw_volume.pop('VolumeId')
         raw_volume['name'] = get_name(raw_volume, raw_volume, 'id')
         return raw_volume['id'], raw_volume

ScoutSuite/providers/aws/resources/ec2/vpcs.py

@@ -1,5 +1,6 @@
+import asyncio
+
 from ScoutSuite.providers.aws.resources.resources import AWSCompositeResources
-from ScoutSuite.providers.aws.facade.facade import AWSFacade
 from ScoutSuite.providers.aws.resources.ec2.instances import EC2Instances
 from ScoutSuite.providers.aws.resources.ec2.securitygroups import SecurityGroups
 from ScoutSuite.providers.aws.resources.ec2.networkinterfaces import NetworkInterfaces
@@ -13,14 +14,23 @@ class Vpcs(AWSCompositeResources):
     ]
 
     async def fetch_all(self, **kwargs):
-        vpcs = self.facade.ec2.get_vpcs(self.scope['region'])
+        vpcs = await self.facade.ec2.get_vpcs(self.scope['region'])
         for vpc in vpcs:
             name, resource = self._parse_vpc(vpc)
             self[name] = resource
 
-        for vpc in self:
-            scope = {'region': self.scope['region'], 'vpc': vpc}
-            await self._fetch_children(self[vpc], scope=scope)
+        # TODO: make a refactoring of the following:
+        if len(self) == 0:
+            return
+        tasks = {
+            asyncio.ensure_future(
+                self._fetch_children(
+                    self[vpc],
+                    {'region': self.scope['region'], 'vpc': vpc}
+                )
+            ) for vpc in self
+        }
+        await asyncio.wait(tasks)
 
     def _parse_vpc(self, vpc):
         return vpc['VpcId'], {}