Merge pull request #10 from ImmobilienScout24/s3_bucket_whitelisting

fixing TC build by removing dependency to Mock in production code

src/main/python/monocyte/handler/__init__.py

@@ -60,7 +60,7 @@ class Handler(object):
     def __init__(self, region_filter, dry_run=True, logger=None, ignored_resources=None, whitelist=None):
         warnings.filterwarnings('error')
         self.region_filter = region_filter
-        self.regions = [region for region in self.fetch_regions() if self.region_filter(region.name)]
+        self.region_names = [region_name for region_name in self.fetch_region_names() if self.region_filter(region_name)]
         self.dry_run = dry_run
         self.ignored_resources = ignored_resources or []
         self.whitelist = whitelist or {}
@@ -82,7 +82,7 @@ def get_account_id(self):
     def get_whitelist(self):
         return self.whitelist.get(self.get_account_id(), {})
 
-    def fetch_regions(self):
+    def fetch_region_names(self):
         raise NotImplementedError("Should have implemented this")
 
     def fetch_unwanted_resources(self):

src/main/python/monocyte/handler/acm.py

@@ -20,7 +20,7 @@
 MIN_VALID_DAYS = 55
 
 class Certificate(Handler):
-    def fetch_regions(self):
+    def fetch_region_names(self):
         # Since we want to check all regions, regardless of what's allowed or
         # not, we handle multi-region stuff ourselves.
         return []

src/main/python/monocyte/handler/cloudformation.py

@@ -22,12 +22,12 @@ class Stack(Handler):
 
     VALID_TARGET_STATES = ["DELETE_COMPLETE", "DELETE_IN_PROGRESS"]
 
-    def fetch_regions(self):
-        return cloudformation.regions()
+    def fetch_region_names(self):
+        return [region.name for region in cloudformation.regions()]
 
     def fetch_unwanted_resources(self):
-        for region in self.regions:
-            connection = cloudformation.connect_to_region(region.name)
+        for region_name in self.region_names:
+            connection = cloudformation.connect_to_region(region_name)
             unwanted_states = set(connection.valid_states)
             unwanted_states.remove("DELETE_COMPLETE")
             resources = connection.list_stacks(stack_status_filters=list(unwanted_states)) or []
@@ -36,7 +36,7 @@ def fetch_unwanted_resources(self):
                                             resource_type=self.resource_type,
                                             resource_id=resource.stack_id,
                                             creation_date=resource.creation_time,
-                                            region=region.name)
+                                            region=region_name)
                 if resource.stack_name in self.ignored_resources:
                     self.logger.info('IGNORE ' + self.to_string(resource_wrapper))
                     continue

src/main/python/monocyte/handler/dynamodb.py

@@ -19,20 +19,20 @@
 
 
 class Table(Handler):
-    def fetch_regions(self):
-        return dynamodb2.regions()
+    def fetch_region_names(self):
+        return [region.name for region in dynamodb2.regions()]
 
     def fetch_unwanted_resources(self):
-        for region in self.regions:
-            connection = dynamodb2.connect_to_region(region.name)
+        for region_name in self.region_names:
+            connection = dynamodb2.connect_to_region(region_name)
             names = connection.list_tables(limit=100) or {}
             for name in names.get("TableNames"):
                 resource = connection.describe_table(name)
                 resource_wrapper = Resource(resource=resource["Table"],
                                             resource_type=self.resource_type,
                                             resource_id=resource["Table"]["TableName"],
                                             creation_date=resource["Table"]["CreationDateTime"],
-                                            region=region.name)
+                                            region=region_name)
                 if name in self.ignored_resources:
                     self.logger.info('IGNORE ' + self.to_string(resource_wrapper))
                     continue

src/main/python/monocyte/handler/ec2.py

@@ -22,19 +22,19 @@
 class Instance(Handler):
     VALID_TARGET_STATES = ["terminated", "shutting-down"]
 
-    def fetch_regions(self):
-        return ec2.regions()
+    def fetch_region_names(self):
+        return [region.name for region in ec2.regions()]
 
     def fetch_unwanted_resources(self):
-        for region in self.regions:
-            connection = ec2.connect_to_region(region.name)
+        for region_name in self.region_names:
+            connection = ec2.connect_to_region(region_name)
             resources = connection.get_only_instances() or []
             for resource in resources:
                 resource_wrapper = Resource(resource=resource,
                                             resource_type=self.resource_type,
                                             resource_id=resource.id,
                                             creation_date=resource.launch_time,
-                                            region=region.name)
+                                            region=region_name)
                 if resource.id in self.ignored_resources:
                     self.logger.info('IGNORE ' + self.to_string(resource_wrapper))
                     continue
@@ -65,19 +65,19 @@ def delete(self, resource):
 
 class Volume(Handler):
 
-    def fetch_regions(self):
-        return ec2.regions()
+    def fetch_region_names(self):
+        return [region.name for region in ec2.regions()]
 
     def fetch_unwanted_resources(self):
-        for region in self.regions:
-            connection = ec2.connect_to_region(region.name)
+        for region_name in self.region_names:
+            connection = ec2.connect_to_region(region_name)
             resources = connection.get_all_volumes() or []
             for resource in resources:
                 resource_wrapper = Resource(resource=resource,
                                             resource_type=self.resource_type,
                                             resource_id=resource.id,
                                             creation_date=resource.create_time,
-                                            region=region.name)
+                                            region=region_name)
                 if resource.id in self.ignored_resources:
                     self.logger.info('IGNORE ' + self.to_string(resource_wrapper))
                     continue

src/main/python/monocyte/handler/iam.py

@@ -6,8 +6,8 @@
 
 
 class User(Handler):
-    def fetch_regions(self):
-        return iam.regions()
+    def fetch_region_names(self):
+        return [region.name for region in iam.regions()]
 
     def get_users(self):
         iam = boto3.client('iam')
@@ -51,8 +51,8 @@ def delete(self, resource):
 
 
 class Policy(Handler):
-    def fetch_regions(self):
-        return iam.regions()
+    def fetch_region_names(self):
+        return [region.name for region in iam.regions()]
 
     def gather_actions(self, policy_document):
         statement = policy_document['Statement']

src/main/python/monocyte/handler/rds2.py

@@ -29,19 +29,19 @@
 
 class Instance(Handler):
 
-    def fetch_regions(self):
-        return rds2.regions()
+    def fetch_region_names(self):
+        return [region.name for region in rds2.regions()]
 
     def fetch_unwanted_resources(self):
-        for region in self.regions:
-            connection = rds2.connect_to_region(region.name)
+        for region_name in self.region_names:
+            connection = rds2.connect_to_region(region_name)
             resources = connection.describe_db_instances() or []
             for resource in resources["DescribeDBInstancesResponse"]["DescribeDBInstancesResult"]["DBInstances"]:
                 resource_wrapper = Resource(resource=resource,
                                             resource_type=self.resource_type,
                                             resource_id=resource["DBInstanceIdentifier"],
                                             creation_date=resource["InstanceCreateTime"],
-                                            region=region.name)
+                                            region=region_name)
                 if resource['DBInstanceIdentifier'] in self.ignored_resources:
                     self.logger.info('IGNORE ' + self.to_string(resource_wrapper))
                     continue
@@ -63,19 +63,19 @@ def delete(self, resource):
 
 class Snapshot(Handler):
 
-    def fetch_regions(self):
-        return rds2.regions()
+    def fetch_region_names(self):
+        return [region.name for region in rds2.regions()]
 
     def fetch_unwanted_resources(self):
-        for region in self.regions:
-            connection = rds2.connect_to_region(region.name)
+        for region_name in self.region_names:
+            connection = rds2.connect_to_region(region_name)
             resources = connection.describe_db_snapshots() or []
             for resource in resources["DescribeDBSnapshotsResponse"]["DescribeDBSnapshotsResult"]["DBSnapshots"]:
                 resource_wrapper = Resource(resource=resource,
                                             resource_type=self.resource_type,
                                             resource_id=resource["DBSnapshotIdentifier"],
                                             creation_date=resource["SnapshotCreateTime"],
-                                            region=region.name)
+                                            region=region_name)
                 if resource['DBSnapshotIdentifier'] in self.ignored_resources:
                     self.logger.info('IGNORE ' + self.to_string(resource_wrapper))
                     continue

src/main/python/monocyte/handler/s3.py

@@ -28,25 +28,15 @@ def map_location(self, region):
     def get_client(self):
         return boto3.client('s3', region_name='eu-central-1')
 
-    def fetch_regions(self):
+    def fetch_region_names(self):
         session = boto3.session.Session()
-        region_names = session.get_available_regions('s3')
-
-        # FIXME: Update parent class so we can just return the names
-        regions = []
-        from mock import Mock
-        for region_name in region_names:
-            region = Mock()
-            region.name = region_name
-            regions.append(region)
-        return regions
+        return session.get_available_regions('s3')
 
     def fetch_unwanted_resources(self):
         client = self.get_client()
         response = client.list_buckets()
         buckets = [(bucket['Name'], bucket['CreationDate'])
                    for bucket in response['Buckets']]
-        region_names = [region.name for region in self.regions]
 
         for bucket_name, creation_date in buckets:
             try:
@@ -58,7 +48,7 @@ def fetch_unwanted_resources(self):
                                       bucket_name)
                 continue
             region_name = self.map_location(response['LocationConstraint'])
-            if region_name not in region_names or self.is_on_whitelist(bucket_name):
+            if region_name not in self.region_names or self.is_on_whitelist(bucket_name):
                 self.logger.debug("Bucket %s in region %s is OK.",
                                   bucket_name, region_name)
                 continue

src/unittest/python/handler/handler_tests.py

@@ -37,5 +37,5 @@ def test_get_whitelist_returns_empty_whitelist_if_there_is_no_whitelist(self):
 
 
 class TestHandler(Handler):
-    def fetch_regions(self):
-        return {}
+    def fetch_region_names(self):
+        return []

src/unittest/python/monocyte_tests.py

@@ -90,10 +90,8 @@ def fetch_unwanted_resources(self):
         return [Resource(Mock(), "ec2 instance", "123456789", datetime.datetime.now(), "us"),
                 Resource(Mock(), "ec2 volume", "33123456789", datetime.datetime.now(), "us")]
 
-    def fetch_regions(self):
-        mock = Mock(RegionInfo)
-        mock.name = "us"
-        return [mock]
+    def fetch_region_names(self):
+        return "us"
 
     def to_string(self, resource):
         pass