cloud-custodian#40 - initial cut at elb tagmap

c7n/resources/elb.py

@@ -63,6 +63,8 @@
 from c7n.manager import ResourceManager, resources
 from c7n.utils import local_session, chunks, type_schema
 
+from functools import partial
+
 log = logging.getLogger('custodian.elb')
 
 
@@ -81,15 +83,15 @@ def resources(self):
             elbs = self._cache.get(
                 {'region': self.config.region, 'resource': 'elb'})
             if elbs is not None:
-                self.log.debug("Using cached rds: %d" % (
+                self.log.debug("Using cached elb: %d" % (
                     len(elbs)))
                 return self.filter_resources(elbs)
 
         c = self.session_factory().client('elb')
         p = c.get_paginator('describe_load_balancers')
         results = p.paginate()
         elbs = list(itertools.chain(
-            *[rp['LoadBalancerDescriptions'] for rp in results]))
+            *[self.get_elbs_from_result_page(c, rp) for rp in results]))
         self._cache.save(
             {'region': self.config.region, 'resource': 'elbs'}, elbs)
 
@@ -106,6 +108,46 @@ def get_resources(self, resource_ids):
                 return []
             raise
 
+    def get_elbs_from_result_page(self, client, rp):
+        elb_descriptions = rp['LoadBalancerDescriptions']
+        self.add_tags_to_results(client, elb_descriptions)
+        return elb_descriptions
+
+    def add_tags_to_results(self, client, elbs):
+        """
+        Gets the tags for the ELBs and adds them to
+        the result set.
+        """
+        elb_names = [elb['LoadBalancerName'] for elb in elbs]
+        names_to_tags = {}
+        fn = partial(self.process_tags, client=client)
+        futures = []
+        with self.executor_factory(max_workers=3) as w:
+            # max 20 ELBs per call (API limitation)
+            for elb_names_chunk in chunks(elb_names, size=20):
+                    futures.append(
+                        w.submit(fn, elb_names_chunk))
+
+        for f in as_completed(futures):
+            if f.exception():
+                self.log.exception("Exception Processing ELB: %s" % (
+                    f.exception()))
+                continue
+            r = f.result()
+            if r:
+                names_to_tags.update(r)
+
+        for elb in elbs:
+            elb['Tags'] = names_to_tags[elb['LoadBalancerName']]
+
+    def process_tags(self, chunk, **kwargs):
+        tag_descriptions = kwargs['client'].describe_tags(LoadBalancerNames=chunk)
+
+        names_to_tags = {}
+        for desc in tag_descriptions['TagDescriptions']:
+            names_to_tags[desc['LoadBalancerName']] = desc['Tags']
+        return names_to_tags
+
 
 @actions.register('delete')
 class Delete(BaseAction):

c7n/resources/rds.py

@@ -98,7 +98,7 @@ def resources(self):
         p = c.get_paginator('describe_db_instances')
         results = p.paginate(Filters=query)
         dbs = list(itertools.chain(
-            *[self.add_tags(c, rp['DBInstances']) for rp in results]))
+            *[self.get_dbs_from_result_page(c, rp) for rp in results]))
         self._cache.save(
             {'region': self.config.region, 'resource': 'rds', 'q': query}, dbs)
         return self.filter_resources(dbs)
@@ -112,14 +112,20 @@ def get_resources(self, resource_ids):
                     DBInstanceIdentifier=db_id)['DBInstances'])
         return results
 
-    def add_tags(self, client, dbs):
+    def get_dbs_from_result_page(self, client, rp):
+        dbs = rp['DBInstances']
+        self.add_tags_to_results(client, dbs)
+        return dbs
+
+    def add_tags_to_results(self, client, dbs):
         """
-        Adds the tags to the list of databases
+        Gets the tags for each of the databases and
+        adds them to the result set.
         """
         account_id = self.get_account_id()
-        fn = partial(self.process_tags, account_id=account_id ,client=client)
+        fn = partial(self.process_tags, account_id=account_id, client=client)
         with self.executor_factory(max_workers=3) as w:
-            return list(w.map(fn,dbs))
+            list(w.map(fn, dbs))
 
     def process_tags(self, db, **kwargs):
         region = self.get_region(db)
@@ -135,12 +141,12 @@ def process_tags(self, db, **kwargs):
         return db
 
     def get_account_id(self):
-        # TODO: This just gets a security group in the account
-        # and uses that as the ARN of the account. See notes
+        # TODO: This just gets a role from the current account
+        # and uses its ARN as the account ARN. See notes
         # at top of file for how else we can do this
-        client = self.session_factory().client('ec2')
+        client = self.session_factory().client('iam')
         account_id = (
-            client.describe_security_groups()['SecurityGroups'][0]['OwnerId']
+            client.list_roles(MaxItems=1)['Roles'][0]['Arn'].split(":")[4]
         )
         return account_id
 

tests/data/placebo/test_healthcheck_protocol_mismatch/elasticloadbalancing.DescribeTags_1.json

@@ -0,0 +1,63 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RequestId": "00000000-0000-0000-0000-000000000000"
+        }, 
+        "TagDescriptions": [
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-protocol-mismatch"
+            },
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-protocol-matches"
+            },
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-no-listeners"
+            },
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-multiple-listeners"
+            }            
+        ]
+    }
+}

tests/data/placebo/test_ssl_ciphers/elasticloadbalancing.DescribeTags_1.json

@@ -0,0 +1,37 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RequestId": "00000000-0000-0000-0000-000000000000"
+        }, 
+        "TagDescriptions": [
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-valid-policy"
+            },
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-invalid-policy"
+            }              
+        ]
+    }
+}