Rds elb tags (#44)

* #35 - initial cut at rds tagmap

* #40 - initial cut at elb tagmap

c7n/manager.py

@@ -14,6 +14,7 @@
 import logging
 
 from c7n import cache
+from c7n.executor import ThreadPoolExecutor
 from c7n.registry import PluginRegistry
 from c7n.utils import dumps
 
@@ -27,7 +28,9 @@ class ResourceManager(object):
     action_registry = None
 
     supports_dry_run = False
-
+
+    executor_factory = ThreadPoolExecutor
+
     def __init__(self, ctx, data):
         self.ctx = ctx
         self.session_factory = ctx.session_factory

c7n/resources/elb.py

@@ -63,6 +63,8 @@
 from c7n.manager import ResourceManager, resources
 from c7n.utils import local_session, chunks, type_schema
 
+from functools import partial
+
 log = logging.getLogger('custodian.elb')
 
 
@@ -81,15 +83,15 @@ def resources(self):
             elbs = self._cache.get(
                 {'region': self.config.region, 'resource': 'elb'})
             if elbs is not None:
-                self.log.debug("Using cached rds: %d" % (
+                self.log.debug("Using cached elb: %d" % (
                     len(elbs)))
                 return self.filter_resources(elbs)
 
         c = self.session_factory().client('elb')
         p = c.get_paginator('describe_load_balancers')
         results = p.paginate()
         elbs = list(itertools.chain(
-            *[rp['LoadBalancerDescriptions'] for rp in results]))
+            *[self.get_elbs_from_result_page(c, rp) for rp in results]))
         self._cache.save(
             {'region': self.config.region, 'resource': 'elbs'}, elbs)
 
@@ -106,6 +108,46 @@ def get_resources(self, resource_ids):
                 return []
             raise
 
+    def get_elbs_from_result_page(self, client, rp):
+        elb_descriptions = rp['LoadBalancerDescriptions']
+        self.add_tags_to_results(client, elb_descriptions)
+        return elb_descriptions
+
+    def add_tags_to_results(self, client, elbs):
+        """
+        Gets the tags for the ELBs and adds them to
+        the result set.
+        """
+        elb_names = [elb['LoadBalancerName'] for elb in elbs]
+        names_to_tags = {}
+        fn = partial(self.process_tags, client=client)
+        futures = []
+        with self.executor_factory(max_workers=3) as w:
+            # max 20 ELBs per call (API limitation)
+            for elb_names_chunk in chunks(elb_names, size=20):
+                    futures.append(
+                        w.submit(fn, elb_names_chunk))
+
+        for f in as_completed(futures):
+            if f.exception():
+                self.log.exception("Exception Processing ELB: %s" % (
+                    f.exception()))
+                continue
+            r = f.result()
+            if r:
+                names_to_tags.update(r)
+
+        for elb in elbs:
+            elb['Tags'] = names_to_tags[elb['LoadBalancerName']]
+
+    def process_tags(self, chunk, **kwargs):
+        tag_descriptions = kwargs['client'].describe_tags(LoadBalancerNames=chunk)
+
+        names_to_tags = {}
+        for desc in tag_descriptions['TagDescriptions']:
+            names_to_tags[desc['LoadBalancerName']] = desc['Tags']
+        return names_to_tags
+
 
 @actions.register('delete')
 class Delete(BaseAction):

c7n/resources/rds.py

@@ -69,6 +69,8 @@
 from c7n.manager import ResourceManager, resources
 from c7n.utils import local_session, type_schema
 
+from functools import partial
+
 log = logging.getLogger('custodian.rds')
 
 
@@ -96,7 +98,7 @@ def resources(self):
         p = c.get_paginator('describe_db_instances')
         results = p.paginate(Filters=query)
         dbs = list(itertools.chain(
-            *[rp['DBInstances'] for rp in results]))
+            *[self.get_dbs_from_result_page(c, rp) for rp in results]))
         self._cache.save(
             {'region': self.config.region, 'resource': 'rds', 'q': query}, dbs)
         return self.filter_resources(dbs)
@@ -110,6 +112,52 @@ def get_resources(self, resource_ids):
                     DBInstanceIdentifier=db_id)['DBInstances'])
         return results
 
+    def get_dbs_from_result_page(self, client, rp):
+        dbs = rp['DBInstances']
+        self.add_tags_to_results(client, dbs)
+        return dbs
+
+    def add_tags_to_results(self, client, dbs):
+        """
+        Gets the tags for each of the databases and
+        adds them to the result set.
+        """
+        account_id = self.get_account_id()
+        fn = partial(self.process_tags, account_id=account_id, client=client)
+        with self.executor_factory(max_workers=3) as w:
+            list(w.map(fn, dbs))
+
+    def process_tags(self, db, **kwargs):
+        region = self.get_region(db)
+        name = db['DBInstanceIdentifier']
+        arn = "arn:aws:rds:%s:%s:db:%s" % (region, kwargs['account_id'], name)
+        tag_list = (
+            kwargs['client'].list_tags_for_resource(ResourceName=arn)['TagList']
+        )
+        tags = []
+        if tag_list:
+            tags.extend(tag_list)
+        db['Tags'] = tags
+        return db
+
+    def get_account_id(self):
+        # TODO: This just gets a role from the current account
+        # and uses its ARN as the account ARN. See notes
+        # at top of file for how else we can do this
+        client = self.session_factory().client('iam')
+        account_id = (
+            client.list_roles(MaxItems=1)['Roles'][0]['Arn'].split(":")[4]
+        )
+        return account_id
+
+    def get_region(self, db):
+        # get the region from the endpoint
+        endpoint_parts = db['Endpoint']['Address'].split('.')
+
+        # format is "<db-id>.<region>.rds.amazonaws.com"
+        region = endpoint_parts[-4]
+        return region
+
 
 @filters.register('default-vpc')
 class DefaultVpc(Filter):

tests/data/placebo/test_healthcheck_protocol_mismatch/elasticloadbalancing.DescribeTags_1.json

@@ -0,0 +1,63 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RequestId": "00000000-0000-0000-0000-000000000000"
+        }, 
+        "TagDescriptions": [
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-protocol-mismatch"
+            },
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-protocol-matches"
+            },
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-no-listeners"
+            },
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-multiple-listeners"
+            }            
+        ]
+    }
+}

tests/data/placebo/test_ssl_ciphers/elasticloadbalancing.DescribeTags_1.json

@@ -0,0 +1,37 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RequestId": "00000000-0000-0000-0000-000000000000"
+        }, 
+        "TagDescriptions": [
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-valid-policy"
+            },
+            {
+                "Tags": [
+                    {
+                        "Value": "value1", 
+                        "Key": "tag1"
+                    }, 
+                    {
+                        "Value": "value2", 
+                        "Key": "tag2"
+                    }
+                ], 
+                "LoadBalancerName": "test-elb-invalid-policy"
+            }              
+        ]
+    }
+}