From aa3994bc60f1e678f378b9118a35d78f7771cff0 Mon Sep 17 00:00:00 2001 From: Jeff Storey Date: Tue, 3 May 2016 11:10:27 -0400 Subject: [PATCH] Rds elb tags (#44) * #35 - initial cut at rds tagmap * #40 - initial cut at elb tagmap --- c7n/manager.py | 5 +- c7n/resources/elb.py | 46 +++++++++++++- c7n/resources/rds.py | 50 ++++++++++++++- .../elasticloadbalancing.DescribeTags_1.json | 63 +++++++++++++++++++ .../elasticloadbalancing.DescribeTags_1.json | 37 +++++++++++ 5 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 tests/data/placebo/test_healthcheck_protocol_mismatch/elasticloadbalancing.DescribeTags_1.json create mode 100644 tests/data/placebo/test_ssl_ciphers/elasticloadbalancing.DescribeTags_1.json diff --git a/c7n/manager.py b/c7n/manager.py index 05ff3b0f285..0cc926cba15 100644 --- a/c7n/manager.py +++ b/c7n/manager.py @@ -14,6 +14,7 @@ import logging from c7n import cache +from c7n.executor import ThreadPoolExecutor from c7n.registry import PluginRegistry from c7n.utils import dumps @@ -27,7 +28,9 @@ class ResourceManager(object): action_registry = None supports_dry_run = False - + + executor_factory = ThreadPoolExecutor + def __init__(self, ctx, data): self.ctx = ctx self.session_factory = ctx.session_factory diff --git a/c7n/resources/elb.py b/c7n/resources/elb.py index f22d313db52..9e87aad82cb 100644 --- a/c7n/resources/elb.py +++ b/c7n/resources/elb.py @@ -63,6 +63,8 @@ from c7n.manager import ResourceManager, resources from c7n.utils import local_session, chunks, type_schema +from functools import partial + log = logging.getLogger('custodian.elb') @@ -81,7 +83,7 @@ def resources(self): elbs = self._cache.get( {'region': self.config.region, 'resource': 'elb'}) if elbs is not None: - self.log.debug("Using cached rds: %d" % ( + self.log.debug("Using cached elb: %d" % ( len(elbs))) return self.filter_resources(elbs) @@ -89,7 +91,7 @@ def resources(self): p = c.get_paginator('describe_load_balancers') results = p.paginate() elbs = list(itertools.chain( - *[rp['LoadBalancerDescriptions'] for rp in results])) + *[self.get_elbs_from_result_page(c, rp) for rp in results])) self._cache.save( {'region': self.config.region, 'resource': 'elbs'}, elbs) @@ -106,6 +108,46 @@ def get_resources(self, resource_ids): return [] raise + def get_elbs_from_result_page(self, client, rp): + elb_descriptions = rp['LoadBalancerDescriptions'] + self.add_tags_to_results(client, elb_descriptions) + return elb_descriptions + + def add_tags_to_results(self, client, elbs): + """ + Gets the tags for the ELBs and adds them to + the result set. + """ + elb_names = [elb['LoadBalancerName'] for elb in elbs] + names_to_tags = {} + fn = partial(self.process_tags, client=client) + futures = [] + with self.executor_factory(max_workers=3) as w: + # max 20 ELBs per call (API limitation) + for elb_names_chunk in chunks(elb_names, size=20): + futures.append( + w.submit(fn, elb_names_chunk)) + + for f in as_completed(futures): + if f.exception(): + self.log.exception("Exception Processing ELB: %s" % ( + f.exception())) + continue + r = f.result() + if r: + names_to_tags.update(r) + + for elb in elbs: + elb['Tags'] = names_to_tags[elb['LoadBalancerName']] + + def process_tags(self, chunk, **kwargs): + tag_descriptions = kwargs['client'].describe_tags(LoadBalancerNames=chunk) + + names_to_tags = {} + for desc in tag_descriptions['TagDescriptions']: + names_to_tags[desc['LoadBalancerName']] = desc['Tags'] + return names_to_tags + @actions.register('delete') class Delete(BaseAction): diff --git a/c7n/resources/rds.py b/c7n/resources/rds.py index 8a570bdfd8b..f0c7232acde 100644 --- a/c7n/resources/rds.py +++ b/c7n/resources/rds.py @@ -69,6 +69,8 @@ from c7n.manager import ResourceManager, resources from c7n.utils import local_session, type_schema +from functools import partial + log = logging.getLogger('custodian.rds') @@ -96,7 +98,7 @@ def resources(self): p = c.get_paginator('describe_db_instances') results = p.paginate(Filters=query) dbs = list(itertools.chain( - *[rp['DBInstances'] for rp in results])) + *[self.get_dbs_from_result_page(c, rp) for rp in results])) self._cache.save( {'region': self.config.region, 'resource': 'rds', 'q': query}, dbs) return self.filter_resources(dbs) @@ -110,6 +112,52 @@ def get_resources(self, resource_ids): DBInstanceIdentifier=db_id)['DBInstances']) return results + def get_dbs_from_result_page(self, client, rp): + dbs = rp['DBInstances'] + self.add_tags_to_results(client, dbs) + return dbs + + def add_tags_to_results(self, client, dbs): + """ + Gets the tags for each of the databases and + adds them to the result set. + """ + account_id = self.get_account_id() + fn = partial(self.process_tags, account_id=account_id, client=client) + with self.executor_factory(max_workers=3) as w: + list(w.map(fn, dbs)) + + def process_tags(self, db, **kwargs): + region = self.get_region(db) + name = db['DBInstanceIdentifier'] + arn = "arn:aws:rds:%s:%s:db:%s" % (region, kwargs['account_id'], name) + tag_list = ( + kwargs['client'].list_tags_for_resource(ResourceName=arn)['TagList'] + ) + tags = [] + if tag_list: + tags.extend(tag_list) + db['Tags'] = tags + return db + + def get_account_id(self): + # TODO: This just gets a role from the current account + # and uses its ARN as the account ARN. See notes + # at top of file for how else we can do this + client = self.session_factory().client('iam') + account_id = ( + client.list_roles(MaxItems=1)['Roles'][0]['Arn'].split(":")[4] + ) + return account_id + + def get_region(self, db): + # get the region from the endpoint + endpoint_parts = db['Endpoint']['Address'].split('.') + + # format is "..rds.amazonaws.com" + region = endpoint_parts[-4] + return region + @filters.register('default-vpc') class DefaultVpc(Filter): diff --git a/tests/data/placebo/test_healthcheck_protocol_mismatch/elasticloadbalancing.DescribeTags_1.json b/tests/data/placebo/test_healthcheck_protocol_mismatch/elasticloadbalancing.DescribeTags_1.json new file mode 100644 index 00000000000..d30537c744f --- /dev/null +++ b/tests/data/placebo/test_healthcheck_protocol_mismatch/elasticloadbalancing.DescribeTags_1.json @@ -0,0 +1,63 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": { + "HTTPStatusCode": 200, + "RequestId": "00000000-0000-0000-0000-000000000000" + }, + "TagDescriptions": [ + { + "Tags": [ + { + "Value": "value1", + "Key": "tag1" + }, + { + "Value": "value2", + "Key": "tag2" + } + ], + "LoadBalancerName": "test-elb-protocol-mismatch" + }, + { + "Tags": [ + { + "Value": "value1", + "Key": "tag1" + }, + { + "Value": "value2", + "Key": "tag2" + } + ], + "LoadBalancerName": "test-elb-protocol-matches" + }, + { + "Tags": [ + { + "Value": "value1", + "Key": "tag1" + }, + { + "Value": "value2", + "Key": "tag2" + } + ], + "LoadBalancerName": "test-elb-no-listeners" + }, + { + "Tags": [ + { + "Value": "value1", + "Key": "tag1" + }, + { + "Value": "value2", + "Key": "tag2" + } + ], + "LoadBalancerName": "test-elb-multiple-listeners" + } + ] + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_ssl_ciphers/elasticloadbalancing.DescribeTags_1.json b/tests/data/placebo/test_ssl_ciphers/elasticloadbalancing.DescribeTags_1.json new file mode 100644 index 00000000000..5f6c797b764 --- /dev/null +++ b/tests/data/placebo/test_ssl_ciphers/elasticloadbalancing.DescribeTags_1.json @@ -0,0 +1,37 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": { + "HTTPStatusCode": 200, + "RequestId": "00000000-0000-0000-0000-000000000000" + }, + "TagDescriptions": [ + { + "Tags": [ + { + "Value": "value1", + "Key": "tag1" + }, + { + "Value": "value2", + "Key": "tag2" + } + ], + "LoadBalancerName": "test-elb-valid-policy" + }, + { + "Tags": [ + { + "Value": "value1", + "Key": "tag1" + }, + { + "Value": "value2", + "Key": "tag2" + } + ], + "LoadBalancerName": "test-elb-invalid-policy" + } + ] + } +} \ No newline at end of file