Merge pull request #16606 from ryansb/rds-cluster-inventory-aioue

Support RDS clusters in AWS dynamic inventory script

contrib/inventory/ec2.ini

@@ -82,6 +82,9 @@ all_instances = False
 # 'all_rds_instances' to True return all RDS instances regardless of state.
 all_rds_instances = False
 
+# Include RDS cluster information (Aurora etc.)
+include_rds_clusters = False
+
 # By default, only ElastiCache clusters and nodes in the 'available' state
 # are returned. Set 'all_elasticache_clusters' and/or 'all_elastic_nodes'
 # to True return all ElastiCache clusters and nodes, regardless of state.

contrib/inventory/ec2.py

@@ -131,6 +131,15 @@
 from boto import route53
 import six
 
+from ansible.module_utils import ec2 as ec2_utils
+
+HAS_BOTO3 = False
+try:
+    import boto3
+    HAS_BOTO3 = True
+except ImportError:
+    pass
+
 from six.moves import configparser
 from collections import defaultdict
 
@@ -265,6 +274,12 @@ def read_settings(self):
         if config.has_option('ec2', 'rds'):
             self.rds_enabled = config.getboolean('ec2', 'rds')
 
+        # Include RDS cluster instances?
+        if config.has_option('ec2', 'include_rds_clusters'):
+            self.include_rds_clusters = config.getboolean('ec2', 'include_rds_clusters')
+        else:
+            self.include_rds_clusters = False
+
         # Include ElastiCache instances?
         self.elasticache_enabled = True
         if config.has_option('ec2', 'elasticache'):
@@ -474,6 +489,8 @@ def do_api_calls_update_cache(self):
             if self.elasticache_enabled:
                 self.get_elasticache_clusters_by_region(region)
                 self.get_elasticache_replication_groups_by_region(region)
+            if self.include_rds_clusters:
+                self.include_rds_clusters_by_region(region)
 
         self.write_to_cache(self.inventory, self.cache_path_cache)
         self.write_to_cache(self.index, self.cache_path_index)
@@ -574,6 +591,65 @@ def get_rds_instances_by_region(self, region):
                 error = "Looks like AWS RDS is down:\n%s" % e.message
             self.fail_with_error(error, 'getting RDS instances')
 
+    def include_rds_clusters_by_region(self, region):
+        if not HAS_BOTO3:
+            self.fail_with_error("Working with RDS clusters requires boto3 - please install boto3 and try again",
+                                 "getting RDS clusters")
+
+        client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials)
+
+        marker, clusters = '', []
+        while marker is not None:
+            resp = client.describe_db_clusters(Marker=marker)
+            clusters.extend(resp["DBClusters"])
+            marker = resp.get('Marker', None)
+
+        account_id = boto.connect_iam().get_user().arn.split(':')[4]
+        c_dict = {}
+        for c in clusters:
+            # remove these datetime objects as there is no serialisation to json
+            # currently in place and we don't need the data yet
+            if 'EarliestRestorableTime' in c:
+                del c['EarliestRestorableTime']
+            if 'LatestRestorableTime' in c:
+                del c['LatestRestorableTime']
+
+            if self.ec2_instance_filters == {}:
+                matches_filter = True
+            else:
+                matches_filter = False
+
+            try:
+                # arn:aws:rds:<region>:<account number>:<resourcetype>:<name>
+                tags = client.list_tags_for_resource(
+                    ResourceName='arn:aws:rds:' + region + ':' + account_id + ':cluster:' + c['DBClusterIdentifier'])
+                c['Tags'] = tags['TagList']
+
+                if self.ec2_instance_filters:
+                    for filter_key, filter_values in self.ec2_instance_filters.items():
+                        # get AWS tag key e.g. tag:env will be 'env'
+                        tag_name = filter_key.split(":", 1)[1]
+                        # Filter values is a list (if you put multiple values for the same tag name)
+                        matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags'])
+
+                        if matches_filter:
+                            # it matches a filter, so stop looking for further matches
+                            break
+
+            except Exception as e:
+                if e.message.find('DBInstanceNotFound') >= 0:
+                    # AWS RDS bug (2016-01-06) means deletion does not fully complete and leave an 'empty' cluster.
+                    # Ignore errors when trying to find tags for these
+                    pass
+
+            # ignore empty clusters caused by AWS bug
+            if len(c['DBClusterMembers']) == 0:
+                continue
+            elif matches_filter:
+                c_dict[c['DBClusterIdentifier']] = c
+
+        self.inventory['db_clusters'] = c_dict
+
     def get_elasticache_clusters_by_region(self, region):
         ''' Makes an AWS API call to the list of ElastiCache clusters (with
         nodes' info) in a particular region.'''

lib/ansible/module_utils/ec2.py

@@ -55,10 +55,19 @@ class AnsibleAWSError(Exception):
 
 
 def boto3_conn(module, conn_type=None, resource=None, region=None, endpoint=None, **params):
+    try:
+        return _boto3_conn(conn_type=None, resource=None, region=None, endpoint=None, **params)
+    except ValueError:
+        module.fail_json(msg='There is an issue in the code of the module. You must specify either both, resource or client to the conn_type parameter in the boto3_conn function call')
+
+def _boto3_conn(conn_type=None, resource=None, region=None, endpoint=None, **params):
     profile = params.pop('profile_name', None)
 
     if conn_type not in ['both', 'resource', 'client']:
-        module.fail_json(msg='There is an issue in the code of the module. You must specify either both, resource or client to the conn_type parameter in the boto3_conn function call')
+        raise ValueError('There is an issue in the calling code. You '
+                         'must specify either both, resource, or client to '
+                         'the conn_type parameter in the boto3_conn function '
+                         'call')
 
     if conn_type == 'resource':
         resource = boto3.session.Session(profile_name=profile).resource(resource, region_name=region, endpoint_url=endpoint, **params)
@@ -71,6 +80,7 @@ def boto3_conn(module, conn_type=None, resource=None, region=None, endpoint=None
         client = boto3.session.Session(profile_name=profile).client(resource, region_name=region, endpoint_url=endpoint, **params)
         return client, resource
 
+boto3_inventory_conn = _boto3_conn
 
 def aws_common_argument_spec():
     return dict(