Refactor inventory plugins (ansible-collections#1218)

Refactor inventory plugins

SUMMARY

Refactor inventory plugins to put common code into module_utils
add unit tests for inventory aws_rds
support integration testing for aws_rds inventory

ISSUE TYPE

Bugfix Pull Request
Feature Pull Request

COMPONENT NAME

aws_rds
aws_ec2

Reviewed-by: Gonéri Le Bouder <goneri@lebouder.net>
Reviewed-by: Bikouo Aubin <None>
Reviewed-by: Mark Chappell <None>

changelogs/fragments/refactor_inventory_plugins.yml

@@ -0,0 +1,3 @@
+---
+minor_changes:
+- Refactor inventory plugins and add aws_rds inventory unit tests (https://github.com/ansible-collections/amazon.aws/pull/1218).

plugins/plugin_utils/inventory.py

@@ -0,0 +1,176 @@
+# Copyright: (c) 2022, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+try:
+    import boto3
+    import botocore
+except ImportError:
+    pass  # will be captured by imported HAS_BOTO3
+
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+from ansible.module_utils._text import to_native
+from ansible.template import Templar
+from ansible.plugins.inventory import BaseInventoryPlugin
+from ansible.plugins.inventory import Cacheable
+from ansible.plugins.inventory import Constructable
+from ansible_collections.amazon.aws.plugins.plugin_utils.botocore import boto3_conn
+from ansible_collections.amazon.aws.plugins.plugin_utils.base import AWSPluginBase
+
+
+def _boto3_session(profile_name=None):
+    if profile_name is None:
+        return boto3.Session()
+    return boto3.session.Session(profile_name=profile_name)
+
+
+class AWSInventoryBase(BaseInventoryPlugin, Constructable, Cacheable, AWSPluginBase):
+
+    def __init__(self):
+
+        super(AWSInventoryBase, self).__init__()
+
+        self.boto_profile = None
+        self.aws_secret_access_key = None
+        self.aws_access_key_id = None
+        self.aws_security_token = None
+        self.iam_role_arn = None
+
+    def _get_credentials(self):
+        '''
+            :return A dictionary of boto client credentials
+        '''
+        boto_params = {
+            'aws_access_key_id': self.aws_access_key_id,
+            'aws_secret_access_key': self.aws_secret_access_key,
+            'aws_session_token': self.aws_security_token,
+        }
+        return {k: v for k, v in boto_params.items() if v}
+
+    def _set_credentials(self, loader):
+        '''
+            :param config_data: contents of the inventory config file
+        '''
+
+        templar = Templar(loader=loader)
+        credentials = {}
+
+        for credential_type in ('aws_profile', 'aws_access_key', 'aws_secret_key', 'aws_security_token', 'iam_role_arn'):
+            if templar.is_template(self.get_option(credential_type)):
+                credentials[credential_type] = templar.template(variable=self.get_option(credential_type), disable_lookups=False)
+            else:
+                credentials[credential_type] = self.get_option(credential_type)
+
+        self.boto_profile = credentials['aws_profile']
+        self.aws_access_key_id = credentials['aws_access_key']
+        self.aws_secret_access_key = credentials['aws_secret_key']
+        self.aws_security_token = credentials['aws_security_token']
+        self.iam_role_arn = credentials['iam_role_arn']
+
+        if not self.boto_profile and not (self.aws_access_key_id and self.aws_secret_access_key):
+            session = botocore.session.get_session()
+            try:
+                credentials = session.get_credentials().get_frozen_credentials()
+            except AttributeError:
+                pass
+            else:
+                self.aws_access_key_id = credentials.access_key
+                self.aws_secret_access_key = credentials.secret_key
+                self.aws_security_token = credentials.token
+
+        if not self.boto_profile and not (self.aws_access_key_id and self.aws_secret_access_key):
+            self.fail_aws("Insufficient boto credentials found. Please provide them in your "
+                          "inventory configuration file or set them as environment variables.")
+
+    def _get_connection(self, credentials, resource, region='us-east-1'):
+        return boto3_conn(self, conn_type='client', resource=resource, region=region, **credentials)
+
+    def _boto3_assume_role(self, credentials, resource, iam_role_arn, profile_name, region=None):
+        """
+        Assume an IAM role passed by iam_role_arn parameter
+
+        :return: a dict containing the credentials of the assumed role
+        """
+
+        try:
+            sts_connection = _boto3_session(profile_name=profile_name).client('sts', region, **credentials)
+            role_session_name = f"ansible_aws_{resource}_dynamic_inventory"
+            sts_session = sts_connection.assume_role(RoleArn=iam_role_arn, RoleSessionName=role_session_name)
+            return dict(
+                aws_access_key_id=sts_session['Credentials']['AccessKeyId'],
+                aws_secret_access_key=sts_session['Credentials']['SecretAccessKey'],
+                aws_session_token=sts_session['Credentials']['SessionToken']
+            )
+        except botocore.exceptions.ClientError as e:
+            self.fail_aws("Unable to assume IAM role: %s" % to_native(e))
+
+    def _boto3_regions(self, credentials, iam_role_arn, resource):
+
+        regions = []
+        if iam_role_arn is not None:
+            try:
+                # Describe regions assuming arn role
+                assumed_credentials = self._boto3_assume_role(credentials, resource, iam_role_arn, self.boto_profile)
+                client = self._get_connection(credentials=assumed_credentials, resource='ec2')
+                resp = client.describe_regions()
+                regions = [x['RegionName'] for x in resp.get('Regions', [])]
+            except botocore.exceptions.NoRegionError:
+                # above seems to fail depending on boto3 version, ignore and lets try something else
+                pass
+        else:
+            try:
+                # as per https://boto3.amazonaws.com/v1/documentation/api/latest/guide/ec2-example-regions-avail-zones.html
+                client = self._get_connection(credentials=credentials, resource='ec2')
+                resp = client.describe_regions()
+                regions = [x['RegionName'] for x in resp.get('Regions', [])]
+            except botocore.exceptions.NoRegionError:
+                # above seems to fail depending on boto3 version, ignore and lets try something else
+                pass
+            except is_boto3_error_code('UnauthorizedOperation') as e:  # pylint: disable=duplicate-except
+                self.fail_aws("Unauthorized operation: %s" % to_native(e))
+
+        # fallback to local list hardcoded in boto3 if still no regions
+        if not regions:
+            session = _boto3_session()
+            regions = session.get_available_regions(resource)
+
+        return regions
+
+    def _get_boto3_connection(self, credentials, iam_role_arn, resource, region):
+        try:
+            assumed_credentials = credentials
+            if iam_role_arn is not None:
+                assumed_credentials = self._boto3_assume_role(credentials, resource, iam_role_arn, self.boto_profile, region)
+            return _boto3_session(profile_name=self.boto_profile).client(resource, region, **assumed_credentials)
+        except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e:
+            if self.boto_profile:
+                try:
+                    return _boto3_session(profile_name=self.boto_profile).client(resource, region)
+                except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e:
+                    self.fail_aws("Insufficient credentials found: %s" % to_native(e))
+            else:
+                self.fail_aws("Insufficient credentials found: %s" % to_native(e))
+
+    def _boto3_conn(self, regions, resource):
+        '''
+            :param regions: A list of regions to create a boto3 client
+
+            Generator that yields a boto3 client and the region
+        '''
+        credentials = self._get_credentials()
+        iam_role_arn = self.iam_role_arn
+
+        if not regions:
+            # list regions as none was provided
+            regions = self._boto3_regions(credentials, iam_role_arn, resource)
+
+        # I give up, now you MUST give me regions
+        if not regions:
+            self.fail_aws('Unable to get regions list from available methods, you must specify the "regions" option to continue.')
+
+        for region in regions:
+            connection = self._get_boto3_connection(credentials, iam_role_arn, resource, region)
+            yield connection, region

tests/integration/targets/inventory_aws_ec2/aliases

@@ -1,3 +1,3 @@
-time=45m
+time=10m
 
 cloud/aws

tests/integration/targets/inventory_aws_ec2/playbooks/manage_ec2_instances.yml

@@ -0,0 +1,21 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+
+  collections:
+    - amazon.aws
+    - community.aws
+
+  vars_files:
+    - vars/main.yml
+
+  module_defaults:
+    group/aws:
+      aws_access_key: '{{ aws_access_key }}'
+      aws_secret_key: '{{ aws_secret_key }}'
+      security_token: '{{ security_token | default(omit) }}'
+      region: '{{ aws_region }}'
+
+  tasks:
+    - include_tasks: "{{ task }}.yml"

tests/integration/targets/inventory_aws_ec2/playbooks/populate_cache.yml

@@ -13,43 +13,7 @@
           region: '{{ aws_region }}'
       block:
 
-        # Create VPC, subnet, security group, and find image_id to create instance
-
-        - include_tasks: setup.yml
-#        - pause:
-#            seconds: 240
-
-        - name: assert group was populated with inventory but is empty
-          assert:
-            that:
-              - "'aws_ec2' in groups"
-              - "not groups.aws_ec2"
-
-        # Create new host, add it to inventory and then terminate it without updating the cache
-
-        - name: create a new host
-          ec2_instance:
-            image_id: '{{ image_id }}'
-            name: '{{ resource_prefix }}'
-            instance_type: t2.micro
-            wait: no
-            security_groups: '{{ sg_id }}'
-            vpc_subnet_id: '{{ subnet_id }}'
-          register: setup_instance
-
-        - meta: refresh_inventory
-
-      always:
-
-        - name: remove setup ec2 instance
-          ec2_instance:
-            instance_type: t2.micro
-            instance_ids: '{{ setup_instance.instance_ids }}'
-            state: absent
-            name: '{{ resource_prefix }}'
-            security_groups: '{{ sg_id }}'
-            vpc_subnet_id: '{{ subnet_id }}'
-          ignore_errors: yes
-          when: setup_instance is defined
+        - debug:
+            var: groups
 
         - include_tasks: tear_down.yml

tests/integration/targets/inventory_aws_ec2/playbooks/setup.yml

@@ -29,7 +29,6 @@
 - name: create a subnet to use for creating an ec2 instance
   ec2_vpc_subnet:
     az: '{{ aws_region }}a'
-    tags: '{{ resource_prefix }}_setup'
     vpc_id: '{{ setup_vpc.vpc.id }}'
     cidr: '{{ subnet_cidr }}'
     state: present
@@ -50,3 +49,13 @@
 
 - set_fact:
     sg_id: '{{ setup_sg.group_id }}'
+
+- name: Create ec2 instance
+  ec2_instance:
+    image_id: '{{ image_id }}'
+    name: '{{ resource_prefix }}'
+    instance_type: t2.micro
+    security_groups: '{{ sg_id }}'
+    vpc_subnet_id: '{{ subnet_id }}'
+    wait: no
+  register: setup_instance

tests/integration/targets/inventory_aws_ec2/playbooks/tear_down.yml

@@ -2,30 +2,54 @@
     vpc_cidr: '10.{{ 256 | random(seed=resource_prefix) }}.0.0/16'
     subnet_cidr: '10.{{ 256 | random(seed=resource_prefix) }}.0.0/24'
 
-- name: remove setup security group
-  ec2_security_group:
-    name: '{{ resource_prefix }}_setup'
-    description: 'created by Ansible integration tests'
-    state: absent
-    vpc_id: '{{ vpc_id }}'
-  ignore_errors: yes
-
-- name: remove setup subnet
-  ec2_vpc_subnet:
-    az: '{{ aws_region }}a'
-    tags: '{{ resource_prefix }}_setup'
-    vpc_id: '{{ vpc_id }}'
-    cidr: '{{ subnet_cidr }}'
-    state: absent
-    resource_tags:
-      Name: '{{ resource_prefix }}_setup'
-  ignore_errors: yes
-
-- name: remove setup VPC
-  ec2_vpc_net:
-    cidr_block: '{{ vpc_cidr }}'
-    state: absent
-    name: '{{ resource_prefix }}_setup'
-    resource_tags:
-      Name: '{{ resource_prefix }}_setup'
-  ignore_errors: yes
+- name: describe vpc
+  ec2_vpc_net_info:
+    filters:
+      tag:Name: '{{ resource_prefix }}_setup'
+  register: vpc_info
+
+- block:
+    - set_fact:
+        vpc_id: "{{ vpc_info.vpcs.0.vpc_id }}"
+
+    - name: list existing instances
+      ec2_instance_info:
+        filters:
+          vpc-id: "{{ vpc_id }}"
+      register: existing
+
+    - name: remove ec2 instances
+      ec2_instance:
+        instance_ids: "{{ existing.instances | map(attribute='instance_id') | list }}"
+        wait: yes
+        state: absent
+
+    - name: remove setup security group
+      ec2_security_group:
+        name: '{{ resource_prefix }}_setup'
+        description: 'created by Ansible integration tests'
+        state: absent
+        vpc_id: '{{ vpc_id }}'
+      ignore_errors: yes
+
+    - name: remove setup subnet
+      ec2_vpc_subnet:
+        az: '{{ aws_region }}a'
+        tags: '{{ resource_prefix }}_setup'
+        vpc_id: '{{ vpc_id }}'
+        cidr: '{{ subnet_cidr }}'
+        state: absent
+        resource_tags:
+          Name: '{{ resource_prefix }}_setup'
+      ignore_errors: yes
+
+    - name: remove setup VPC
+      ec2_vpc_net:
+        cidr_block: '{{ vpc_cidr }}'
+        state: absent
+        name: '{{ resource_prefix }}_setup'
+        resource_tags:
+          Name: '{{ resource_prefix }}_setup'
+      ignore_errors: yes
+
+  when: vpc_info.vpcs | length > 0

tests/integration/targets/inventory_aws_ec2/playbooks/test_inventory_cache.yml

@@ -7,7 +7,7 @@
       assert:
         that:
           - "'aws_ec2' in groups"
-          - "groups.aws_ec2 | length == 1"
+          - "groups.aws_ec2 | length > 0"
 
     - meta: refresh_inventory