Translate openstack inventory from script to plugin

.github/BOTMETA.yml

@@ -988,6 +988,14 @@ files:
   lib/ansible/plugins/connection/persistent.py:
     maintainers: $team_networking
     labels: networking
+  lib/ansible/plugins/inventory/openstack.py:
+    maintainers: $team_openstack
+    keywords:
+    - openstack
+    - inventory
+    labels:
+    - cloud
+    - openstack
   lib/ansible/plugins/netconf/:
     maintainers: $team_networking
     labels: networking

lib/ansible/plugins/inventory/openstack.py

@@ -0,0 +1,323 @@
+# Copyright (c) 2012, Marco Vito Moscaritolo <marco@agavee.com>
+# Copyright (c) 2013, Jesse Keating <jesse.keating@rackspace.com>
+# Copyright (c) 2015, Hewlett-Packard Development Company, L.P.
+# Copyright (c) 2016, Rackspace Australia
+# Copyright (c) 2017, Red Hat, Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+'''
+DOCUMENTATION:
+    name: openstack
+    plugin_type: inventory
+    short_description: OpenStack inventory source
+    description:
+        - Get inventory hosts from OpenStack clouds
+        - Uses openstack.(yml|yaml) YAML configuration file to configure the inventory plugin
+        - Uses standard clouds.yaml YAML configuration file to configure cloud credentials
+    options:
+        show_all:
+            description: toggles showing all vms vs only those with a working IP
+            type: boolean
+            default: False
+        inventory_hostname:
+            description: |
+                What to register as the inventory hostname.
+                If set to 'uuid' the uuid of the server will be used and a
+                group will be created for the server name.
+                If set to 'name' the name of the server will be used unless
+                there are more than one server with the same name in which
+                case the 'uuid' logic will be used.
+                Default is to do 'name', which is the opposite of the old
+                openstack.py inventory script's option use_hostnames)
+            type: string
+            choices:
+                - name
+                - uuid
+            default: "name"
+        expand_hostvars:
+            description: |
+                Run extra commands on each host to fill in additional
+                information about the host. May interrogate cinder and
+                neutron and can be expensive for people with many hosts.
+                (Note, the default value of this is opposite from the default
+                old openstack.py inventory script's option expand_hostvars)
+            type: boolean
+            default: False
+        private:
+            description: |
+                Use the private interface of each server, if it has one, as
+                the host's IP in the inventory. This can be useful if you are
+                running ansible inside a server in the cloud and would rather
+                communicate to your servers over the private network.
+            type: boolean
+            default: False
+        only_clouds:
+            description: |
+                List of clouds from clouds.yaml to use, instead of using
+                the whole list.
+            type: list
+            default: []
+        fail_on_errors:
+            description: |
+                Causes the inventory to fail and return no hosts if one cloud
+                has failed (for example, bad credentials or being offline).
+                When set to False, the inventory will return as many hosts as
+                it can from as many clouds as it can contact. (Note, the
+                default value of this is opposite from the old openstack.py
+                inventory script's option fail_on_errors)
+            type: boolean
+            default: False
+        clouds_yaml_path:
+            description: |
+                Override path to clouds.yaml file. If this value is given it
+                will be searched first. The default path for the
+                ansible inventory adds /etc/ansible/openstack.yaml and
+                /etc/ansible/openstack.yml to the regular locations documented
+                at https://docs.openstack.org/os-client-config/latest/user/configuration.html#config-files
+            type: string
+            default: None
+        compose:
+            description: Create vars from jinja2 expressions.
+            type: dictionary
+            default: {}
+        groups:
+            description: Add hosts to group based on Jinja2 conditionals.
+            type: dictionary
+            default: {}
+EXAMPLES:
+# file must be named openstack.yaml or openstack.yml
+# Make the plugin behave like the default behavior of the old script
+simple_config_file:
+    plugin: openstack
+    inventory_hostname: 'name'
+    expand_hostvars: true
+    fail_on_errors: true
+'''
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import collections
+
+from ansible.errors import AnsibleParserError
+from ansible.plugins.inventory import BaseInventoryPlugin
+
+try:
+    import os_client_config
+    import shade
+    import shade.inventory
+    HAS_SHADE = True
+except ImportError:
+    HAS_SHADE = False
+
+
+class InventoryModule(BaseInventoryPlugin):
+    ''' Host inventory provider for ansible using OpenStack clouds. '''
+
+    NAME = 'openstack'
+
+    def parse(self, inventory, loader, path, cache=True):
+
+        super(InventoryModule, self).parse(inventory, loader, path)
+
+        cache_key = self.get_cache_prefix(path)
+
+        # file is config file
+        try:
+            self._config_data = self.loader.load_from_file(path)
+        except Exception as e:
+            raise AnsibleParserError(e)
+
+        if not self._config_data:
+            # empty. this is not my config file
+            return False
+        if 'plugin' in self._config_data and self._config_data['plugin'] != self.NAME:
+            # plugin config file, but not for us
+            return False
+        elif 'plugin' not in self._config_data and 'clouds' not in self._config_data:
+            # it's not a clouds.yaml file either
+            return False
+
+        if not HAS_SHADE:
+            self.display.warning(
+                'shade is required for the OpenStack inventory plugin.'
+                ' OpenStack inventory sources will be skipped.')
+            return False
+
+        # The user has pointed us at a clouds.yaml file. Use defaults for
+        # everything.
+        if 'clouds' in self._config_data:
+            self._config_data = {}
+
+        source_data = None
+        if cache and cache_key in inventory.cache:
+            try:
+                source_data = inventory.cache[cache_key]
+            except KeyError:
+                pass
+
+        if not source_data:
+            clouds_yaml_path = self._config_data.get('clouds_yaml_path')
+            if clouds_yaml_path:
+                config_files = (clouds_yaml_path +
+                                os_client_config.config.CONFIG_FILES)
+            else:
+                config_files = None
+
+            # TODO(mordred) Integrate shade's logging with ansible's logging
+            shade.simple_logging()
+
+            cloud_inventory = shade.inventory.OpenStackInventory(
+                config_files=config_files,
+                private=self._config_data.get('private', False))
+            only_clouds = self._config_data.get('only_clouds', [])
+            if only_clouds and not isinstance(only_clouds, list):
+                raise ValueError(
+                    'OpenStack Inventory Config Error: only_clouds must be'
+                    ' a list')
+            if only_clouds:
+                new_clouds = []
+                for cloud in cloud_inventory.clouds:
+                    if cloud.name in only_clouds:
+                        new_clouds.append(cloud)
+                cloud_inventory.clouds = new_clouds
+
+            expand_hostvars = self._config_data.get('expand_hostvars', False)
+            fail_on_errors = self._config_data.get('fail_on_errors', False)
+
+            source_data = cloud_inventory.list_hosts(
+                expand=expand_hostvars, fail_on_cloud_config=fail_on_errors)
+
+            inventory.cache[cache_key] = source_data
+
+        self._populate_from_source(source_data)
+
+    def _populate_from_source(self, source_data):
+        groups = collections.defaultdict(list)
+        firstpass = collections.defaultdict(list)
+        hostvars = {}
+
+        use_server_id = (
+            self._config_data.get('inventory_hostname', 'name') != 'name')
+        show_all = self._config_data.get('show_all', False)
+
+        for server in source_data:
+            if 'interface_ip' not in server and not show_all:
+                continue
+            firstpass[server['name']].append(server)
+
+        for name, servers in firstpass.items():
+            if len(servers) == 1 and not use_server_id:
+                self._append_hostvars(hostvars, groups, name, servers[0])
+            else:
+                server_ids = set()
+                # Trap for duplicate results
+                for server in servers:
+                    server_ids.add(server['id'])
+                if len(server_ids) == 1 and not use_server_id:
+                    self._append_hostvars(hostvars, groups, name, servers[0])
+                else:
+                    for server in servers:
+                        self._append_hostvars(
+                            hostvars, groups, server['id'], server,
+                            namegroup=True)
+
+        self._set_variables(hostvars, groups)
+
+    def _set_variables(self, hostvars, groups):
+
+        # set vars in inventory from hostvars
+        for host in hostvars:
+
+            # create composite vars
+            self._set_composite_vars(
+                self._config_data.get('compose'), hostvars, host)
+
+            # actually update inventory
+            for key in hostvars[host]:
+                self.inventory.set_variable(host, key, hostvars[host][key])
+
+            # constructed groups based on conditionals
+            self._add_host_to_composed_groups(
+                self._config_data.get('groups'), hostvars, host)
+
+        for group_name, group_hosts in groups.items():
+            self.inventory.add_group(group_name)
+            for host in group_hosts:
+                self.inventory.add_child(group_name, host)
+
+    def _get_groups_from_server(self, server_vars, namegroup=True):
+        groups = []
+
+        region = server_vars['region']
+        cloud = server_vars['cloud']
+        metadata = server_vars.get('metadata', {})
+
+        # Create a group for the cloud
+        groups.append(cloud)
+
+        # Create a group on region
+        groups.append(region)
+
+        # And one by cloud_region
+        groups.append("%s_%s" % (cloud, region))
+
+        # Check if group metadata key in servers' metadata
+        if 'group' in metadata:
+            groups.append(metadata['group'])
+
+        for extra_group in metadata.get('groups', '').split(','):
+            if extra_group:
+                groups.append(extra_group.strip())
+
+        groups.append('instance-%s' % server_vars['id'])
+        if namegroup:
+            groups.append(server_vars['name'])
+
+        for key in ('flavor', 'image'):
+            if 'name' in server_vars[key]:
+                groups.append('%s-%s' % (key, server_vars[key]['name']))
+
+        for key, value in iter(metadata.items()):
+            groups.append('meta-%s_%s' % (key, value))
+
+        az = server_vars.get('az', None)
+        if az:
+            # Make groups for az, region_az and cloud_region_az
+            groups.append(az)
+            groups.append('%s_%s' % (region, az))
+            groups.append('%s_%s_%s' % (cloud, region, az))
+        return groups
+
+    def _append_hostvars(self, hostvars, groups, current_host,
+                         server, namegroup=False):
+        hostvars[current_host] = dict(
+            ansible_ssh_host=server['interface_ip'],
+            ansible_host=server['interface_ip'],
+            openstack=server)
+        self.inventory.add_host(current_host)
+
+        for group in self._get_groups_from_server(server, namegroup=namegroup):
+            groups[group].append(current_host)
+
+    def verify_file(self, path):
+
+        if super(InventoryModule, self).verify_file(path):
+            for fn in ('openstack', 'clouds'):
+                for suffix in ('yaml', 'yml'):
+                    maybe = '{fn}.{suffix}'.format(fn=fn, suffix=suffix)
+                    if path.endswith(maybe):
+                        return True
+        return False