parse inventory using Ansible Python API

src/ansible-cmdb.py

@@ -158,6 +158,9 @@ def parse_user_params(user_params):
     parser.add_option("-C", "--cust-cols", dest="cust_cols", action="store", default=None, help="Path to a custom columns definition file")
     parser.add_option("-l", "--limit", dest="limit", action="store", default=None, help="Limit hosts to pattern")
     parser.add_option("--exclude-cols", dest="exclude_columns", action="store", default=None, help="Exclude cols from output")
+    parser.add_option("--use-ansible-api", dest="use_ansible_api", action="store_true", default=False,
+                      help="Use the Ansible python API to read the inventory files")
+
     (options, args) = parser.parse_args()
 
     if len(args) < 1:
@@ -199,8 +202,12 @@ def parse_user_params(user_params):
     log.debug('inventory files = {0}'.format(hosts_files))
     log.debug('template params = {0}'.format(params))
 
-    ansible = ansiblecmdb.Ansible(args, hosts_files, options.fact_cache,
-                                  limit=options.limit, debug=options.debug)
+    if options.use_ansible_api:
+        ansible = ansiblecmdb.AnsibleViaAPI(args, hosts_files, options.fact_cache,
+                                            limit=options.limit, debug=options.debug)
+    else:
+        ansible = ansiblecmdb.Ansible(args, hosts_files, options.fact_cache,
+                                      limit=options.limit, debug=options.debug)
 
     # Render a template with the gathered host info
     renderer = render.Render(options.template, ['.', tpl_dir])

src/ansiblecmdb/__init__.py

@@ -1,3 +1,4 @@
 from . import ihateyaml
 from .parser import HostsParser, DynInvParser
-from .ansible import Ansible
+from .ansible_cmdb import Ansible
+from .ansible_via_api import AnsibleViaAPI

src/ansiblecmdb/ansible.py → src/ansiblecmdb/ansible_cmdb.py

@@ -48,10 +48,16 @@ def __init__(self, fact_dirs, inventory_paths=None, fact_cache=False,
         self.hosts = {}
         self.log = logging.getLogger(__name__)
 
-        # Process facts gathered by Ansible's setup module of fact caching.
+        self.load_facts()
+        self.load_inventories()
+
+    def load_facts(self):
+        """Process facts gathered by Ansible's setup module of fact caching."""
         for fact_dir in self.fact_dirs:
             self._parse_fact_dir(fact_dir, self.fact_cache)
 
+    def load_inventories(self):
+        """Load inventories from inventory_paths."""
         # Scan the inventory for known hosts.
         for inventory_path in self.inventory_paths:
             self._handle_inventory(inventory_path)

src/ansiblecmdb/ansible_via_api.py

@@ -0,0 +1,103 @@
+from ansible.parsing.dataloader import DataLoader
+from ansible.inventory.manager import InventoryManager
+from ansible.vars.manager import VariableManager
+
+from ansiblecmdb import Ansible
+
+
+class AnsibleViaAPI(Ansible):
+    """
+    Gather Ansible host information using the Ansible Python API.
+
+    `fact_dirs` is a list of paths to directories containing facts
+    gathered by Ansible's 'setup' module.
+
+    `inventory_paths` is a list with files or directories containing the
+    inventory.
+    """
+    def load_inventories(self):
+        """Load host inventories using the Ansible Python API."""
+        loader = DataLoader()
+        inventory = InventoryManager(loader=loader, sources=self.inventory_paths)
+        variable_manager = VariableManager(loader=loader, inventory=inventory)
+
+        # some Ansible variables we don't need.
+        ignore = ['ansible_playbook_python',
+                  'groups',
+                  'inventory_dir',
+                  'inventory_file',
+                  'omit',
+                  'playbook_dir']
+
+        # Handle limits here because Ansible understands more complex
+        # limit syntax than ansible-cmdb (e.g. globbing matches []?*
+        # and :& and matches).  Remove any extra hosts that were
+        # loaded by facts.  We could optimize a bit by arranging to
+        # load facts after inventory and skipping loading any facts
+        # files for hosts not included in limited hosts, but for now
+        # we do the simplest thing that can work.
+        if self.limit:
+            inventory.subset(self.limit)
+            limited_hosts = inventory.get_hosts()
+            for h in self.hosts.keys():
+                if h not in limited_hosts:
+                    del self.hosts[h]
+
+        for host in inventory.get_hosts():
+            vars = variable_manager.get_vars(host=host)
+            for key in ignore:
+                vars.pop(key, None)
+
+            hostname = vars['inventory_hostname']
+            groupnames = vars.pop('group_names', [])
+            merge_host_key_val(self.hosts, hostname, 'name', hostname)
+            merge_host_key_val(self.hosts, hostname, 'groups', set(groupnames))
+            merge_host_key_val(self.hosts, hostname, 'hostvars', vars)
+
+    def get_hosts(self):
+        """
+        Return a dict of parsed hosts info, with the limit applied if required.
+        """
+        # We override this method since we already applied the limit
+        # when we loaded the inventory.
+        return self.hosts
+
+
+def merge_host_key_val(hosts_dict, hostname, key, val):
+    """
+    Update hosts_dict[`hostname`][`key`] with `val`, taking into
+    account all the possibilities of missing keys and merging
+    `val` into an existing list, set or dictionary target value.
+    When merging into a dict target value any matching keys will
+    be overwritten by the new value.  Merging into a list or set
+    target value does not remove existing entries but instead adds
+    the new values to the collection.  If the target value is
+    is not a dict or collection it will be overwritten.
+
+    This will be called with key in ['hostvars', 'groups', 'name'],
+    although the implementation would work with any hashable key.
+    """
+    if hostname not in hosts_dict:
+        hosts_dict[hostname] = {
+            'name': hostname,
+            'hostvars': {},
+            'groups': set()
+        }
+
+    hostdata = hosts_dict[hostname]
+    if key not in hostdata:
+        hostdata[key] = val
+        return
+
+    # We handle the list case because the analogous util.deepupdate
+    # does.  It might be needed in deepupdate for facts, but the
+    # host inventory that we build is all dicts and sets.
+    target = hostdata[key]
+    if hasattr(target, 'update'):
+        target.update(val)   # merge into target dict
+    elif hasattr(target, 'union'):
+        target.union(val)    # union into target set
+    elif hasattr(target, 'extend'):
+        target.extend(val)   # extend target list
+    else:
+        hostdata[key] = val  # overwrite non-mergeable target value

test/f_inventory/mixeddir/dyninv.py

@@ -20,6 +20,7 @@
     },
     "marietta": [ "host6.example.com" ],
     "5points": [ "host7.example.com" ],
+    "ungrouped": [ "moocow.example.com", "llama.example.com" ],
     "_meta" : {
         "hostvars": {
             "moocow.example.com": {

test/test.sh

@@ -2,6 +2,8 @@
 
 echo "Python v2"
 python2 test.py
+python2 test_ansible_api.py
 
 echo "Python v3"
 python3 test.py
+python3 test_ansible_api.py

test/test_ansible_api.py

@@ -0,0 +1,139 @@
+import sys
+import unittest
+import os
+
+sys.path.insert(0, os.path.realpath('../lib'))
+sys.path.insert(0, os.path.realpath('../src'))
+import ansiblecmdb
+
+
+class ExtendTestCase(unittest.TestCase):
+    """
+    Test the extending of facts.
+    """
+    def testExtendOverrideParams(self):
+        """
+        Test that we can override a native fact
+        """
+        fact_dirs = ['f_extend/out_setup', 'f_extend/extend']
+        ansible = ansiblecmdb.AnsibleViaAPI(fact_dirs)
+        env_editor = ansible.hosts['debian.dev.local']['ansible_facts']['ansible_env']['EDITOR']
+        self.assertEqual(env_editor, 'nano')
+
+    def testExtendAddParams(self):
+        """
+        Test that we can add new facts
+        """
+        fact_dirs = ['f_extend/out_setup', 'f_extend/extend']
+        ansible = ansiblecmdb.AnsibleViaAPI(fact_dirs)
+        software = ansible.hosts['debian.dev.local']['software']
+        self.assertIn('Apache2', software)
+
+
+class HostParseTestCase(unittest.TestCase):
+    """
+    Test specifics of the hosts inventory parser
+    """
+    def testChildGroupHosts(self):
+        """
+        Test that children groups contain all hosts they should.
+        """
+        fact_dirs = ['f_hostparse/out']
+        inventories = ['f_hostparse/hosts']
+        ansible = ansiblecmdb.AnsibleViaAPI(fact_dirs, inventories)
+        groups = ansible.hosts['db.dev.local']['groups']
+        self.assertIn('db', groups)
+        self.assertIn('dev', groups)
+        self.assertIn('dev_local', groups)
+
+    def testChildGroupVars(self):
+        """
+        Test that all vars applied against a child group are set on the hosts.
+        """
+        fact_dirs = ['f_hostparse/out']
+        inventories = ['f_hostparse/hosts']
+        ansible = ansiblecmdb.AnsibleViaAPI(fact_dirs, inventories)
+        host_vars = ansible.hosts['db.dev.local']['hostvars']
+        self.assertEqual(host_vars['function'], 'db')
+        self.assertEqual(host_vars['dtap'], 'dev')
+
+    def testExpandHostDef(self):
+        """
+        Verify that host ranges are properly expanded. E.g. db[01-03].local ->
+        db01.local, db02.local, db03.local.
+        """
+        fact_dirs = ['f_hostparse/out']
+        inventories = ['f_hostparse/hosts']
+        ansible = ansiblecmdb.AnsibleViaAPI(fact_dirs, inventories)
+        self.assertIn('web02.dev.local', ansible.hosts)
+        self.assertIn('fe03.dev02.local', ansible.hosts)
+
+
+class InventoryTestCase(unittest.TestCase):
+    def testHostsDir(self):
+        """
+        Verify that we can specify a directory as the hosts inventory file and
+        that all files are parsed.
+        """
+        fact_dirs = ['f_inventory/out']
+        inventories = ['f_inventory/hostsdir']
+        ansible = ansiblecmdb.AnsibleViaAPI(fact_dirs, inventories)
+        host_vars = ansible.hosts['db.dev.local']['hostvars']
+        groups = ansible.hosts['db.dev.local']['groups']
+        self.assertEqual(host_vars['function'], 'db')
+        self.assertIn('db', groups)
+
+    def testDynInv(self):
+        """
+        Verify that we can specify a path to a dynamic inventory as the
+        inventory file, and it will be executed, it's output parsed and added
+        as available hosts.
+        """
+        fact_dirs = ['f_inventory/out'] # Reuse f_hostparse
+        inventories = ['f_inventory/dyninv.py']
+        ansible = ansiblecmdb.AnsibleViaAPI(fact_dirs, inventories)
+        self.assertIn('host5.example.com', ansible.hosts)
+        host_vars = ansible.hosts['host5.example.com']['hostvars']
+        groups = ansible.hosts['host5.example.com']['groups']
+        self.assertEqual(host_vars['b'], False)
+        self.assertIn("atlanta", groups)
+
+    def testMixedDir(self):
+        """
+        Verify that a mixed dir of hosts files and dynamic inventory scripts is
+        parsed correctly.
+        """
+        fact_dirs = ['f_inventory/out']
+        inventories = ['f_inventory/mixeddir']
+        ansible = ansiblecmdb.AnsibleViaAPI(fact_dirs, inventories)
+        # results from dynamic inventory
+        self.assertIn("host4.example.com", ansible.hosts)
+        self.assertIn("moocow.example.com", ansible.hosts)
+        # results from normal hosts file.
+        self.assertIn("web03.dev.local", ansible.hosts)
+        # INI file ignored.
+        self.assertNotIn("ini_setting", ansible.hosts)
+
+class FactCacheTestCase(unittest.TestCase):
+    """
+    Test that we properly read fact-cached output dirs.
+    """
+    def testFactCache(self):
+        fact_dirs = ['f_factcache/out']
+        inventories = ['f_factcache/hosts']
+        ansible = ansiblecmdb.AnsibleViaAPI(fact_dirs, inventories, fact_cache=True)
+        host_vars = ansible.hosts['debian.dev.local']['hostvars']
+        groups = ansible.hosts['debian.dev.local']['groups']
+        ansible_facts = ansible.hosts['debian.dev.local']['ansible_facts']
+        self.assertIn('dev', groups)
+        self.assertEqual(host_vars['dtap'], 'dev')
+        self.assertIn('ansible_env', ansible_facts)
+
+
+if __name__ == '__main__':
+    unittest.main(exit=True)
+
+    try:
+        os.unlink('../src/ansible-cmdbc') # FIXME: Where is this coming from? Our weird import I assume.
+    except Exception:
+        pass