Merge pull request #249 from tcuthbert/textfsm

Add Textfsm Feature

.gitignore

@@ -53,3 +53,7 @@ _build
 
 # PyCharm
 .idea
+
+# 3rd party
+vendor/*
+.python-version

conf/trigger_settings.py

@@ -279,6 +279,9 @@
 # Assign NETDEVICES_SOURCE to NETDEVICES_FILE for backwards compatibility
 NETDEVICES_FILE = NETDEVICES_SOURCE
 
+# TextFSM Template Path. Commando will attempt to match a given show command with a template within this folder.
+TEXTFSM_TEMPLATE_DIR = os.getenv('TEXTFSM_TEMPLATE_DIR', os.path.join(PREFIX, 'vendor/ntc_templates'))
+
 # Whether to treat the RANCID root as a normal instance, or as the root to
 # multiple instances. This is only checked when using RANCID as a data source.
 RANCID_RECURSE_SUBDIRS = os.environ.get('RANCID_RECURSE_SUBDIRS', False)

docs/changelog.rst

@@ -10,6 +10,7 @@ Changelog
 Enhancements
 ------------
 
++ Added TextFSM parser to process unstructured CLI output.
 + Added a new prompt pattern to ``settings.CONTINUE_PROMPTS``.
 + New continue prompts no longer need to be lower-cased.
 + Clarified the error text when an enable password is required but not provided

docs/usage/configuration.rst

@@ -165,6 +165,19 @@ Default::
 
     'aol'
 
+.. setting:: TEXTFSM_TEMPLATE_DIR
+
+TEXTFSM_TEMPLATE_DIR
+~~~~~~~~~~~~~~~~~~~~
+
+Default path to TextFSM template directory. It is recommended to pull the Network to Code templates
+from here `Network to Code templates <github.com/networktocode/ntc-ansible/tree/master/ntc_templates>` and place them inside
+the vendor directory inside the trigger root.
+
+Default::
+
+    '/etc/trigger/vendor/ntc_templates'
+
 .. setting:: FIREWALL_DIR
 
 FIREWALL_DIR

setup.py

@@ -20,6 +20,7 @@
     'pyparsing==1.5.7',
     'pytz',
     'SimpleParse',
+    'gtextfsm',
     'redis', # The python interface, not the daemon!
 ]
 

tests/test_templates.py

@@ -0,0 +1,59 @@
+import unittest
+import os
+import mock
+from trigger.utils.templates import *
+from trigger.conf import settings
+from contextlib import contextmanager
+from StringIO import StringIO
+import cStringIO
+
+
+try:
+    import textfsm
+except ImportError:
+    print("""
+    Woops, looks like you're missing the textfsm library.
+
+    Try installing it like this::
+
+        >>> pip install gtextfsm
+    """)
+
+
+cli_data = """*02:00:42.743 UTC Sat Feb 20 2016"""
+
+text_fsm_data = """Value TIME (\d+:\d+:\d+\.\d+)
+Value TIMEZONE (\w+)
+Value DAYWEEK (\w+)
+Value MONTH (\w+)
+Value DAY (\d+)
+Value YEAR (\d+)
+
+Start
+  ^[\*]?${TIME}\s${TIMEZONE}\s${DAYWEEK}\s${MONTH}\s${DAY}\s${YEAR} -> Record
+"""
+
+class CheckTemplates(unittest.TestCase):
+    """Test structured CLI object data."""
+
+    def setUp(self):
+        data = cStringIO.StringIO(text_fsm_data)
+        self.re_table = textfsm.TextFSM(data)
+        self.assertTrue(isinstance(self.re_table, textfsm.textfsm.TextFSM))
+
+    def testTemplatePath(self):
+        """Test that template path is correct."""
+        t_path = get_template_path("show clock", dev_type="cisco_ios")
+        self.failUnless("vendor/ntc_templates/cisco_ios_show_clock.template" in t_path)
+
+    def testGetTextFsmObject(self):
+        """Test that we get structured data back from cli output."""
+        data = get_textfsm_object(self.re_table, cli_data)
+        self.assertTrue(isinstance(data, dict))
+        keys = ['dayweek', 'time', 'timezone', 'year', 'day', 'month']
+        for key in keys:
+            self.assertTrue(data.has_key(key))
+
+
+if __name__ == "__main__":
+    unittest.main()

trigger/cmds.py

@@ -28,6 +28,7 @@
 from twisted.internet import defer, task
 
 from trigger.netdevices import NetDevices
+from trigger.utils.templates import load_cmd_template, get_textfsm_object, get_template_path
 from trigger.conf import settings
 from trigger import exceptions
 
@@ -133,6 +134,9 @@ class Commando(object):
     # How results are stored (defaults to {})
     results = None
 
+    # How parsed results are stored (defaults to {})
+    parsed_results = None
+
     # How errors are stored (defaults to {})
     errors = None
 
@@ -162,6 +166,7 @@ def __init__(self, devices=None, commands=None, creds=None,
         # Always fallback to {} for these
         self.errors = self.errors if self.errors is not None else {}
         self.results = self.results if self.results is not None else {}
+        self.parsed_results = self.parsed_results if self.parsed_results is not None else {}
 
         #self.deferrals = []
         self.supported_platforms = self._validate_platforms()
@@ -278,7 +283,10 @@ def _add_worker(self):
                                    force_cli=self.force_cli,
                                    command_interval=self.command_interval)
 
-            # Add the parser callback for great justice!
+            # Add the template parser callback for great justice!
+            async.addCallback(self.parse_template, device, commands)
+
+            # Add the parser callback for even greater justice!
             async.addCallback(self.parse, device, commands)
 
             # If parse fails, still decrement and track the error
@@ -401,6 +409,44 @@ class constructor.
         func = self._lookup_method(device, method='generate')
         return func(device, commands, extra)
 
+    def parse_template(self, results, device, commands=None):
+        """
+        Generator function that processes unstructured CLI data and yields either
+        a TextFSM based object or generic raw output.
+
+        :param results:
+            The unstructured "raw" CLI data from device.
+        :type  results:
+            str
+        :param device:
+            NetDevice object
+        :type device:
+            `~trigger.netdevices.NetDevice`
+        """
+
+        device_type = ""
+        vendor_mapping = {
+                "cisco": "cisco_ios",
+                "cisco_nexus": "cisco_nexus",
+                "arista": "arista_eos"
+                }
+        if device.model.lower() == 'nexus':
+            device_type = "cisco_nxos"
+        else:
+            try:
+                device_type = vendor_mapping[device.vendor]
+            except:
+                log.msg("Unable to find template for given device")
+
+        for idx, command in enumerate(commands):
+            try:
+                re_table = load_cmd_template(command, dev_type=device_type)
+                fsm = get_textfsm_object(re_table, results[idx])
+                self.append_parsed_results(device, self.map_parsed_results(command, fsm))
+            except:
+                log.msg("Unable to load TextFSM template, updating with unstructured output")
+            yield results[idx]
+
     def parse(self, results, device, commands=None):
         """
         Parse output from a device. Calls to ``self._lookup_method`` to find
@@ -464,6 +510,25 @@ def store_error(self, device, error):
         self.errors[devname] = error
         return True
 
+    def append_parsed_results(self, device, results):
+        """
+        A simple method for appending results called by template parser
+        method.
+
+        If you want to customize the default method for storing parsed
+        results, overload this in your subclass.
+
+        :param device:
+            A `~trigger.netdevices.NetDevice` object
+
+        :param results:
+            The results to store. Anything you want really.
+        """
+        devname = str(device)
+        log.msg("Appending results for %r: %r" % (devname, results))
+        self.parsed_results[devname] = results
+        return True
+
     def store_results(self, device, results):
         """
         A simple method for storing results called by all default
@@ -483,6 +548,13 @@ def store_results(self, device, results):
         self.results[devname] = results
         return True
 
+    def map_parsed_results(self, command=None, fsm=None):
+        """Return a dict of ``{command: fsm, ...}``"""
+        if fsm is None:
+            fsm = {}
+
+        return {command: fsm}
+
     def map_results(self, commands=None, results=None):
         """Return a dict of ``{command: result, ...}``"""
         if commands is None:
@@ -648,6 +720,7 @@ def monitor_result(self, result, reactor):
         return task.deferLater(reactor, 0.5, self.monitor_result, result, reactor)
 
 
+
 class NetACLInfo(Commando):
     """
     Class to fetch and parse interface information. Exposes a config

trigger/conf/global_settings.py

@@ -294,6 +294,9 @@
 # Assign NETDEVICES_SOURCE to NETDEVICES_FILE for backwards compatibility
 NETDEVICES_FILE = NETDEVICES_SOURCE
 
+# TextFSM Template Path. Commando will attempt to match a given show command with a template within this folder.
+TEXTFSM_TEMPLATE_DIR = os.getenv('TEXTFSM_TEMPLATE_DIR', os.path.join(PREFIX, 'vendor/ntc_templates'))
+
 # Whether to treat the RANCID root as a normal instance, or as the root to
 # multiple instances. This is only checked when using RANCID as a data source.
 RANCID_RECURSE_SUBDIRS = os.environ.get('RANCID_RECURSE_SUBDIRS', False)

trigger/utils/templates.py

@@ -0,0 +1,76 @@
+#coding=utf-8
+
+"""
+Templating functions for unstructured CLI output.
+"""
+
+__author__ = 'Thomas Cuthbert'
+__maintainer__ = 'Thomas Cuthbert'
+__email__ = 'tcuthbert90@gmail.com'
+__copyright__ = 'Copyright 2016 Trigger Org'
+
+
+import sys
+import os
+from trigger.conf import settings
+from twisted.python import log
+
+try:
+    import textfsm
+except ImportError:
+    print("""
+    Woops, looks like you're missing the textfsm library.
+
+    Try installing it like this::
+
+        >>> pip install gtextfsm
+    """)
+
+
+# Exports
+__all__ = ('get_template_path', 'load_cmd_template', 'get_textfsm_object')
+
+
+def get_template_path(cmd, dev_type=None):
+    """
+    Return textfsm templates from the directory pointed to by the TEXTFSM_TEMPLATE_DIR trigger variable.
+
+    :param dev_type: Type of device ie cisco_ios, arista_eos
+    :type  dev_type: str
+    :param cmd: CLI command to load template.
+    :type  cmd: str
+    :returns: String template path
+    """
+    t_dir = settings.TEXTFSM_TEMPLATE_DIR
+    return os.path.join(t_dir, '{1}_{2}.template'.format(t_dir, dev_type, cmd.replace(' ', '_'))) or None
+
+
+def load_cmd_template(cmd, dev_type=None):
+    """
+    :param dev_type: Type of device ie cisco_ios, arista_eos
+    :type  dev_type: str
+    :param cmd: CLI command to load template.
+    :type  cmd: str
+    :returns: String template path
+    """
+    try:
+        with open(get_template_path(cmd, dev_type=dev_type), 'rb') as f:
+            return textfsm.TextFSM(f)
+    except:
+        log.msg("Unable to load template:\n{0} :: {1}".format(cmd, dev_type))
+
+
+def get_textfsm_object(re_table, cli_output):
+    "Returns structure object from TextFSM data."
+    from collections import defaultdict
+    rv = defaultdict(list)
+    keys = re_table.header
+    values = re_table.ParseText(cli_output)
+    l = []
+    for item in values:
+        l.extend(zip(map(lambda x: x.lower(), keys), item))
+
+    for k, v in l:
+        rv[k].append(v)
+
+    return dict(rv)