Merge pull request #1563 from dmulyalin/develop

Add ttp_parse helper function to codebase

mypy.ini

@@ -47,5 +47,8 @@ ignore_missing_imports = True
 [mypy-textfsm]
 ignore_missing_imports = True
 
+[mypy-ttp]
+ignore_missing_imports = True
+
 [mypy-pytest]
 ignore_missing_imports = True

napalm/base/helpers.py

@@ -18,6 +18,13 @@
 from netaddr import mac_unix
 from netutils.config.parser import IOSConfigParser
 
+try:
+    from ttp import quick_parse as ttp_quick_parse
+
+    TTP_INSTALLED = True
+except ImportError:
+    TTP_INSTALLED = False
+
 # local modules
 import napalm.base.exceptions
 from napalm.base import constants
@@ -311,6 +318,82 @@ def textfsm_extractor(
     )
 
 
+def ttp_parse(
+    cls: "napalm.base.NetworkDriver",
+    template: str,
+    raw_text: str,
+    structure: str = "flat_list",
+) -> Union[None, List, Dict]:
+    """
+    Applies a TTP template over a raw text and return the parsing results.
+
+    Main usage of this method will be to extract data form a non-structured output
+    from a network device and return parsed values.
+
+    :param cls: Instance of the driver class
+    :param template: Specifies the name or the content of the template to be used
+    :param raw_text: Text output as the devices prompts on the CLI
+    :param structure: Results structure to apply to parsing results
+    :return: parsing results structure
+
+    ``template`` can be inline TTP template string, reference to TTP Templates
+    repository template in a form of ``ttp://path/to/template`` or name of template
+    file within ``{NAPALM_install_dir}/utils/ttp_templates/{template}.txt`` folder.
+    """
+    if not TTP_INSTALLED:
+        msg = "\nTTP is not installed. Please PIP install ttp:\n" "pip install ttp\n"
+        raise napalm.base.exceptions.ModuleImportError(msg)
+
+    result = None
+
+    for c in cls.__class__.mro():
+        if c is object:
+            continue
+        module = sys.modules[c.__module__].__file__
+        if module:
+            current_dir = os.path.dirname(os.path.abspath(module))
+        else:
+            continue
+        template_dir_path = "{current_dir}/utils/ttp_templates".format(
+            current_dir=current_dir
+        )
+
+        # check if inline template given, use it as is
+        if "{{" in template and "}}" in template:
+            template = template
+        # check if template from ttp_templates repo, use it as is
+        elif template.startswith("ttp://"):
+            template = template
+        # default to using template in NAPALM folder
+        else:
+            template = "{template_dir_path}/{template}.txt".format(
+                template_dir_path=template_dir_path, template=template
+            )
+            if not os.path.exists(template):
+                msg = "Template '{template}' not found".format(template=template)
+                logging.error(msg)
+                raise napalm.base.exceptions.TemplateRenderException(msg)
+
+        # parse data
+        try:
+            result = ttp_quick_parse(
+                data=str(raw_text),
+                template=template,
+                result_kwargs={"structure": structure},
+                parse_kwargs={"one": True},
+            )
+            break
+        except Exception as e:
+            msg = "TTP template:\n'{template}'\nError: {error}".format(
+                template=template, error=e
+            )
+            logging.exception(e)
+            logging.error(msg)
+            raise napalm.base.exceptions.TemplateRenderException(msg)
+
+    return result
+
+
 def find_txt(
     xml_tree: etree._Element,
     path: str,

requirements-dev.txt

@@ -13,4 +13,6 @@ mypy==0.931
 types-requests==2.27.9
 types-six==1.16.10
 types-setuptools==57.4.9
-types-PyYAML==6.0.4
+types-PyYAML==6.0.4
+ttp==0.8.4
+ttp_templates==0.1.3

requirements.txt

@@ -13,4 +13,6 @@ junos-eznc>=2.2.1
 scp
 lxml>=4.3.0
 ncclient
+ttp
+ttp_templates
 netutils>=1.0.0

test/base/test_helpers.py

@@ -37,6 +37,20 @@
 except ImportError:
     HAS_NETADDR = False
 
+try:
+    from ttp import ttp  # noqa
+
+    HAS_TTP = True
+except ImportError:
+    HAS_TTP = False
+
+try:
+    from ttp_templates import get_template  # noqa
+
+    HAS_TTP_TEMPLATES = True
+except ImportError:
+    HAS_TTP_TEMPLATES = False
+
 # NAPALM base
 import napalm.base.helpers
 import napalm.base.constants as C
@@ -673,6 +687,74 @@ def test_sanitized_config(self):
         ret = napalm.base.helpers.sanitize_config(config, C.CISCO_SANITIZE_FILTERS)
         self.assertEqual(ret, expected)
 
+    def test_ttp_parse(self):
+        """
+        Tests helper function ```ttp_parse```:
+
+            * check if raises TemplateRenderException when template does not exist
+            * check if raises TemplateRenderException for /utils/ttp_templates/bad_template.txt
+            * check if returns expected result as output for ./utils/ttp_templates/good_template.txt
+            * check if returns expected result as output for inline template
+            * check if returns expected result as output for template from ttp templates
+        """
+
+        self.assertTrue(
+            HAS_TTP, "Install TTP: python3 -m pip install ttp"
+        )  # before anything else, let's see if TTP is available
+        _TTP_TEST_STRING = """
+interface Gi1/1
+ description FOO
+!
+interface Gi1/2
+ description BAR
+!
+        """
+        _TTP_TEST_TEMPLATE = '<group>\ninterface {{ interface }}\n description {{ description | re(".+") }}\n! {{ _end_ }}\n</group>'  # noqa:E501
+        _EXPECTED_RESULT = [
+            {"description": "FOO", "interface": "Gi1/1"},
+            {"description": "BAR", "interface": "Gi1/2"},
+        ]
+        self.assertRaises(
+            napalm.base.exceptions.TemplateRenderException,
+            napalm.base.helpers.ttp_parse,
+            self.network_driver,
+            "__this_template_does_not_exist__",
+            _TTP_TEST_STRING,
+        )
+
+        self.assertRaises(
+            napalm.base.exceptions.TemplateRenderException,
+            napalm.base.helpers.ttp_parse,
+            self.network_driver,
+            "bad_template",
+            _TTP_TEST_STRING,
+        )
+
+        # use ./utils/ttp_templates/good_template.txt
+        result = napalm.base.helpers.ttp_parse(
+            self.network_driver, "good_template", _TTP_TEST_STRING
+        )
+        self.assertEqual(result, _EXPECTED_RESULT)
+
+        # use inline template
+        result = napalm.base.helpers.ttp_parse(
+            self.network_driver, _TTP_TEST_TEMPLATE, _TTP_TEST_STRING
+        )
+        self.assertEqual(result, _EXPECTED_RESULT)
+
+        self.assertTrue(
+            HAS_TTP_TEMPLATES,
+            "Install TTP Templates: python3 -m pip install ttp-templates",
+        )  # before anything else, let's see if TTP_Templates is available
+
+        # use template from ttp templates package
+        result = napalm.base.helpers.ttp_parse(
+            self.network_driver,
+            "ttp://platform/test_platform_show_run_pipe_sec_interface.txt",
+            _TTP_TEST_STRING,
+        )
+        self.assertEqual(result, _EXPECTED_RESULT)
+
 
 class FakeNetworkDriver(NetworkDriver):
     def __init__(self):

test/base/utils/ttp_templates/bad_template.txt

@@ -0,0 +1,5 @@
+<group
+interface {{ interface }}
+ description {{ description | re(".+") }}
+! {{ _end_ }}
+</group>

test/base/utils/ttp_templates/good_template.txt

@@ -0,0 +1,5 @@
+<group>
+interface {{ interface }}
+ description {{ description | re(".+") }}
+! {{ _end_ }}
+</group>