textfsm_mod.py

# -*- coding: utf-8 -*-
'''
TextFSM
=======

.. versionadded:: 2018.3.0

Execution module that processes plain text and extracts data
using TextFSM templates. The output is presented in JSON serializable
data, and can be easily re-used in other modules, or directly
inside the renderer (Jinja, Mako, Genshi, etc.).

:depends:   - textfsm Python library

.. note::

    For Python 2/3 compatibility, it is more recommended to
    install the ``jtextfsm`` library: ``pip install jtextfsm``.
'''
from __future__ import absolute_import

# Import python libs
import os
import logging

# Import third party modules
try:
    import textfsm
    HAS_TEXTFSM = True
except ImportError:
    HAS_TEXTFSM = False

try:
    import clitable
    HAS_CLITABLE = True
except ImportError:
    HAS_CLITABLE = False

try:
    from salt.utils.files import fopen
except ImportError:
    from salt.utils import fopen

log = logging.getLogger(__name__)

__virtualname__ = 'textfsm'
__proxyenabled__ = ['*']


def __virtual__():
    '''
    Only load this execution module if TextFSM is installed.
    '''
    if HAS_TEXTFSM:
        return __virtualname__
    return (False, 'The textfsm execution module failed to load: requires the textfsm library.')


def _clitable_to_dict(objects, fsm_handler):
    '''
    Converts TextFSM cli_table object to list of dictionaries.
    '''
    objs = []
    log.debug('Cli Table:')
    log.debug(objects)
    log.debug('FSM handler:')
    log.debug(fsm_handler)
    for row in objects:
        temp_dict = {}
        for index, element in enumerate(row):
            temp_dict[fsm_handler.header[index].lower()] = element
        objs.append(temp_dict)
    log.debug('Extraction result:')
    log.debug(objs)
    return objs


def extract(template_path, raw_text=None, raw_text_file=None, saltenv='base'):
    r'''
    Extracts the data entities from the unstructured
    raw text sent as input and returns the data
    mapping, processing using the TextFSM template.

    template_path
        The path to the TextFSM template.
        This can be specified using the absolute path
        to the file, or using one of the following URL schemes:

        - ``salt://``, to fetch the template from the Salt fileserver.
        - ``http://`` or ``https://``
        - ``ftp://``
        - ``s3://``
        - ``swift://``

    raw_text: ``None``
        The unstructured text to be parsed.

    raw_text_file: ``None``
        Text file to read, having the raw text to be parsed using the TextFSM template.
        Supports the same URL schemes as the ``template_path`` argument.

    saltenv: ``base``
        Salt fileserver envrionment from which to retrieve the file.
        Ignored if ``template_path`` is not a ``salt://`` URL.

    CLI Example:

    .. code-block:: bash

        salt '*' textfsm.extract salt://textfsm/juniper_version_template raw_text_file=s3://junos_ver.txt
        salt '*' textfsm.extract http://some-server/textfsm/juniper_version_template raw_text='Hostname: router.abc ... snip ...'

    Jinja template example:

    .. code-block:: jinja

        {%- set raw_text = 'Hostname: router.abc ... snip ...' -%}
        {%- set textfsm_extract = salt.textfsm.extract('https://some-server/textfsm/juniper_version_template', raw_text) -%}

    Raw text example:

    .. code-block:: text

        Hostname: router.abc
        Model: mx960
        JUNOS Base OS boot [9.1S3.5]
        JUNOS Base OS Software Suite [9.1S3.5]
        JUNOS Kernel Software Suite [9.1S3.5]
        JUNOS Crypto Software Suite [9.1S3.5]
        JUNOS Packet Forwarding Engine Support (M/T Common) [9.1S3.5]
        JUNOS Packet Forwarding Engine Support (MX Common) [9.1S3.5]
        JUNOS Online Documentation [9.1S3.5]
        JUNOS Routing Software Suite [9.1S3.5]

    TextFSM Example:

    .. code-block:: text

        Value Chassis (\S+)
        Value Required Model (\S+)
        Value Boot (.*)
        Value Base (.*)
        Value Kernel (.*)
        Value Crypto (.*)
        Value Documentation (.*)
        Value Routing (.*)

        Start
        # Support multiple chassis systems.
          ^\S+:$$ -> Continue.Record
          ^${Chassis}:$$
          ^Model: ${Model}
          ^JUNOS Base OS boot \[${Boot}\]
          ^JUNOS Software Release \[${Base}\]
          ^JUNOS Base OS Software Suite \[${Base}\]
          ^JUNOS Kernel Software Suite \[${Kernel}\]
          ^JUNOS Crypto Software Suite \[${Crypto}\]
          ^JUNOS Online Documentation \[${Documentation}\]
          ^JUNOS Routing Software Suite \[${Routing}\]

    Output example:

    .. code-block:: json

        {
            "comment": "",
            "result": true,
            "out": [
                {
                    "kernel": "9.1S3.5",
                    "documentation": "9.1S3.5",
                    "boot": "9.1S3.5",
                    "crypto": "9.1S3.5",
                    "chassis": "",
                    "routing": "9.1S3.5",
                    "base": "9.1S3.5",
                    "model": "mx960"
                }
            ]
        }
    '''
    ret = {
        'result': False,
        'comment': '',
        'out': None
    }
    log.debug('Using the saltenv: {}'.format(saltenv))
    log.debug('Caching {} using the Salt fileserver'.format(template_path))
    tpl_cached_path = __salt__['cp.cache_file'](template_path, saltenv=saltenv)
    if tpl_cached_path is False:
        ret['comment'] = 'Unable to read the TextFSM template from {}'.format(template_path)
        log.error(ret['comment'])
        return ret
    try:
        log.debug('Reading TextFSM template from cache path: {}'.format(tpl_cached_path))
        # Disabling pylint W8470 to nto complain about fopen.
        # Unfortunately textFSM needs the file handle rather than the content...
        # pylint: disable=W8470
        tpl_file_handle = fopen(tpl_cached_path, 'r')
        # pylint: disable=W8470
        log.debug(tpl_file_handle.read())
        tpl_file_handle.seek(0)  # move the object position back at the top of the file
        fsm_handler = textfsm.TextFSM(tpl_file_handle)
    except textfsm.TextFSMTemplateError as tfte:
        log.error('Unable to parse the TextFSM template', exc_info=True)
        ret['comment'] = 'Unable to parse the TextFSM template from {}: {}. Please check the logs.'.format(
            template_path, tfte)
        return ret
    if not raw_text and raw_text_file:
        log.debug('Trying to read the raw input from {}'.format(raw_text_file))
        raw_text = __salt__['cp.get_file_str'](raw_text_file, saltenv=saltenv)
        if raw_text is False:
            ret['comment'] = 'Unable to read from {}. Please specify a valid input file or text.'.format(raw_text_file)
            log.error(ret['comment'])
            return ret
    if not raw_text:
        ret['comment'] = 'Please specify a valid input file or text.'
        log.error(ret['comment'])
        return ret
    log.debug('Processing the raw text:')
    log.debug(raw_text)
    objects = fsm_handler.ParseText(raw_text)
    ret['out'] = _clitable_to_dict(objects, fsm_handler)
    ret['result'] = True
    return ret


def index(command,
          platform=None,
          platform_grain_name=None,
          platform_column_name=None,
          output=None,
          output_file=None,
          textfsm_path=None,
          index_file=None,
          saltenv='base',
          include_empty=False,
          include_pat=None,
          exclude_pat=None):
    '''
    Dynamically identify the template required to extract the
    information from the unstructured raw text.

    The output has the same structure as the ``extract`` execution
    function, the difference being that ``index`` is capable
    to identify what template to use, based on the platform
    details and the ``command``.

    command
        The command executed on the device, to get the output.

    platform
        The platform name, as defined in the TextFSM index file.

        .. note::
            For ease of use, it is recommended to define the TextFSM
            indexfile with values that can be matches using the grains.

    platform_grain_name
        The name of the grain used to identify the platform name
        in the TextFSM index file.

        .. note::
            This option can be also specified in the minion configuration
            file or pillar as ``textfsm_platform_grain``.

        .. note::
            This option is ignored when ``platform`` is specified.

    platform_column_name: ``Platform``
        The column name used to identify the platform,
        exactly as specified in the TextFSM index file.
        Default: ``Platform``.

        .. note::
            This is field is case sensitive, make sure
            to assign the correct value to this option,
            exactly as defined in the index file.

        .. note::
            This option can be also specified in the minion configuration
            file or pillar as ``textfsm_platform_column_name``.

    output
        The raw output from the device, to be parsed
        and extract the structured data.

    output_file
        The path to a file that contains the raw output from the device,
        used to extract the structured data.
        This option supports the usual Salt-specific schemes: ``file://``,
        ``salt://``, ``http://``, ``https://``, ``ftp://``, ``s3://``, ``swift://``.

    textfsm_path
        The path where the TextFSM templates can be found. This can be either
        absolute path on the server, either specified using the following URL
        schemes: ``file://``, ``salt://``, ``http://``, ``https://``, ``ftp://``,
        ``s3://``, ``swift://``.

        .. note::
            This needs to be a directory with a flat structure, having an
            index file (whose name can be specified using the ``index_file`` option)
            and a number of TextFSM templates.

        .. note::
            This option can be also specified in the minion configuration
            file or pillar as ``textfsm_path``.

    index_file: ``index``
        The name of the TextFSM index file, under the ``textfsm_path``. Default: ``index``.

        .. note::
            This option can be also specified in the minion configuration
            file or pillar as ``textfsm_index_file``.

    saltenv: ``base``
        Salt fileserver envrionment from which to retrieve the file.
        Ignored if ``textfsm_path`` is not a ``salt://`` URL.

    include_empty: ``False``
        Include empty files under the ``textfsm_path``.

    include_pat
        Glob or regex to narrow down the files cached from the given path.
        If matching with a regex, the regex must be prefixed with ``E@``,
        otherwise the expression will be interpreted as a glob.

    exclude_pat
        Glob or regex to exclude certain files from being cached from the given path.
        If matching with a regex, the regex must be prefixed with ``E@``,
        otherwise the expression will be interpreted as a glob.

        .. note::
            If used with ``include_pat``, files matching this pattern will be
            excluded from the subset of files defined by ``include_pat``.

    CLI Example:

    .. code-block:: bash

        salt '*' textfsm.index 'sh ver' platform=Juniper output_file=salt://textfsm/juniper_version_example textfsm_path=salt://textfsm/
        salt '*' textfsm.index 'sh ver' output_file=salt://textfsm/juniper_version_example textfsm_path=ftp://textfsm/ platform_column_name=Vendor
        salt '*' textfsm.index 'sh ver' output_file=salt://textfsm/juniper_version_example textfsm_path=https://some-server/textfsm/ platform_column_name=Vendor platform_grain_name=vendor

    TextFSM index file example:

    ``salt://textfsm/index``

    .. code-block:: text

        Template, Hostname, Vendor, Command
        juniper_version_template, .*, Juniper, sh[[ow]] ve[[rsion]]

    The usage can be simplified,
    by defining (some of) the following options: ``textfsm_platform_grain``,
    ``textfsm_path``, ``textfsm_platform_column_name``, or ``textfsm_index_file``,
    in the (proxy) minion configuration file or pillar.

    Configuration example:

    .. code-block:: yaml

        textfsm_platform_grain: vendor
        textfsm_path: salt://textfsm/
        textfsm_platform_column_name: Vendor

    And the CLI usage becomes as simple as:

    .. code-block:: bash

        salt '*' textfsm.index 'sh ver' output_file=salt://textfsm/juniper_version_example

    Usgae inside a Jinja template:

    .. code-block:: jinja

        {%- set command = 'sh ver' -%}
        {%- set output = salt.net.cli(command) -%}
        {%- set textfsm_extract = salt.textfsm.index(command, output=output) -%}
    '''
    ret = {
        'out': None,
        'result': False,
        'comment': ''
    }
    if not HAS_CLITABLE:
        ret['comment'] = 'TextFSM doesnt seem that has clitable embedded.'
        log.error(ret['comment'])
        return ret
    if not platform:
        platform_grain_name = __opts__.get('textfsm_platform_grain') or\
                              __pillar__.get('textfsm_platform_grain', platform_grain_name)
        if platform_grain_name:
            log.debug('Using the {} grain to identify the platform name'.format(platform_grain_name))
            platform = __grains__.get(platform_grain_name)
            if not platform:
                ret['comment'] = 'Unable to identify the platform name using the {} grain.'.format(platform_grain_name)
                return ret
            log.info('Using platform: {}'.format(platform))
        else:
            ret['comment'] = 'No platform specified, no platform grain identifier configured.'
            log.error(ret['comment'])
            return ret
    if not textfsm_path:
        log.debug('No TextFSM templates path specified, trying to look into the opts and pillar')
        textfsm_path = __opts__.get('textfsm_path') or __pillar__.get('textfsm_path')
        if not textfsm_path:
            ret['comment'] = 'No TextFSM templates path specified. Please configure in opts/pillar/function args.'
            log.error(ret['comment'])
            return ret
    log.debug('Using the saltenv: {}'.format(saltenv))
    log.debug('Caching {} using the Salt fileserver'.format(textfsm_path))
    textfsm_cachedir_ret = __salt__['cp.cache_dir'](textfsm_path,
                                                    saltenv=saltenv,
                                                    include_empty=include_empty,
                                                    include_pat=include_pat,
                                                    exclude_pat=exclude_pat)
    log.debug('Cache fun return:')
    log.debug(textfsm_cachedir_ret)
    if not textfsm_cachedir_ret:
        ret['comment'] = 'Unable to fetch from {}. Is the TextFSM path correctly specified?'.format(textfsm_path)
        log.error(ret['comment'])
        return ret
    textfsm_cachedir = os.path.dirname(textfsm_cachedir_ret[0])  # first item
    index_file = __opts__.get('textfsm_index_file') or __pillar__.get('textfsm_index_file', 'index')
    index_file_path = os.path.join(textfsm_cachedir, index_file)
    log.debug('Using the cached index file: {}'.format(index_file_path))
    log.debug('TextFSM templates cached under: {}'.format(textfsm_cachedir))
    textfsm_obj = clitable.CliTable(index_file_path, textfsm_cachedir)
    attrs = {
        'Command': command
    }
    platform_column_name = __opts__.get('textfsm_platform_column_name') or\
                           __pillar__.get('textfsm_platform_column_name', 'Platform')
    log.info('Using the TextFSM platform idenfiticator: {}'.format(platform_column_name))
    attrs[platform_column_name] = platform
    log.debug('Processing the TextFSM index file using the attributes: {}'.format(attrs))
    if not output and output_file:
        log.debug('Processing the output from {}'.format(output_file))
        output = __salt__['cp.get_file_str'](output_file, saltenv=saltenv)
        if output is False:
            ret['comment'] = 'Unable to read from {}. Please specify a valid file or text.'.format(output_file)
            log.error(ret['comment'])
            return ret
    if not output:
        ret['comment'] = 'Please specify a valid output text or file'
        log.error(ret['comment'])
        return ret
    log.debug('Processing the raw text:')
    log.debug(output)
    try:
        # Parse output through template
        textfsm_obj.ParseCmd(output, attrs)
        ret['out'] = _clitable_to_dict(textfsm_obj, textfsm_obj)
        ret['result'] = True
    except clitable.CliTableError as cterr:
        log.error('Unable to proces the CliTable', exc_info=True)
        ret['comment'] = 'Unable to process the output: {}'.format(cterr)
    return ret