diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..8d0cc70f --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,177 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/napalm.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/napalm.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/napalm" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/napalm" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..a69c5c31 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,274 @@ +"""conf.py.""" +# -*- coding: utf-8 -*- +# +# napalm documentation build configuration file, created by +# sphinx-quickstart on Tue Dec 16 13:17:14 2014. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. +# +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('../')) + +import os + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +autoclass_content = 'both' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'napalm-yang' +copyright = u'2017, David Barroso' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0' +# The full version, including alpha/beta/rc tags. +release = '1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +else: + html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'napalmdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'napalm.tex', u'NAPALM Documentation', + u'David Barroso', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'napalm', u'NAPALM Documentation', + [u'David Barroso'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'napalm-yang', u'NAPALM YANG Documentation', + u'David Barroso', 'napalm-yang', 'YANG framework for any device', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + +# Disable pdf and epub generation +enable_pdf_build = False +enable_epub_build = False diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..0245be3c --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,31 @@ +napalm-yang +=========== + +`YANG (RFC6020) `_ is a data modelling language, it's a way of defining how data is supposed to look like. The `napalm-yang `_ library provides a framework to use models defined with YANG in the context of network management. It provides mechanisms to transform native data/config into YANG and vice versa. + +You can take a look to the following `tutorial `_ to see what this is about and how to get started. + +Installation +------------ + +To install ``napalm-yang`` you can use pip as with any other driver:: + + pip install -U napalm-yang + +Documentation +------------- + +.. toctree:: + :maxdepth: 1 + + yang/profiles + yang/basics + yang/writing_profiles + yang/parsers + yang/parsers/XMLParser + yang/parsers/TextParser + yang/translators + yang/translators/XMLTranslator + yang/translators/TextTranslator + yang/api + yang/jinja_filters diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..0635cc88 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,6 @@ +sphinx +sphinx-rtd-theme +sphinxcontrib-napoleon +invoke +napalm +napalm_yang diff --git a/docs/yang/api.rst b/docs/yang/api.rst new file mode 100644 index 00000000..2d599518 --- /dev/null +++ b/docs/yang/api.rst @@ -0,0 +1,23 @@ +API +=== + +Models +------ + +Models are generated by ``pyangbind`` so it's better to check it's documentation for up to date +information: http://pynms.io/pyangbind/generic_methods/ + +Utils +----- + +.. autofunction:: napalm_yang.utils.model_to_dict + +.. autofunction:: napalm_yang.utils.diff + +Root +---- + +.. autoclass:: napalm_yang.base.Root + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/yang/basics.rst b/docs/yang/basics.rst new file mode 100644 index 00000000..349e92a1 --- /dev/null +++ b/docs/yang/basics.rst @@ -0,0 +1,53 @@ +YANG Basics +----------- + +It's not really necessary to understand how YANG works to write a profile but you need some basic +understanding. + +Basic Types +___________ + +* **container** - A container is just a placeholder, sort of like a map or dictionary. A container + doesn't store any information per se, instead, it contains attributes of any type. For example, + the following ``config`` object would be a valid container with three attributes of various types:: + + container config: + leaf description: string + leaf mtu: uint16 + leaf enabled: boolean + +* **leaf** - A leaf is an attribute that stores information. Leafs are of a type and values have to + be valid for the given type. For example:: + + leaf descrpition: string # Any string is valid + leaf mtu: uint16 # -1 is not valid but 1500 is + leaf enabled: boolean # true, false, 1, 0, True, False are valid + +.. note:: + There can be further restrictions, for example the leaf ``prefix-length`` is of type ``uint8`` but + it's further restricted with the option ``range 0..32`` + +* **YANG lists** - A YANG list represents a container in the tree that will represent individual + members of a list. For example:: + + container interfaces: + list interface: + container config: + leaf description: string + leaf mtu: uint16 + leaf enabled: boolean + +As we start adding elements to the interface list, each individual interface will have it's own +attributes. For example:: + + interfaces: + interface["eth1"]: + config: + description: "An interface" + mtu: 1500 + enabled: true + interface["eth2"]: + config: + description: "Another interface" + mtu: 9000 + enabled: false diff --git a/docs/yang/jinja_filters.rst b/docs/yang/jinja_filters.rst new file mode 100644 index 00000000..2699a909 --- /dev/null +++ b/docs/yang/jinja_filters.rst @@ -0,0 +1,8 @@ +Jinja2 Filters +============== + +IP address +---------- + +.. automodule:: napalm_yang.jinja_filters.ip_filters + :members: diff --git a/docs/yang/parsers.rst b/docs/yang/parsers.rst new file mode 100644 index 00000000..955ea16e --- /dev/null +++ b/docs/yang/parsers.rst @@ -0,0 +1,150 @@ +Parsers +^^^^^^^ + +Parsers are responsible for mapping native configuration/show_commands to a YANG model. + +Special fields +============== + +When parsing attributes, some fields may depend on the parser you are using but some +will be available regardless. Some may be even be mandatory. + +mode +---- + +* **Mandatory**: Yes +* **Description**: Which parsing/translation action to use for this particular field. +* **Example**: Parse the description field with a simple regular expression:: + + _process: + mode: search + regexp: "description (?P.*)" + from: "{{ bookmarks.interface[interface_key] }}" + +when +---- + +* **Mandatory**: No +* **Description**: The evaluation of this field will determine if the action is executed or + skipped. This action is probably not very useful when parsing but it's available if you need it. +* **Example**: Configure ``switchport`` on IOS devices only if the interface is not a Loopback + or a Management interface:: + + ipv4: + _process: unnecessary + config: + _process: unnecessary + enabled: + _process: + - mode: element + value: " no switchport\n" + negate: " switchport\n" + in: "interface.{{ interface_key }}" + when: "{{ model and interface_key[0:4] not in ['Mana', 'Loop'] }}" + +from +---- + +* **Mandatory**: Yes +* **Description**: Configuration to read. In combination with ``bookmarks`` provides the content we + are operating with. +* **Example**: Get IP addresses from both both interfaces and subinterfaces:: + + address: + _process: + mode: xpath + xpath: "family/inet/address" + key: name + from: "{{ bookmarks['parent'] }}" + +Special Variables +================= + +.. _yang_special_field_keys: + +keys +---- + +When traversing lists, you will have all the relevant keys for the object available, including on nested +lists. Let's see it with an example, let's say we are currently parsing +``interfaces/interface["et1"]/subinterfaces/subinterface["0"].ipv4.addresses.address["10.0.0.1"]``. +At this particular point you will have the following keys available: + +* **address_key** - ``10.0.0.1`` +* **subinterface_key** - ``0`` +* **interface_key** - ``et1`` +* **parent_key** - ``0`` + +When a list is traversed you will always have available a key with name ``$(attribute)_key``. In +addition, you will have ``parent_key`` as the key of the immediate parent object. In the example +above, ``parent_key`` will correspond to ``0`` as it's the immediate parent of the address object. + +.. _yang_special_field_bookmarks: + +bookmarks +--------- + +Bookmarks are points of interest in the configuration. Usually, you will be gathering blocks of +configurations and parsing on those but sometimes, the configuration you need might be somewhere +else. For those cases, you will be able to access those with the bookmarks. Using the same example +as before, +``interfaces/interface["et1"]/subinterfaces/subinterface["0"].ipv4.addresses.address["10.0.0.1"]``, +you will have the following bookmarks: + +* ``bookmarks.interfaces`` - The root of the configuration +* ``bookmarks.interface["et1"]`` - The block of configuration that corresponds to the interface + ``et1`` +* ``bookmarks.subinterface["0"]`` - The block of configuration that corresponds to the subinterface + ``0`` of ``et1``. +* ``bookmarks.address["10.0.0.1"]`` - The block of configuration for the address belonging to the + subinterface. +* ``bookmarks.parent`` - The block of configuration for the immediate parent, in this case, the + subinterface ``0``. + +Note you can use keys instead and do ``bookmarks.subinterface[parent_key]`` or +``bookmarks.subinterface[subinterface_key]``. + +extra_vars +---------- + +Some actions let's you provide additional information for later use. Those will be stored on the +``extra_vars`` dictionary. For example:: + + address: + _process: + mode: block + regexp: "(?Pip address (?P(?P.*))\\/(?P\\d+))(?P secondary)*" + from: "{{ bookmarks['parent'] }}" + config: + _process: unnecessary + ip: + _process: + mode: value + value: "{{ extra_vars.ip }}" + +The first regexp captures a bunch of vars that later can be used by just reading them from +``extra_Vars``. + + +Metadata +========= + +The metadata tells the profile how to process that module and how to get the necessary data from +the device. For example:: + + --- + metadata: + parser: XMLParser + execute: + - method: _rpc + args: + get: "" + +* **execute** is a list of calls to do to from the device to extract the data. + + * **method** is the method from the device to call. + * **args** are arguments that will be passed to the method. + +In addition, some methods like ``parse_config`` and ``parse_state`` may have mechanisms to pass the +information needed to the parser instead of relying on a live device to obtain it. For parsers, you +will just have to pass a string with the same information the profile is trying to gather. diff --git a/docs/yang/parsers/TextParser.rst b/docs/yang/parsers/TextParser.rst new file mode 100644 index 00000000..743d9587 --- /dev/null +++ b/docs/yang/parsers/TextParser.rst @@ -0,0 +1,184 @@ +TextParser +========== + +Will apply regular expressions to text to extract data from it. + +To explain how this parser works, let's use the following configuration:: + + interface Ethernet1 + no switchport + ! + interface Ethernet1.1 + description blah + ! + interface Loopback1 + no switchport + ip address 192.168.0.1/24 + ip address 192.168.1.1/24 secondary + ! + +.. note:: The regular expressions on this parser have the ``MULTILINE`` and ``IGNORECASE`` flags turned on. + +List - block +------------ + +Using a regular expression it divides the configuration in blocks where each block is relevant for +a different element of the list. + +Arguments: + + * **regexp** (mandatory) - Regular expression to apply. Note that it must capture two things at least; + ``block``, which will be the entire block of configuration relevant for the interface and + ``key``, which will be the key of the element. + + +Example 1 + + Capture the interfaces:: + + _process: + mode: block + regexp: "(?Pinterface (?P(\\w|-)*\\d+)\n(?:.|\n)*?^!$)" + from: "{{ bookmarks.interfaces }}" + + So the regexp is basically doing two things. Capturing each block of text that starts with + ``interface (a word)(a number)\n`` (no dots allowed as a dot means it's subinterface) and then + finishing in ``!``. It's also getting the ``key``. So after this step we will have a list like:: + + - key: Ethernet1 + block: interface Ethernet1 + no switchport + ! + - key: Loopback1 + block: interface Loopback1 + no switchport + ip address 192.168.0.1/24 + ip address 192.168.1.1/24 secondary + ! + + Note that ``Ethernet1.1`` is missing as it's not matching the key. + +Example 2 + + As we process ``Ethernet1`` we will want it's subinterfaces so we can use a similar regexp as + before but looking for a ``dot`` in the key, using the ``interface_key (Ethernet1)`` as part + of the regexp. We also have to make sure in the from we went back to the root of the config:: + + subinterface: + _process: + mode: block + regexp: "(?Pinterface {{interface_key}}\\.(?P\\d+)\\n(?:.|\\n)*?^!$)" + from: "{{ bookmarks.interfaces }}" + + +Example 3. + + Sometimes we can get easily more information in one go than just the ``key`` and the ``block``. For + those cases we can capture more groups and they will be stored in the ``extra_vars`` dictionary:: + + address: + _process: + mode: block + regexp: "(?Pip address (?P(?P.*))\\/(?P\\d+))(?P secondary)*" + from: "{{ bookmarks['parent'] }}" + +Leaf - search +------------- + +Extract ``value`` from a regexp. + +Arguments: + +* **regexp** (mandatory) - Regular expression to apply. Note the regular expression has to capture the ``value`` + at least but it can capture others if you want. +* **default** (optional) - Value to assign if the regexp returns nothing. + +Example. + + Get the description of an interface:: + + description: + _process: + mode: search + regexp: "description (?P.*)" + from: "{{ bookmarks.interface[interface_key] }}" + +Leaf - value +------------ + +Apply a user-defined value to the object. + +Arguments: + +* **value** (mandatory): What value to apply + +Example. + + Evaluate a value we already extracted and set model to ``True`` if is not ``None``:: + + secondary: + _process: + mode: value + value: "{{ extra_vars.secondary != None }}" + +Leaf - is_absent +---------------- + +Works exactly like search but if the evaluation is ``None``, it will return ``True``. + +Example. + + Check if an interface is an IP interface or not:: + + ipv4: + _process: unnecessary + config: + _process: unnecessary + enabled: + _process: + mode: is_absent + regexp: "(?P^\\W*switchport$)" + from: "{{ bookmarks['parent'] }}" + +Leaf - is_present +----------------- + +Works exactly like search but if the evaluation is ``None``, it will return ``False``. + +Example. + + Check if an interface is enabled:: + + enabled: + _process: + mode: is_present + regexp: "(?Pno shutdown)" + from: "{{ bookmarks.interface[interface_key] }}" + +Leaf - map +---------- + +Works exactly like search but we do a lookup of the value on a map. + + +Arguments: + +* **regexp** (mandatory) - Same as ``search`` +* **default** (optional) - Same as ``search`` +* **map** (optional) - Map where to do the lookup function. + +Example. + + Check type of interface by extracting the name and doing a lookup:: + + _process: + mode: map + regexp: "(?P(\\w|-)*)\\d+" + from: "{{ interface_key }}" + map: + Ethernet: ethernetCsmacd + Management: ethernetCsmacd + Loopback: softwareLoopback + Port-Channel: ieee8023adLag + Vlan: l3ipvlan + diff --git a/docs/yang/parsers/XMLParser.rst b/docs/yang/parsers/XMLParser.rst new file mode 100644 index 00000000..e244734f --- /dev/null +++ b/docs/yang/parsers/XMLParser.rst @@ -0,0 +1,151 @@ +XMLParser +========= + +This extractor will read an XML an extract data from it. + +To illustrate the examples below we will use the following configuration:: + + + + + ge-0/0/0 + adasdasd + + + lo0 + + + + + +List - xpath +------------ + +Advances in the XML document up to the point where the relevant list of elements is found. + +Arguments: + +* **xpath** (mandatory): elements to traverse +* **key** (mandatory): which element is the key of the list + +Example: + + Starting from the root, the following action will move us to ``interface`` so we can + parse each interface individually:: + + interface: + _process: + mode: xpath + xpath: "interfaces/interface" + key: name + from: "{{ bookmarks.interfaces }}" + + This means after this action we will have a list of interface blocks like this:: + + + - + ge-0/0/0 + adasdasd + + - + lo0 + + + + And we will be able to keep processing them individually. + +Leaf - xpath +------------ + +Extracts a value from an element. + +Arguments: + +* **xpath** (mandatory): element to extract +* **regexp** (optional): Apply regexp to the value of the element. Must capture ``value`` group. + See "leaf - map" example for more details. +* **default** (optional): Set this value if no element is found. +* **attribute** (optional): Instead of the ``text`` of the element extracted, extract this attribute of the element. + +Example: + + For each interface, read the element ``description`` and map it into the object:: + + description: + _process: + mode: xpath + xpath: description + from: "{{ bookmarks['parent'] }}" + +Leaf - value +------------ + +Apply a user-defined value to the object. + +Arguments: + +* **value** (mandatory): What value to apply + +Example: + + In the following example we can assign a value we already have to the ``interface.name`` attribute:: + + name: + _process: + mode: value + value: "{{ interface_key }}" + +Leaf - map +---------- + +Extract value and do a lookup to choose value. + +Arguments: + +* **xpath** (mandatory): Same as ``xpath`` action. +* **regexp** (optional): Same as ``xpath`` action. +* **map** (mandatory): Dictionary where we will do the lookup action. + +Example: + + We can read an element, extract some information and then apply the lookup function, for example, we can + read the interface name, extract some of the first few characters and figure out the type of interface + like this:: + + type: + _process: + mode: map + xpath: name + regexp: "(?P[a-z]+).*" + from: "{{ bookmarks['parent'] }}" + map: + ge: ethernetCsmacd + lo: softwareLoopback + ae: ieee8023adLag + + The regular expression will give `ge` and `lo` which we can map into `ethernetCsmacd` and + `ieee8023adLag` respectively. + +Leaf - is_absent +---------------- + +Works exactly like ``xpath`` but if the evaluation is ``None``, it will return ``True``. + +Example: + + We could check if an interface is enabled with this:: + + enabled: + _process: + mode: is_absent + xpath: "disable" + from: "{{ bookmarks['parent'] }}" + + As `disable` is missing in the interface `ge-0/0/0` we know it's enabled while `lo0` will be not + as it was present. + +Leaf - is_present +----------------- + +Works exactly like ``xpath`` but if the evaluation is ``None``, it will return ``False``. + diff --git a/docs/yang/profiles.rst b/docs/yang/profiles.rst new file mode 100644 index 00000000..a4154638 --- /dev/null +++ b/docs/yang/profiles.rst @@ -0,0 +1,69 @@ +Profiles +-------- + +In order to correctly map YANG objects to native configuration and vice versa, ``napalm-yang`` uses the concept of **profiles**. Profiles, identify the type of device you are dealing with, which can vary depending on the OS, version and/or platform you are using. + +If you are using a napalm driver and have access to your device, you will have access to the ``profile`` property which you can pass to any function that requires to know the profile. If you are not using a napalm driver or don't have access to the device, a profile is just a list of strings so you can just specify it directly. For example:: + + # Without access to the device + model.parse_config(profile=["junos"], config=my_configuration) + + # With access + with driver(hostname, username, password) as d: + model.parse_config(device=d) + + # With access but overriding profile + with driver(hostname, username, password) as d: + model.parse_config(device=d, profile=["junos13", "junos"]) + +.. note:: As you noticed a device may have multiple profiles. When that happens, each model that is + parsed will loop through the profiles from left to right and use the first profile that + implements that model (note that a YANG model is often comprised of multiple modules). This + is useful as there might be small variances between different systems + but not enough to justify reimplementing everything. + +You can find the profiles `here `_ but what is exactly is a profile? A profile is a bunch of YAML files that follows the structure of a YANG model and describes two things: + +#. How to parse native configuration/state and map it into a model. +#. How to translate a model and map it into native configuration. + +For example, for a given interface, the snippet below specifies how to map configuration into the ``openconfig_interface`` model on EOS:: + + enabled: + _process: + mode: is_present + regexp: "(?Pno shutdown)" + from: "{{ parse_bookmarks.interface[interface_key] }}" + description: + _process: + mode: search + regexp: "description (?P.*)" + from: "{{ parse_bookmarks.interface[interface_key] }}" + mtu: + _process: + mode: search + regexp: "mtu (?P[0-9]+)" + from: "{{ parse_bookmarks.interface[interface_key] }}" + +And the following snippet how to map the same attributes from the ``openconfig_interface`` to native configuration:: + + enabled: + _process: + - mode: element + value: " shutdown\n" + when: "{{ not model }}" + description: + _process: + - mode: element + value: " description {{ model }}\n" + negate: " default description" + mtu: + _process: + - mode: element + value: " mtu {{ model }}\n" + negate: " default mtu\n" + +.. note:: + Profiles can also deal with structured data like XML or JSON. + +As you can see it's not extremely difficult to understand what they are doing, in the next section we will learn how to write our own profiles. diff --git a/docs/yang/translators.rst b/docs/yang/translators.rst new file mode 100644 index 00000000..6012a04d --- /dev/null +++ b/docs/yang/translators.rst @@ -0,0 +1,110 @@ +Translators +^^^^^^^^^^^ + +Translators are responsible for transforming a model into native configuration. + +Special fields +============== + +When translating an object, some fields might depend on the translator you are using but some will +available regardless. Some may be even be mandatory. + +mode +---- + +* **mandatory**: yes +* **description**: which parsing/translation action to use for this particular field +* **example**: Translate description attribute of an interface to native configuration:: + + description: + _process: + - mode: element + value: " description {{ model }}\n" + negate: " default description" + +when +---- + +* **mandatory**: no +* **description**: the evaluation of this field will determine if the action is executed or + skipped. This action is probably not very useful when parsing but it's available if you need it. +* **example**: configure ``switchport`` on IOS devices only if the interface is not a loopback + or a management interface:: + + ipv4: + _process: unnecessary + config: + _process: unnecessary + enabled: + _process: + - mode: element + value: " no switchport\n" + negate: " switchport\n" + in: "interface.{{ interface_key }}" + when: "{{ model and interface_key[0:4] not in ['mana', 'loop'] }}" + +in +-- + +* **mandatory**: no +* **description**: where to add the configuration. Sometimes the configuration might have to be + installed on a different object from the one you are parsing. For example, when configuring a + tagged subinterface on junos you will have to add also a ``vlan-tagging`` option on the parent + interface. On ``IOS/EOS``, when configuring interfaces, you have to also add the configuration in + the root of the configuration and not as a child of the parent interface:: + + vlan: + _process: unnecessary + config: + _process: unnecessary + vlan_id: + _process: + - mode: element + element: "vlan-tagging" + in: "interface.{{ interface_key }}" # <--- add element to parent interface + when: "{{ model > 0 }}" + value: null + - mode: element + element: "vlan-id" + when: "{{ model > 0 }}" + + (...) + subinterface: + _process: + mode: container + key_value: "interface {{ interface_key}}.{{ subinterface_key }}\n" + negate: "no interface {{ interface_key}}.{{ subinterface_key }}\n" + in: "interfaces" # <--- add element to root of configuration + +.. note:: This field follows the same logic as the :ref:`yang_special_field_bookmarks` special field. + +Special variables +================= + +keys +---- + +See :ref:`yang_special_field_keys`. + +model +----- + +This is the current model/attribute being translated. You have the entire object at your disposal, +not only it's value so you can do things like:: + + vlan_id: + _process: + - mode: element + value: " encapsulation dot1q vlan {{ model }}\n" + +Or:: + + config: + _process: unnecessary + ip: + _process: unnecessary + prefix_length: + _process: + - mode: element + value: " ip address {{ model._parent.ip }}/{{ model }} {{ 'secondary' if model._parent.secondary else '' }}\n" + negate: " default ip address {{ model._parent.ip }}/{{ model }}\n" diff --git a/docs/yang/translators/TextTranslator.rst b/docs/yang/translators/TextTranslator.rst new file mode 100644 index 00000000..1fe73977 --- /dev/null +++ b/docs/yang/translators/TextTranslator.rst @@ -0,0 +1,81 @@ +TextTranslator +============== + +TextTranslator is responsible of translating a model into text configuration. + +Metadata +-------- + +* **root** - Set to true if this is the root of the model. + +List - container +---------------- + +Create/Removes each element of the list. + +Arguments: + + * **key_value** (mandatory): How to create the element. + * **negate** (mandatory): How to eliminate/default the element. + * **replace** (optional): Whether the element has to be defaulted or not during the replace operation. + * **end** (optional): Closing command to signal end of element + +Example 1: + + Create/Default interfaces:: + + interfaces: + _process: unnecessary + interface: + _process: + mode: container + key_value: "interface {{ interface_key }}\n" + negate: "{{ 'no' if interface_key[0:4] in ['Port', 'Loop'] else 'default' }} interface {{ interface_key }}\n" + end: " exit\n" + +Example 2: + + Configure IP addresses. As the parent interface is defaulted already, don't do it again:: + + address: + _process: + mode: container + key_value: " ip address {{ model.config.ip }} {{ model.config.prefix_length|cidr_to_netmask }}{{ ' secondary' if model.config.secondary else '' }}\n" + negate: " default ip address {{ model.config.ip }} {{ model.config.prefix_length|cidr_to_netmask }}{{ ' secondary' if model.config.secondary else '' }}\n" + replace: false + +Leaf - element +-------------- + +Configures an attribute. + +Arguments: + + * **value** (mandatory): How to configure the attribute + * **negate** (mandatory): How to default the attribute + +Example 1: + + Configure description:: + + description: + _process: + - mode: element + value: " description {{ model }}\n" + negate: " default description" + +Example 2: + + Configure an IP address borrowing values from other fields:: + + address: + _process: unnecessary + config: + _process: unnecessary + ip: + _process: unnecessary + prefix_length: + _process: + - mode: element + value: " ip address {{ model._parent.ip }}/{{ model }} {{ 'secondary' if model._parent.secondary else '' }}\n" + negate: " default ip address {{ model._parent.ip }}/{{ model }} {{ 'secondary' if model._parent.secondary else '' }}\n" diff --git a/docs/yang/translators/XMLTranslator.rst b/docs/yang/translators/XMLTranslator.rst new file mode 100644 index 00000000..73f18a3c --- /dev/null +++ b/docs/yang/translators/XMLTranslator.rst @@ -0,0 +1,121 @@ +XMLTranslator +============= + +XMLTranslator is responsible of translating a model into XML configuration. + +Metadata +-------- + +* **xml_root** - Set this value on the root of the model to instantiate the XML object. + +For example:: + + --- + metadata: + processor: XMLTranslator + xml_root: configuration + +This will instantiate the XML object ````. + +Container - container +--------------------- + +Creates a container. + +Arguments: + + * **container** (mandatory) - Which container to create + * **replace** (optional) - Whether this element has to be replaced in case of merge/replace or + it's not necessary (remember XML is hierarchical which means you can unset things directly in + the root). + +Example: + + Create the ``interfaces`` container:: + + _process: + mode: container + container: interfaces + replace: true + +List - container +---------------- + +For each element of the list, create a container. + +Arguments: + + + * **container** (mandatory) - Which container to create + * **key_element** (mandatory) - Lists require a key element, this is the name of the element. + * **key_value** (mandatory) - Key element value. + + +Example: + + Create interfaces:: + + interface: + _process: + mode: container + container: interface + key_element: name + key_value: "{{ interface_key }}" + + This will result elements such as:: + + + ge-0/0/0 + + + lo0 + + +Leaf - element +-------------- + +Adds an element to a container. + +Arguments: + + * **element** (mandatory): Element name. + * **value** (optional): Override value. Default is value of the object. + +Example 1: + + Configure description:: + + description: + _process: + - mode: element + element: description + +Example 2: + + Enable or disable an interface:: + + enabled: + _process: + - mode: element + element: "disable" + when: "{{ not model }}" + value: null + + We override the value and set it to ``null`` because to disable we just have to create the + element, we don't have to set any value. + +Example 3: + + Configure an IP address borrowing values from other fields:: + + config: + _process: unnecessary + ip: + _process: unnecessary + prefix_length: + _process: + - mode: element + element: name + value: "{{ model._parent.ip }}/{{ model }}" + when: "{{ model }}" + diff --git a/docs/yang/writing_profiles.rst b/docs/yang/writing_profiles.rst new file mode 100644 index 00000000..f88aca84 --- /dev/null +++ b/docs/yang/writing_profiles.rst @@ -0,0 +1,81 @@ +Writing Profiles +================ + +As it's been already mentioned, a profile is a bunch of YAML files that describe how to map native +configuration and how to translate an object into native configuration. In order to read native +configuration we will use **parsers**, to translate a YANG model into native configuration we will +use **translators**. + +Both parsers and translators follow three basic rules: + +#. One directory per module. +#. One file per model. +#. Exact same representation of the model inside the file: + +For example:: + + $ tree napalm_yang/mappings/eos/parsers/config + napalm_yang/mappings/eos/parsers/config + ├── napalm-if-ip + │   └── secondary.yaml + ├── openconfig-if-ip + │   └── ipv4.yaml + ├── openconfig-interfaces + │   └── interfaces.yaml + └── openconfig-vlan + ├── routed-vlan.yaml + └── vlan.yaml + + 4 directories, 5 files + $ cat napalm_yang/mappings/eos/parsers/config/openconfig-vlan/vlan.yaml + --- + metadata: + (trimmed for brevity) + + vlan: + (trimmed for brevity) + config: + (trimmed for brevity) + vlan_id: + (trimmed for brevity) + +If we check the content of the file ``vlan.yaml`` we can clearly see two parts: + +* **metadata** - This part specifies what parser or translator we want to use as there are several + depending on the type of data we are parsing from or translating to and some options that the + parser/translator might need. For example:: + + metadata: + processor: XMLParser + execute: + - method: _rpc + args: + get: "" + +In this case we are using the ``XMLParser`` parser and in order to get the data we need from the +device we have to call the method ``_rpc`` with the ``args`` parameters. This is, by the way, an +RPC call for a junos device. + +* **vlan** - This is the part that follows the model specification. In this case is ``vlan`` but in + others it might be ``interfaces``, ``addressess`` or something else, this will be model dependent + but it's basically whatever it's not ``metadata``. This part will follow the model specification + and add rules on each attribute to tell the parser/translator what needs to be done. For + example:: + + vlan: + _process: unnecessary + config: + _process: unnecessary + vlan_id: + _process: + mode: xpath + xpath: "vlan-id" + from: "{{ parse_bookmarks['parent'] }}" + +As we are dealing with a parser we have to specify the ``_process`` attribute at each step (translators +require the attribute ``_process``). There are two special types of actions; ``unnecessary`` and +``not_implemented``. Both do exactly the same, skip any action and move onto the next attribute. The +only difference is purely aesthetically and for documentation purposes. + +Something else worth noting is that each attribute inside ``_process/_process`` is evaluated as a +``jinja2`` template so you can do variable substitutions, evaluations, etc...