Add lexer for YANG 1.1 (#1408)

* Add yang lexer for issue #1407

* fix copyright statement

* adjust examplefile for yang

* fix to avoid duplicate code in lexer

* add more testcases for yang lexer

* simplify yang lexer

* simplify default rule in yang lexer

* change example yang file

* add version to yang lexer

doc/languages.rst

@@ -287,6 +287,7 @@ Other markup
 * XML
 * XSLT
 * YAML
+* YANG
 * Windows Registry files
 
 ... that's all?

pygments/lexers/_mapping.py

@@ -488,6 +488,7 @@
     'XtlangLexer': ('pygments.lexers.lisp', 'xtlang', ('extempore',), ('*.xtm',), ()),
     'YamlJinjaLexer': ('pygments.lexers.templates', 'YAML+Jinja', ('yaml+jinja', 'salt', 'sls'), ('*.sls',), ('text/x-yaml+jinja', 'text/x-sls')),
     'YamlLexer': ('pygments.lexers.data', 'YAML', ('yaml',), ('*.yaml', '*.yml'), ('text/x-yaml',)),
+    'YangLexer': ('pygments.lexers.yang', 'YANG', ('yang',), ('*.yang',), ('application/yang',)),
     'ZeekLexer': ('pygments.lexers.dsls', 'Zeek', ('zeek', 'bro'), ('*.zeek', '*.bro'), ()),
     'ZephirLexer': ('pygments.lexers.php', 'Zephir', ('zephir',), ('*.zep',), ()),
     'ZigLexer': ('pygments.lexers.zig', 'Zig', ('zig',), ('*.zig',), ('text/zig',)),

pygments/lexers/yang.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.yang
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the YANG 1.1 modeling language. See :rfc:`7950`.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import (RegexLexer, bygroups, words)
+from pygments.token import (Text, Token, Name, String, Comment,
+                            Number)
+
+__all__ = ['YangLexer']
+
+class YangLexer(RegexLexer):
+    """
+    Lexer for `YANG <https://tools.ietf.org/html/rfc7950/>`_, based on RFC7950
+
+    .. versionadded:: 2.7
+    """
+    name = 'YANG'
+    aliases = ['yang']
+    filenames = ['*.yang']
+    mimetypes = ['application/yang']
+
+    #Keywords from RFC7950 ; oriented at BNF style
+    TOP_STMTS_KEYWORDS = ("module", "submodule")
+    MODULE_HEADER_STMT_KEYWORDS = ("yang-version", "namespace", "prefix", "belongs-to")
+    META_STMT_KEYWORDS = ("organization", "contact", "description",
+                          "reference", "revision")
+    LINKAGE_STMTS_KEYWORDS = ("import", "include", "revision-date")
+    BODY_STMT_KEYWORDS = ("extension", "feature", "identity", "typedef",
+                          "grouping", "augment", "rpc", "notification",
+                          "deviation", "action", "argument", "identity",
+                          "if-feature", "input", "output")
+    DATA_DEF_STMT_KEYWORDS = ("container", "leaf-list", "leaf", "list",
+                              "choice", "anydata", "anyxml", "uses",
+                              "case", "config", "deviate", "must",
+                              "when", "presence", "refine")
+    TYPE_STMT_KEYWORDS = ("type", "units", "default", "status", "bit",
+                          "enum", "error-app-tag", "error-message",
+                          "fraction-digits", "length", "min-elements",
+                          "max-elements", "modifier", "ordered-by", "path",
+                          "pattern", "position", "range", "require-instance",
+                          "value", "yin-element", "base")
+    LIST_STMT_KEYWORDS = ("key", "mandatory", "unique")
+
+    #RFC7950 other keywords
+    CONSTANTS_KEYWORDS = ("true", "false", "current", "obsolete", "deprecated",
+                          "add", "delete", "replace", "not-supported",
+                          "invert-match", "max", "min", "unbounded", "user")
+
+    #RFC7950 Built-In Types
+    TYPES = ("binary", "bits", "boolean", "decimal64", "empty", "enumeration",
+             "int8", "int16", "int32", "int64", "string", "uint8", "uint16",
+             "uint32", "uint64", "union", "leafref", "identityref", "instance-identifier")
+
+    suffix_re_pattern = r'(?=[^\w\-\:])'
+
+    tokens = {
+        'comments': [
+            (r'[^*/]', Comment),
+            (r'/\*', Comment, '#push'),
+            (r'\*/', Comment, '#pop'),
+            (r'[*/]', Comment),
+        ],
+        "root": [
+            (r'\s+', Text.Whitespace),
+            (r'[\{\}\;]+', Token.Punctuation),
+            (r'(?<![\-\w])(and|or|not|\+|\.)(?![\-\w])', Token.Operator),
+
+            (r'"(?:\\"|[^"])*?"', String.Double),
+            (r"'(?:\\'|[^'])*?'", String.Single),
+
+            (r'/\*', Comment, 'comments'),
+            (r'//.*?$', Comment),
+
+            #match BNF stmt for `node-identifier` with [ prefix ":"]
+            (r'(?:^|(?<=[\s{};]))([\w.-]+)(:)([\w.-]+)(?=[\s{};])',
+             bygroups(Name.Namespace, Token.Punctuation, Name.Variable)),
+
+            #match BNF stmt `date-arg-str`
+            (r'([0-9]{4}\-[0-9]{2}\-[0-9]{2})(?=[\s\{\}\;])', Name.Label),
+            (r'([0-9]+\.[0-9]+)(?=[\s\{\}\;])', Number.Float),
+            (r'([0-9]+)(?=[\s\{\}\;])', Number.Integer),
+
+            (words(TOP_STMTS_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(MODULE_HEADER_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(META_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(LINKAGE_STMTS_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(BODY_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(DATA_DEF_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(TYPE_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(LIST_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(TYPES, suffix=suffix_re_pattern), Name.Class),
+            (words(CONSTANTS_KEYWORDS, suffix=suffix_re_pattern), Name.Class),
+
+            (r'[^;{}\s\'\"]+', Name.Variable),
+        ]
+    }

tests/examplefiles/test.yang

@@ -0,0 +1,64 @@
+module server-system {
+    yang-version 1.1;
+    namespace "http://autlan.dt/gribok/yang/example";
+    prefix ex;
+
+    import ietf-yang-types {
+      prefix yang;
+      reference
+        "RFC 6991: Common YANG Data Types.";
+    }
+
+    organization "Gribok";
+    contact "gribok@example.org";
+    description
+        "An example module";
+
+    revision 2020-04-03 {
+        description "Example yang";
+    }
+
+    /*
+     * Comment for container system
+     */
+
+    container system {
+        leaf host-name {
+            type string;
+            description "Hostname for this system";
+        }
+
+        leaf-list domain-search {
+            type string;
+            description "List of domain names to search";
+        }
+
+        container login {
+            leaf message {
+                type string;
+                description
+                    "Message given at start of login session";
+            }
+
+            list user {
+                key "name";
+                leaf name {
+                    type string;
+                }
+                leaf uuid {
+                  type yang:uuid;
+                }
+                leaf full-name {
+                    type string;
+                    mandatory true;
+                    description
+                      "The full name of user  See also 'name'.  This could
+                       be, for example, a reference to the user name";
+                }
+                leaf class {
+                    type string;
+                }
+            }
+        }
+    }
+}

tests/test_yang.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+"""
+    Basic Yang Test
+    ~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import pytest
+
+from pygments.token import Operator, Number, Text, Token
+from pygments.lexers import YangLexer
+
+@pytest.fixture(scope='module')
+def lexer():
+    yield YangLexer()
+
+def test_namespace_1(lexer):
+    """
+    Namespace `urn:test:std:yang` should not be explicitly highlighted
+    """
+    fragment = u'namespace urn:test:std:yang;\n'
+    tokens = [
+        (Token.Keyword, u'namespace'),
+        (Token.Text.Whitespace, u' '),
+        (Token.Name.Variable, u'urn:test:std:yang'),
+        (Token.Punctuation, u';'),
+        (Token.Text.Whitespace, u'\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+def test_namespace_2(lexer):
+    """
+    namespace-prefix `yang` should be explicitly highlighted
+    """
+    fragment = u'type yang:counter64;\n'
+    tokens = [
+        (Token.Keyword, u'type'),
+        (Token.Text.Whitespace, u' '),
+        (Token.Name.Namespace, u'yang'),
+        (Token.Punctuation, u':'),
+        (Token.Name.Variable, u'counter64'),
+        (Token.Punctuation, u';'),
+        (Token.Text.Whitespace, u'\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+def test_revision_date(lexer):
+    """
+    Revision-date `2020-08-03` should be explicitly highlighted
+    """
+    fragment = u'revision 2020-03-08{\n'
+    tokens = [
+        (Token.Keyword, u'revision'),
+        (Token.Text.Whitespace, u' '),
+        (Token.Name.Label, u'2020-03-08'),
+        (Token.Punctuation, u'{'),
+        (Token.Text.Whitespace, u'\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+def test_integer_value(lexer):
+    """
+    Integer value `5` should be explicitly highlighted
+    """
+    fragment = u'value 5;\n'
+    tokens = [
+        (Token.Keyword, u'value'),
+        (Token.Text.Whitespace, u' '),
+        (Token.Number.Integer, u'5'),
+        (Token.Punctuation, u';'),
+        (Token.Text.Whitespace, u'\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+def test_string_value(lexer):
+    """
+    String value `"5"` should be not explicitly highlighted
+    """
+    fragment = u'value "5";\n'
+    tokens = [
+        (Token.Keyword, u'value'),
+        (Token.Text.Whitespace, u' '),
+        (Token.String.Double, u'"5"'),
+        (Token.Punctuation, u';'),
+        (Token.Text.Whitespace, u'\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+def test_float_value(lexer):
+    """
+    Float value `1.1` should be explicitly highlighted
+    """
+    fragment = u'yang-version 1.1;\n'
+    tokens = [
+        (Token.Keyword, u'yang-version'),
+        (Token.Text.Whitespace, u' '),
+        (Token.Number.Float, u'1.1'),
+        (Token.Punctuation, u';'),
+        (Token.Text.Whitespace, u'\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens