feat(parse): add parse function (str => AST)

pyangext/utils.py

@@ -1,33 +1,62 @@
 # -*- coding: utf-8 -*-
 """Utility belt for working with ``pyang`` and ``pyangext``."""
+import logging
+from os.path import isfile
+from warnings import warn
+
 from six import StringIO
 
 from pyang import Context, FileRepository
+from pyang.error import err_level, err_to_str, error_codes, is_warning
 from pyang.translators import yang
+from pyang.yang_parser import YangParser
 
 from .definitions import PREFIX_SEPARATOR
 
-__all__ = ['create_context', 'compare_prefixed', 'select', 'find', 'dump']
+__all__ = [
+    'create_context',
+    'compare_prefixed',
+    'select',
+    'find',
+    'dump',
+    'check',
+    'parse',
+]
+
+logging.basicConfig(level=logging.INFO)
+logging.captureWarnings(True)
+LOGGER = logging.getLogger(__name__)
 
 DEFAULT_OPTIONS = {
+    'path': [],
+    'deviations': [],
+    'features': [],
     'format': 'yang',
-    'verbose': True,
-    'list_errors': True,
-    'print_error_code': True,
-    'yang_remove_unused_imports': True,
-    'yang_canonical': True,
-    'trim_yin': False,
     'keep_comments': True,
-    'features': [],
-    'deviations': [],
-    'path': [],
+    'no_path_recurse': False,
+    'trim_yin': False,
+    'yang_canonical': True,
+    'yang_remove_unused_imports': True,
+    # -- errors
+    'ignore_error_tags': [],
+    'ignore_errors': [],
+    'list_errors': True,
+    'print_error_code': False,
+    'errors': [],
+    'warnings': [code for code, desc in error_codes.items() if desc[0] > 4],
+    'verbose': True,
 }
 """Default options for pyang command line"""
 
-DEFAULT_ATTRIBUTES = {
-    'trim_yin': False,
-}
-"""Default parameters for pyang context"""
+_COPY_OPTIONS = [
+    'canonical',
+    'max_line_len',
+    'max_identifier_len',
+    'trim_yin',
+    'lax_xpath_checks',
+    'strict',
+]
+"""copy options to pyang context options"""
 
 
 class objectify(object):  # pylint: disable=invalid-name
@@ -63,8 +92,8 @@ def create_context(path='.', *options, **kwargs):
     ctx = Context(repo)
     ctx.opts = opts
 
-    for attr, value in DEFAULT_ATTRIBUTES.items():
-        setattr(ctx, attr, value)
+    for attr in _COPY_OPTIONS:
+        setattr(ctx, attr, getattr(opts, attr))
 
     return ctx
 
@@ -158,3 +187,98 @@ def dump(node, file_obj=None, prev_indent='', indent_string='  ', ctx=None):
 
     # oneliners <3: if no file_obj get buffer content and close it!
     return file_obj or (_file_obj.getvalue(), _file_obj.close())[0]
+
+
+def check(ctx, rescue=False):
+    """Check existence of errors or warnings in context.
+
+    Code mostly borrowed from ``pyang`` script.
+
+    Arguments:
+        ctx (pyang.Context): pyang context to be checked.
+
+    Keyword Arguments:
+        rescue (boolean): if ``True``, no exception/warning will be raised.
+
+    Raises:
+        SyntaxError: if errors detected
+
+    Warnings:
+        SyntaxWarning: if warnings detected
+
+    Returns:
+        tuple: (list of errors, list of warnings), if ``rescue`` is ``True``
+    """
+    errors = []
+    warnings = []
+    opts = ctx.opts
+
+    if opts.ignore_errors:
+        return (errors, warnings)
+
+    for (epos, etag, eargs) in ctx.errors:
+        if etag in opts.ignore_error_tags:
+            continue
+        if not ctx.implicit_errors and hasattr(epos.top, 'i_modulename'):
+            # this module was added implicitly (by import); skip this error
+            # the code includes submodules
+            continue
+        elevel = err_level(etag)
+        explain = etag if opts.print_error_code else err_to_str(etag, eargs)
+        message = '({}) {}'.format(str(epos), explain)
+        if is_warning(elevel) and etag not in opts.errors:
+            if 'error' in opts.warnings and etag not in opts.warnings:
+                pass
+            elif 'none' in opts.warnings:
+                continue
+            else:
+                warnings.append(message)
+                continue
+
+        errors.append(message)
+
+    if rescue:
+        return (errors, warnings)
+
+    if warnings:
+        for message in warnings:
+            warn(message, SyntaxWarning)
+
+    if errors:
+        raise SyntaxError('\n'.join(errors))
+
+
+def parse(text, ctx=None):
+    """Parse a YANG statement into an Abstract Syntax Tree
+
+    Arguments:
+        text (str): file name for a YANG module or text
+        ctx (optional pyang.Context): context used to validate text
+
+    Returns:
+        pyang.statements.Statement
+    """
+    parser = YangParser()
+
+    filename = 'parser-input'
+
+    ctx_ = ctx or create_context()
+
+    if isfile(text):
+        filename = text
+        with open(filename, 'r') as fp:
+            text = fp.read()
+
+    # ensure reported errors are just from parsing
+    old_errors = ctx_.errors
+    ctx_.errors = []
+
+    ast = parser.parse(ctx_, filename, text)
+
+    # look for errors and warnings
+    check(ctx_)
+
+    # restore other errors
+    ctx_.errors = old_errors
+
+    return ast

pylintrc

@@ -277,7 +277,7 @@ ignored-modules=
 # List of classes names for which member attributes should not be checked
 # (useful for classes with attributes dynamically set). This supports can work
 # with qualified names.
-ignored-classes=
+ignored-classes=pytest
 
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular

tests/test_dump.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 # pylint: disable=redefined-outer-name
 """
-tests for YANG builder
+tests for YANG AST to string conversion
 """
 from pyang.statements import Statement
 

tests/test_parse.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# pylint: disable=redefined-outer-name
+"""
+tests for YANG content to AST conversion
+"""
+from __future__ import print_function
+
+import warnings
+
+import pytest
+
+from pyang.statements import Statement
+
+from pyangext.utils import create_context, find, parse
+
+__author__ = "Anderson Bravalheri"
+__copyright__ = "Copyright (C) 2016 Anderson Bravalheri"
+__license__ = "mozilla"
+
+
+@pytest.fixture
+def ctx():
+    """creates a context with custom configuration to print error code."""
+    return create_context(max_line_len=140, print_error_code=True)
+
+
+WITH_WARN = pytest.mark.parametrize('warning_type, text', [
+    ('LONG_LINE',
+     'module a {'
+     'leaf aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+     'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+     'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+     'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa;'
+     '}'),
+    # YANG parser does not produce several warnings...
+])
+
+
+@WITH_WARN
+def test_warn_invalid_string(ctx, warning_type, text):
+    """
+    parse should warn when something went wrong
+    """
+    with pytest.warns(SyntaxWarning) as info:
+        parse(text, ctx)
+
+    assert warning_type in '\n'.join(str(record.message) for record in info)
+
+
+@WITH_WARN
+def test_raise_invalid_string_if_warning_option(ctx, warning_type, text):
+    """
+    warns should be transformed in error if warnings option contains 'error'
+    """
+    ctx.opts.warnings = ['error']
+    with pytest.raises(SyntaxError) as info:
+        parse(text, ctx)
+
+    assert warning_type in str(info.value)
+
+
+@WITH_WARN
+def test_not_warn_invalid_string_if_none_option(ctx, warning_type, text):
+    """
+    warns should ignore if warnings option contains 'none'
+    """
+    ctx.opts.warnings = ['none']
+    assert parse(text, ctx)
+    print(warning_type)
+
+
+WITH_ERRORS = pytest.mark.parametrize('error_type, text', [
+    ('EXPECTED_ARGUMENT', 'module a { keyword }'),
+    ('INCOMPLETE_STATEMENT', 'module a { leaf a { leaf type }'),
+    ('EOF_ERROR', '/* unterminated comment'),
+    #  this do not need to be exaustive,
+    #  just ensure exceptions are raised
+])
+
+
+@WITH_ERRORS
+def test_raise_invalid_string(ctx, error_type, text):
+    """
+    parse should raise SyntaxErrors
+    """
+    with pytest.raises(SyntaxError) as info:
+        parse(text, ctx)
+    assert error_type in str(info.value)
+
+
+@WITH_ERRORS
+def test_not_raise_invalid_string_if_ignore_option(ctx, error_type, text):
+    """
+    No exception should be raised if ignore_errors option is true
+    But no tree is generated
+    """
+    ctx.opts.ignore_errors = True
+    assert parse(text, ctx) is None
+    print(error_type)
+
+
+@WITH_ERRORS
+def test_not_raise_invalid_string_if_ignore_tag(ctx, error_type, text):
+    """
+    No exception should be raised if ignore_errors_tags contain error type
+    But no tree is generated
+    """
+    ctx.opts.ignore_error_tags = [error_type]
+    assert parse(text, ctx) is None
+
+
+@pytest.mark.parametrize('text', [
+    'leaf id { type int32; }',
+    'container user { leaf name { type string; } }',
+    'description "BEST DESCRIPTION EVER!!";',
+    '// this is a single line comment\nleaf name { type string; }',
+    '/* this is a\nmulti-line\ncomment */leaf name { type string; }',
+    # this do not need to be exaustive,
+    # just parser produce Statement
+])
+def test_parse_valid_string(text):
+    """
+    should parse valid YANG string into Statement
+    """
+    assert isinstance(parse(text), Statement)
+
+
+@pytest.fixture
+def ok_yang():
+    """YANG modules without errors or warning"""
+    return """
+        module test {
+            namespace urn:yang:test;
+            prefix test;
+
+            revision 2008-01-02 {
+                description "first update";
+            }
+
+            leaf name {
+                type string {
+                    length "0..8";
+                    pattern "[0-9a-fA-F]*";
+                }
+            }
+        }
+    """
+
+
+def test_parse_nested(ok_yang):
+    """
+    should parse all nested statements
+    """
+    with warnings.catch_warnings(record=True) as info:
+        module = parse(ok_yang)
+        assert find(module, 'namespace', 'urn:yang:test')
+        assert find(module, 'prefix', 'test')
+        revision = find(module, 'revision', '2008-01-02')[0]
+        assert find(revision, 'description', 'first update')
+        leaf = find(module, 'leaf', 'name')[0]
+        leaf_type = find(leaf, 'type', 'string')[0]
+        assert find(leaf_type, 'length', '0..8')
+        assert find(leaf_type, 'pattern', '[0-9a-fA-F]*')
+
+    assert not info
+
+
+def test_parse_from_file(tmpdir, ok_yang):
+    """
+    should parse content from file if first argument is file
+    """
+    yang_file = tmpdir.join('test')
+    yang_file.write(ok_yang)
+    test_parse_nested(str(yang_file))

tests/test_paths.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 # # pylint: disable=redefined-outer-name
 """
-tests for pyangext cli.call
+tests for pyangext path discovery utilities
 """
 import os