add directory traversal to command line utility (to be documented)

setup.py

@@ -58,7 +58,7 @@ def read(*names, **kwargs):
         'ONIX', 'validation', 'EDItEUR', 'XML', 'RelaxNG', 'XMLSchema'
     ],
     install_requires=[
-        'click', 'lxml', 'defusedxml'
+        'click', 'lxml', 'defusedxml', 'scandir'
     ],
     extras_require={
         # eg:

src/onixcheck/__main__.py

@@ -1,19 +1,24 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
+import os
 import sys
 import click
 import logging
 from onixcheck import validate
 from onixcheck.exeptions import get_logger
+from onixcheck.utils import iter_files, DEFAULT_EXTENSIONS
 
 log = get_logger()
 
 
 @click.command()
-@click.argument('infile', type=click.File('rb'))
+@click.argument('infile', type=click.File('rb'), required=False)
+@click.option('--path', '-p', type=click.Path(exists=True, resolve_path=True), help='Validate all files in path')
+@click.option('--ext', '-e', multiple=True, default={'xml'}, help='Extension to check (default: xml)')
+@click.option('--recursive', '-r', default=False, is_flag=True, help='Scan subdirectories in path')
 @click.option('--debug', '-d', is_flag=True, help='Show debug information')
 @click.version_option(prog_name='Onixcheck')
-def main(infile, debug):
+def main(infile, path, ext, recursive, debug):
 
     if debug:
         logging.basicConfig(
@@ -22,18 +27,46 @@ def main(infile, debug):
         )
         click.echo('DEBUG logging enabled')
 
-    click.echo('Validating: %s' % click.format_filename(infile.name))
+    # validate current working dir
+    if not infile and not path:
+        path = os.getcwdu()
 
-    messages = validate(infile)
-    is_valid = messages == []
+    all_valid = True
 
-    if is_valid:
-        click.echo('VALID - No errors found')
+    if infile:
+        click.echo('Validating: %s' % click.format_filename(infile.name))
+        messages = validate(infile)
+        is_valid = messages == []
+        if is_valid:
+            click.echo('VALID - No errors found')
+        else:
+            click.echo('INVALID - errors found:')
+            all_valid = False
+            for msg in messages:
+                    click.echo(msg)
+
+    if path:
+        tree_or_dir = 'tree' if recursive else 'dir'
+        click.echo()
+        click.echo('Validating all files in %s %s' % (tree_or_dir, path))
+
+        for onix_file_path in iter_files(path, ext, recursive):
+            click.echo()
+            click.echo('Validating: %s' % click.format_filename(onix_file_path))
+            with open(onix_file_path, 'rb') as onix_file:
+                messages = validate(onix_file)
+                is_valid = messages == []
+
+            if is_valid:
+                click.echo('VALID - No errors found')
+            else:
+                click.echo('INVALID - errors found:')
+                all_valid = False
+                for msg in messages:
+                    click.echo(msg)
+    if all_valid:
         sys.exit(0)
     else:
-        click.echo('INVALID - errors found:')
-        for msg in messages:
-            click.echo(msg)
         sys.exit(1)
 
 

src/onixcheck/models.py

@@ -153,7 +153,7 @@ def from_logentry(cls, logentry, filename=''):
 
         :param _LogEntry logentry: Validatation error from LXML
         :param str filename: Optional filename to prefix error location
-        :return ValidationError:
+        :return Message:
         """
         l = logentry
         location = '%s:%s:%s' % (filename, l.line, l.column)
@@ -167,3 +167,19 @@ def from_logentry(cls, logentry, filename=''):
             message=message,
             error_type=l.type_name
         )
+
+    @classmethod
+    def from_exception(cls, exc, filename=''):
+        """
+
+        :param Exception exc:
+        :param str filename: Optional filename to prefix error location
+        :return Message:
+        """
+        return cls(
+            level='CRITICAL',
+            validator='ONIXCHECK',
+            location=filename,
+            message=exc.message,
+            error_type='EXCEPTION'
+        )

src/onixcheck/utils.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+"""Generic or common utility functions"""
+from __future__ import print_function, unicode_literals
+from os.path import splitext, join
+from scandir import scandir, walk
+
+DEFAULT_EXTENSIONS = {'xml', 'onx', 'onix'}
+
+
+def iter_files(root, exts=None, recursive=False):
+    """
+    Iterate over file paths that match the given extensions within root.
+
+    :param str or unicode root: Root folder to start collecting files
+    :param iterable exts: file extensions to include
+    :param bool recursive: Wether to walk the complete directory tree
+    :return iterable: Generator of absolute file paths with given extensions
+    """
+
+    if exts is None:
+        exts = DEFAULT_EXTENSIONS
+
+    exts = {x.lower() for x in exts}
+
+    if recursive is False:
+        for entry in scandir(root):
+            ext = splitext(entry.name)[-1].lstrip('.').lower()
+            if entry.is_file() and ext in exts:
+                yield entry.path
+    else:
+        for root, folders, files in walk(root):
+            for f in files:
+                if splitext(f)[-1].lstrip('.').lower() in exts:
+                    yield join(root, f)

src/onixcheck/validation.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 from os.path import basename
+from onixcheck.exeptions import OnixError
 from onixcheck.models import OnixFile, Message
 
 
@@ -10,15 +11,19 @@ def validate(infile):
     :param file or path infile: File Obj or path to file
     :return list: List Message objs (invalid onix) or empty list (valid onix)
     """
-    onix_file = OnixFile(infile)
-    validator = onix_file.get_validator()
-    validator(onix_file.xml_tree())
-    errors = validator.error_log
-    msg = Message.from_logentry
-
     if hasattr(infile, 'name'):
         filename = infile.name
     else:
         filename = basename(infile)
 
+    try:
+        onix_file = OnixFile(infile)
+    except OnixError as e:
+        return [Message.from_exception(e, filename)]
+
+    validator = onix_file.get_validator()
+    validator(onix_file.xml_tree())
+    errors = validator.error_log
+    msg = Message.from_logentry
+
     return [msg(e, filename)for e in errors]

tests/test_utils.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function
+import types
+from onixcheck import utils
+from os.path import abspath, dirname
+
+
+TEST_DIR = abspath(dirname(utils.__file__))
+
+
+def test_iter_files_simple():
+    gen = utils.iter_files(TEST_DIR)
+    assert isinstance(gen, types.GeneratorType)
+    assert list(gen) > 5
+
+
+def test_iter_files_no_matches():
+    gen = utils.iter_files(TEST_DIR, ['noext'])
+    assert len(list(gen)) == 0
+
+
+def test_iter_files_flat():
+    gen = utils.iter_files(TEST_DIR, ['xsd'])
+    assert len(list(gen)) == 0
+    gen = utils.iter_files(TEST_DIR, ['PY'])
+    assert len(list(gen)) > 5
+
+
+def test_iter_files_recursive():
+    gen = utils.iter_files(TEST_DIR, ['xsd'], recursive=True)
+    assert len(list(gen)) > 5
+