global: addition of cli command for schema validation

* NEW Adds a new command which validates schemas.
  Command comes with an '--ignore_indexes' option.

Signed-off-by: PaulinaLach <paulina.malgorzata.lach@cern.ch>

INSTALL.rst

@@ -1,2 +1,9 @@
 Installation
 ============
+Invenio's DoSchema module is on PyPI so all you need is:
+
+.. code-block:: console
+
+    $ pip install doschema
+
+

MANIFEST.in

@@ -45,3 +45,6 @@ recursive-include docs *.py
 recursive-include docs *.rst
 recursive-include docs Makefile
 recursive-include tests *.py
+recursive-include examples *.py
+recursive-include examples *.json
+recursive-include tests *.json

docs/conf.py

@@ -26,7 +26,6 @@
 
 import os
 
-import sphinx.environment
 
 # -- General configuration ------------------------------------------------
 

docs/usage.rst

@@ -22,6 +22,7 @@
     as an Intergovernmental Organization or submit itself to any jurisdiction.
 
 
+=====
 Usage
 =====
 

doschema/__init__.py

@@ -22,7 +22,72 @@
 # waive the privileges and immunities granted to it by virtue of its status
 # as an Intergovernmental Organization or submit itself to any jurisdiction.
 
-"""JSON Schema utility functions and commands."""
+r"""JSON Schema utility functions and commands.
+
+Compatibility Validation
+-------------------------
+
+It validates compatibility between different JSON schemas versions.
+
+A schema is backward compatible if the fields' type remain the same in all
+JSON schemas declaring it and JSON schemas are type consistent within
+themselves too.
+
+>>> import json
+>>> from io import open
+>>>
+>>> import doschema.validation
+>>> from doschema.utils import detect_encoding
+>>>
+>>> schemas = [
+...     './examples/jsonschema_for_repetition.json',
+...     './examples/jsonschema_repetition.json'
+... ]
+>>>
+>>> schema_validator = doschema.validation.JSONSchemaValidator()
+>>> for schema in schemas:
+...     with open(schema, 'rb') as infile:
+...         byte_file = infile.read()
+...         encoding = detect_encoding(byte_file)
+...         string_file = byte_file.decode(encoding)
+...         json_schema = json.loads(string_file)
+...         schema_validator.validate(json_schema, schema)
+
+By default the index of "array" "items" are ignored. Thus all the values of
+an array should have the same type in order to be compatible.
+This behavior can be disabled by setting "ignore_index = False" in the
+validator's constructor.
+
+>>> import json
+>>> from io import open
+>>>
+>>> import doschema.validation
+>>> from doschema.utils import detect_encoding
+>>>
+>>> schemas = [
+...     './examples/jsonschema_with_index_option.json'
+... ]
+>>>
+>>> schema_validator = doschema.validation.JSONSchemaValidator(
+...     ignore_index = False
+... )
+>>> for schema in schemas:
+...     with open(schema, 'rb') as infile:
+...         byte_file = infile.read()
+...         encoding = detect_encoding(byte_file)
+...         string_file = byte_file.decode(encoding)
+...         json_schema = json.loads(string_file)
+...         schema_validator.validate(json_schema, schema)
+
+CLI usage
+--------------
+.. code-block:: console
+
+    $ doschema validate jsonschema_for_repetition.json \
+    jsonschema_repetition.json
+    $ doschema validate jsonschema_with_index_option.json --with_index
+
+"""
 
 from __future__ import absolute_import, print_function
 

doschema/cli.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of DoSchema.
+# Copyright (C) 2016 CERN.
+#
+# DoSchema is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# DoSchema is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with DoSchema; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+"""CLI commands."""
+
+import json
+from io import open
+
+import click
+
+import doschema.validation
+from doschema.errors import JSONSchemaCompatibilityError
+from doschema.utils import detect_encoding
+
+
+@click.group()
+def cli():
+    """CLI group."""
+    pass  # pragma: no cover
+
+
+@cli.command()
+@click.argument(
+    'schemas',
+    type=click.Path(
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        resolve_path=True
+    ),
+    nargs=-1
+)
+@click.option(
+    '--ignore_index/--with_index',
+    default=True,
+    help="Enable/Disable conflict detection between different indices of "
+    "array fields in JSON-Schemas. Enabled by default."
+)
+def validate(schemas, ignore_index):
+    """Main function for cli."""
+    try:
+        schema_validator = doschema.validation.JSONSchemaValidator(
+            ignore_index)
+        for schema in schemas:
+            with open(schema, 'rb') as infile:
+                byte_file = infile.read()
+                encoding = detect_encoding(byte_file)
+                string_file = byte_file.decode(encoding)
+                json_schema = json.loads(string_file)
+                schema_validator.validate(json_schema, schema)
+    except JSONSchemaCompatibilityError as e:
+        raise click.ClickException(str(e))

doschema/errors.py

@@ -39,9 +39,15 @@ class JSONSchemaCompatibilityError(DoSchemaError):
 
     def __init__(self, err_msg, schema, prev_schema=None):
         """Constructor."""
-        super(JSONSchemaCompatibilityError, self).__init__(err_msg)
-        """Error message."""
-        self.schema = schema
-        """Index of schema in which field occurs now."""
         self.prev_schema = prev_schema
         """Index of schema in which field has occured before."""
+        self.schema = schema
+        """Index of schema in which field occurs now."""
+        super(JSONSchemaCompatibilityError, self).__init__(err_msg)
+        """Error message."""
+
+
+class EncodingError(DoSchemaError):
+    """Exception raised when file encoding is not compatible."""
+
+    pass

doschema/utils.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of DoSchema.
+# Copyright (C) 2016 CERN.
+#
+# DoSchema is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# DoSchema is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with DoSchema; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+"""Utils module."""
+
+import chardet
+
+
+def detect_encoding(byte_file):
+    """Detect encoding of a file with schema."""
+    encoding = chardet.detect(byte_file)['encoding']
+    if encoding in ['UTF-16BE', 'UTF-16LE']:
+        encoding = 'UTF-16'
+    elif encoding in ['UTF-32BE', 'UTF-32LE']:
+        encoding = 'UTF-32'
+    return encoding

examples/cli_example_ignore_option.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of DoSchema.
+# Copyright (C) 2016 CERN.
+#
+# DoSchema is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# DoSchema is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with DoSchema; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+
+"""In this example, there is no option set, so by default
+"--ignore_index" option is enabled.
+Thus array indexes are ignored and for each array field, all items have to be
+of the same type.
+
+Run this example:
+.. code-block:: console
+    $ cd examples
+    $ python app.py
+The same result could be created with the cli:
+.. code-block:: console
+    $ doschema file1.json file2.json
+"""
+
+import json
+from io import open
+
+import doschema.validation
+from doschema.utils import detect_encoding
+
+
+schemas = [
+    './examples/jsonschema_ignore_index_option.json'
+]
+
+schema_validator = doschema.validation.JSONSchemaValidator()
+for schema in schemas:
+    with open(schema, 'rb') as infile:
+        byte_file = infile.read()
+        encoding = detect_encoding(byte_file)
+        string_file = byte_file.decode(encoding)
+        json_schema = json.loads(string_file)
+        schema_validator.validate(json_schema, schema)

examples/cli_example_with_option.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of DoSchema.
+# Copyright (C) 2016 CERN.
+#
+# DoSchema is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# DoSchema is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with DoSchema; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+
+"""In this example, "with_index" option is enabled.
+Thus, types in arrays will be checked with their indexes and items of the same
+array can have different types.
+
+Run this example:
+.. code-block:: console
+    $ cd examples
+    $ python app.py
+The same result could be created with the cli:
+.. code-block:: console
+    $ doschema file1.json file2.json --with_index
+"""
+
+import json
+from io import open
+
+import doschema.validation
+from doschema.utils import detect_encoding
+
+
+schemas = [
+    './examples/jsonschema_with_index_option.json'
+]
+
+schema_validator = doschema.validation.JSONSchemaValidator(ignore_index=False)
+for schema in schemas:
+    with open(schema, 'rb') as infile:
+        byte_file = infile.read()
+        encoding = detect_encoding(byte_file)
+        string_file = byte_file.decode(encoding)
+        json_schema = json.loads(string_file)
+        schema_validator.validate(json_schema, schema)

examples/jsonschema_for_repetition.json

@@ -0,0 +1,8 @@
+{
+    "type":"object",
+    "properties": {
+        "abc": {
+            "type":"integer"
+        }
+    }
+}

examples/jsonschema_ignore_index_option.json

@@ -0,0 +1,25 @@
+{
+    "type":"object",
+    "properties":{
+        "experiment_info":{
+            "type":"array",
+            "items":[
+                {
+                    "type":"object",
+                    "properties":{
+                        "field_A":{
+                            "type":"string"
+                        }
+                    }
+                },{
+                    "type":"object",
+                    "properties":{
+                        "field_A":{
+                            "type":"string"
+                        }
+                    }
+                }
+            ]
+        }
+    }
+}