diff --git a/INSTALL.rst b/INSTALL.rst index 11e4437..fab9d5b 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -1,2 +1,9 @@ Installation ============ +Invenio's DoSchema module is on PyPI so all you need is: + +.. code-block:: console + + $ pip install doschema + + diff --git a/MANIFEST.in b/MANIFEST.in index 7ba214f..2fd590f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -45,3 +45,6 @@ recursive-include docs *.py recursive-include docs *.rst recursive-include docs Makefile recursive-include tests *.py +recursive-include examples *.py +recursive-include examples *.json +recursive-include tests *.json diff --git a/docs/conf.py b/docs/conf.py index 4e448e4..f933da5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,6 @@ import os -import sphinx.environment # -- General configuration ------------------------------------------------ diff --git a/docs/usage.rst b/docs/usage.rst index 3db6fda..59d7c9d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -22,6 +22,7 @@ as an Intergovernmental Organization or submit itself to any jurisdiction. +===== Usage ===== diff --git a/doschema/__init__.py b/doschema/__init__.py index a90011d..fa5d820 100644 --- a/doschema/__init__.py +++ b/doschema/__init__.py @@ -22,7 +22,72 @@ # waive the privileges and immunities granted to it by virtue of its status # as an Intergovernmental Organization or submit itself to any jurisdiction. -"""JSON Schema utility functions and commands.""" +r"""JSON Schema utility functions and commands. + +Compatibility Validation +------------------------- + +It validates compatibility between different JSON schemas versions. + +A schema is backward compatible if the fields' type remain the same in all +JSON schemas declaring it and JSON schemas are type consistent within +themselves too. + +>>> import json +>>> from io import open +>>> +>>> import doschema.validation +>>> from doschema.utils import detect_encoding +>>> +>>> schemas = [ +... './examples/jsonschema_for_repetition.json', +... './examples/jsonschema_repetition.json' +... ] +>>> +>>> schema_validator = doschema.validation.JSONSchemaValidator() +>>> for schema in schemas: +... with open(schema, 'rb') as infile: +... byte_file = infile.read() +... encoding = detect_encoding(byte_file) +... string_file = byte_file.decode(encoding) +... json_schema = json.loads(string_file) +... schema_validator.validate(json_schema, schema) + +By default the index of "array" "items" are ignored. Thus all the values of +an array should have the same type in order to be compatible. +This behavior can be disabled by setting "ignore_index = False" in the +validator's constructor. + +>>> import json +>>> from io import open +>>> +>>> import doschema.validation +>>> from doschema.utils import detect_encoding +>>> +>>> schemas = [ +... './examples/jsonschema_with_index_option.json' +... ] +>>> +>>> schema_validator = doschema.validation.JSONSchemaValidator( +... ignore_index = False +... ) +>>> for schema in schemas: +... with open(schema, 'rb') as infile: +... byte_file = infile.read() +... encoding = detect_encoding(byte_file) +... string_file = byte_file.decode(encoding) +... json_schema = json.loads(string_file) +... schema_validator.validate(json_schema, schema) + +CLI usage +-------------- +.. code-block:: console + + $ doschema validate jsonschema_for_repetition.json \ + jsonschema_repetition.json + $ doschema validate jsonschema_with_index_option.json --with_index + +""" from __future__ import absolute_import, print_function diff --git a/doschema/cli.py b/doschema/cli.py new file mode 100644 index 0000000..9387a85 --- /dev/null +++ b/doschema/cli.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# +# This file is part of DoSchema. +# Copyright (C) 2016 CERN. +# +# DoSchema is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# DoSchema is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with DoSchema; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +"""CLI commands.""" + +import json +from io import open + +import click + +import doschema.validation +from doschema.errors import JSONSchemaCompatibilityError +from doschema.utils import detect_encoding + + +@click.group() +def cli(): + """CLI group.""" + pass # pragma: no cover + + +@cli.command() +@click.argument( + 'schemas', + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + resolve_path=True + ), + nargs=-1 +) +@click.option( + '--ignore_index/--with_index', + default=True, + help="Enable/Disable conflict detection between different indices of " + "array fields in JSON-Schemas. Enabled by default." +) +def validate(schemas, ignore_index): + """Main function for cli.""" + try: + schema_validator = doschema.validation.JSONSchemaValidator( + ignore_index) + for schema in schemas: + with open(schema, 'rb') as infile: + byte_file = infile.read() + encoding = detect_encoding(byte_file) + string_file = byte_file.decode(encoding) + json_schema = json.loads(string_file) + schema_validator.validate(json_schema, schema) + except JSONSchemaCompatibilityError as e: + raise click.ClickException(str(e)) diff --git a/doschema/errors.py b/doschema/errors.py index fc93482..d22c2cd 100644 --- a/doschema/errors.py +++ b/doschema/errors.py @@ -39,9 +39,15 @@ class JSONSchemaCompatibilityError(DoSchemaError): def __init__(self, err_msg, schema, prev_schema=None): """Constructor.""" - super(JSONSchemaCompatibilityError, self).__init__(err_msg) - """Error message.""" - self.schema = schema - """Index of schema in which field occurs now.""" self.prev_schema = prev_schema """Index of schema in which field has occured before.""" + self.schema = schema + """Index of schema in which field occurs now.""" + super(JSONSchemaCompatibilityError, self).__init__(err_msg) + """Error message.""" + + +class EncodingError(DoSchemaError): + """Exception raised when file encoding is not compatible.""" + + pass diff --git a/doschema/utils.py b/doschema/utils.py new file mode 100644 index 0000000..b4170fc --- /dev/null +++ b/doschema/utils.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# +# This file is part of DoSchema. +# Copyright (C) 2016 CERN. +# +# DoSchema is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# DoSchema is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with DoSchema; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +"""Utils module.""" + +import chardet + + +def detect_encoding(byte_file): + """Detect encoding of a file with schema.""" + encoding = chardet.detect(byte_file)['encoding'] + if encoding in ['UTF-16BE', 'UTF-16LE']: + encoding = 'UTF-16' + elif encoding in ['UTF-32BE', 'UTF-32LE']: + encoding = 'UTF-32' + return encoding diff --git a/examples/cli_example_ignore_option.py b/examples/cli_example_ignore_option.py new file mode 100644 index 0000000..45b7c83 --- /dev/null +++ b/examples/cli_example_ignore_option.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# +# This file is part of DoSchema. +# Copyright (C) 2016 CERN. +# +# DoSchema is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# DoSchema is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with DoSchema; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + + +"""In this example, there is no option set, so by default +"--ignore_index" option is enabled. +Thus array indexes are ignored and for each array field, all items have to be +of the same type. + +Run this example: +.. code-block:: console + $ cd examples + $ python app.py +The same result could be created with the cli: +.. code-block:: console + $ doschema file1.json file2.json +""" + +import json +from io import open + +import doschema.validation +from doschema.utils import detect_encoding + + +schemas = [ + './examples/jsonschema_ignore_index_option.json' +] + +schema_validator = doschema.validation.JSONSchemaValidator() +for schema in schemas: + with open(schema, 'rb') as infile: + byte_file = infile.read() + encoding = detect_encoding(byte_file) + string_file = byte_file.decode(encoding) + json_schema = json.loads(string_file) + schema_validator.validate(json_schema, schema) diff --git a/examples/cli_example_with_option.py b/examples/cli_example_with_option.py new file mode 100644 index 0000000..fc80494 --- /dev/null +++ b/examples/cli_example_with_option.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# This file is part of DoSchema. +# Copyright (C) 2016 CERN. +# +# DoSchema is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# DoSchema is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with DoSchema; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + + +"""In this example, "with_index" option is enabled. +Thus, types in arrays will be checked with their indexes and items of the same +array can have different types. + +Run this example: +.. code-block:: console + $ cd examples + $ python app.py +The same result could be created with the cli: +.. code-block:: console + $ doschema file1.json file2.json --with_index +""" + +import json +from io import open + +import doschema.validation +from doschema.utils import detect_encoding + + +schemas = [ + './examples/jsonschema_with_index_option.json' +] + +schema_validator = doschema.validation.JSONSchemaValidator(ignore_index=False) +for schema in schemas: + with open(schema, 'rb') as infile: + byte_file = infile.read() + encoding = detect_encoding(byte_file) + string_file = byte_file.decode(encoding) + json_schema = json.loads(string_file) + schema_validator.validate(json_schema, schema) diff --git a/examples/jsonschema_for_repetition.json b/examples/jsonschema_for_repetition.json new file mode 100644 index 0000000..0ac4126 --- /dev/null +++ b/examples/jsonschema_for_repetition.json @@ -0,0 +1,8 @@ +{ + "type":"object", + "properties": { + "abc": { + "type":"integer" + } + } +} diff --git a/examples/jsonschema_ignore_index_option.json b/examples/jsonschema_ignore_index_option.json new file mode 100644 index 0000000..71a3ffb --- /dev/null +++ b/examples/jsonschema_ignore_index_option.json @@ -0,0 +1,25 @@ +{ + "type":"object", + "properties":{ + "experiment_info":{ + "type":"array", + "items":[ + { + "type":"object", + "properties":{ + "field_A":{ + "type":"string" + } + } + },{ + "type":"object", + "properties":{ + "field_A":{ + "type":"string" + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/examples/jsonschema_no_repetition.json b/examples/jsonschema_no_repetition.json new file mode 100644 index 0000000..3cdefec --- /dev/null +++ b/examples/jsonschema_no_repetition.json @@ -0,0 +1,8 @@ +{ + "type":"object", + "properties": { + "abc": { + "type":"string" + } + } +} diff --git a/examples/jsonschema_other_ref.json b/examples/jsonschema_other_ref.json new file mode 100644 index 0000000..658d85a --- /dev/null +++ b/examples/jsonschema_other_ref.json @@ -0,0 +1,34 @@ +{ + "$schema":"http://json-schema.org/draft-04/schema#", + "definitions":{ + "address":{ + "type":"object", + "properties":{ + "street_address":{ + "type":"differenttype" + } + } + } + }, + "type":"object", + "properties":{ + "billing_address":{ + "$ref":"#/definitions/address" + }, + "shipping_address":{ + "allOf":[ + {"$ref":"#/definitions/address"}, + { + "properties":{ + "type":{ + "enum":[ + "residential", + "business" + ] + } + } + } + ] + } + } +} diff --git a/examples/jsonschema_ref.json b/examples/jsonschema_ref.json new file mode 100644 index 0000000..434b676 --- /dev/null +++ b/examples/jsonschema_ref.json @@ -0,0 +1,34 @@ +{ + "$schema":"http://json-schema.org/draft-04/schema#", + "definitions":{ + "address":{ + "type":"object", + "properties":{ + "street_address":{ + "type":"string" + } + } + } + }, + "type":"object", + "properties":{ + "billing_address":{ + "$ref":"#/definitions/address" + }, + "shipping_address":{ + "allOf":[ + {"$ref":"#/definitions/address"}, + { + "properties":{ + "type":{ + "enum":[ + "residential", + "business" + ] + } + } + } + ] + } + } +} diff --git a/examples/jsonschema_repetition.json b/examples/jsonschema_repetition.json new file mode 100644 index 0000000..8d2c90c --- /dev/null +++ b/examples/jsonschema_repetition.json @@ -0,0 +1,8 @@ +{ + "type": "object", + "properties": { + "abc": { + "type": "integer" + } + } +} diff --git a/examples/jsonschema_with_index_option.json b/examples/jsonschema_with_index_option.json new file mode 100644 index 0000000..58cc514 --- /dev/null +++ b/examples/jsonschema_with_index_option.json @@ -0,0 +1,25 @@ +{ + "type":"object", + "properties":{ + "experiment_info":{ + "type":"array", + "items":[ + { + "type":"object", + "properties":{ + "field_A":{ + "type":"string" + } + } + },{ + "type":"object", + "properties":{ + "field_A":{ + "type":"number" + } + } + } + ] + } + } +} diff --git a/setup.py b/setup.py index 1eccc9b..b6905c0 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,9 @@ install_requires = [ 'jsonschema>=2.5.1', + 'click>=5.1', + 'chardet>=2.3.0', + ] packages = find_packages() @@ -85,6 +88,9 @@ include_package_data=True, platforms='any', entry_points={ + 'console_scripts': [ + 'doschema = doschema.cli:cli', + ], }, extras_require=extras_require, install_requires=install_requires, diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..122ab6f --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# +# This file is part of DoSchema. +# Copyright (C) 2016 CERN. +# +# DoSchema is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# DoSchema is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with DoSchema; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +import os + +from click.testing import CliRunner + +from doschema.cli import validate + + +def path_helper(filename): + """Make the path for test files.""" + return os.path.join('.', 'examples', filename) + + +def test_repetition(): + """Test that adding field with the same type passes.""" + runner = CliRunner() + schemas = [ + 'jsonschema_for_repetition.json', + 'jsonschema_repetition.json' + ] + + files = [path_helper(filename) for filename in schemas] + result = runner.invoke(validate, files) + + assert result.exit_code == 0 + + +def test_difference(): + """Test that adding field with different type fails.""" + runner = CliRunner() + schemas = [ + 'jsonschema_for_repetition.json', + 'jsonschema_no_repetition.json' + ] + + files = [path_helper(filename) for filename in schemas] + result = runner.invoke(validate, files) + + assert result.exit_code == 1 + + +def test_ref(): + """Test that references works.""" + runner = CliRunner() + schemas = ['jsonschema_ref.json', 'jsonschema_other_ref.json'] + + files = [path_helper(filename) for filename in schemas] + result = runner.invoke(validate, files) + + assert result.exit_code == 1 + + +def test_ignore_option(): + """Test that with no option "--ignore_index" option is set.""" + runner = CliRunner() + schemas = ['jsonschema_ignore_index_option.json'] + + files = [path_helper(filename) for filename in schemas] + result = runner.invoke(validate, files) + + assert result.exit_code == 0 + + +def test_with_option(): + """Test that "--with_index" option references works.""" + runner = CliRunner() + schemas = ['jsonschema_with_index_option.json'] + + files = [path_helper(filename) for filename in schemas] + files.append('--with_index') + result = runner.invoke(validate, files) + assert result.exit_code == 0