Skip to content

Commit

Permalink
Jsonschema 4.18+ is now required.
Browse files Browse the repository at this point in the history
  • Loading branch information
jpmckinney authored and odscjames committed Dec 6, 2023
1 parent b508c45 commit d09c290
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 47 deletions.
5 changes: 3 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
### Removed

- Dropped support for Python 3.6 & 3.7, as these are now end of life.
- Drop jsonschema 3 support
- Jsonschema 4.18+ is now required. Support for 3 and older versions of 4 is removed.

### Changed

- Restore jsonschema's type validator, as its performance has improved in recent Python versions https://github.com/OpenDataServices/lib-cove/pull/127
- Allow `SchemaJsonMixin` classes to define a `validator` method, that accepts lib-cove's JSON Schema draft 4 validator class and its format checker, and returns a validator instance. https://github.com/OpenDataServices/lib-cove/pull/128
- Allow `SchemaJsonMixin` classes to define a `registry` value, TODO

### Fixed

- Calculate additional codelist values for schema using `anyOf` or `oneOf`, like OCDS record packages https://github.com/open-contracting/lib-cove-ocds/issues/106
- Descend into nullable objects and arrays. (For example, OCDS `parties/details` is nullable, and additional codes for `parties/details/scale` were unreported.) https://github.com/OpenDataServices/lib-cove/pull/131
- Process subschemas with our custom validator. Fixes an issue in later versions of Jsonschema.

## [0.31.0] - 2023-07-06

Expand Down
63 changes: 26 additions & 37 deletions libcove/lib/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,21 @@
import jsonref
import jsonschema.validators
import requests
from flattentool import unflatten
from jsonschema import FormatChecker
from jsonschema._utils import extras_msg, find_additional_properties, uniq
from jsonschema.exceptions import UndefinedTypeCheck, ValidationError
from referencing import Registry, Resource

from .exceptions import cove_spreadsheet_conversion_error
from .tools import decimal_default, get_request

try:
from functools import cached_property
except ImportError:
from cached_property import cached_property

from flattentool import unflatten
from jsonschema import FormatChecker, RefResolver
from jsonschema._utils import extras_msg, find_additional_properties, uniq
from jsonschema.exceptions import UndefinedTypeCheck, ValidationError

from .exceptions import cove_spreadsheet_conversion_error
from .tools import decimal_default, get_request

REQUIRED_RE = re.compile(r"^'([^']+)'")

Expand Down Expand Up @@ -813,14 +815,8 @@ def get_schema_validation_errors(
if extra_checkers:
format_checker.checkers.update(extra_checkers)

# Force jsonschema to use our validator.
# https://github.com/python-jsonschema/jsonschema/issues/994
jsonschema.validators.validates("http://json-schema.org/draft-04/schema#")(
validator
)

if hasattr(schema_obj, "validator"):
our_validator = schema_obj.validator(validator, format_checker)
if hasattr(schema_obj, "registry"):
registry = schema_obj.registry
else:
if getattr(schema_obj, "extended", None):
resolver = CustomRefResolver(
Expand All @@ -839,9 +835,17 @@ def get_schema_validation_errors(
schema_url=schema_obj.schema_host,
)

our_validator = validator(
pkg_schema_obj, format_checker=format_checker, resolver=resolver
)
registry = Registry(retrieve=resolver.retrieve)

# Force jsonschema to use our validator.
# https://github.com/python-jsonschema/jsonschema/issues/994
jsonschema.validators.validates("http://json-schema.org/draft-04/schema#")(
validator
)

our_validator = validator(
pkg_schema_obj, format_checker=format_checker, registry=registry
)

for e in our_validator.iter_errors(json_data):
message = e.message
Expand Down Expand Up @@ -1165,7 +1169,7 @@ def get_fields_present(*args, **kwargs):
}


class CustomRefResolver(RefResolver):
class CustomRefResolver:
"""This RefResolver is only for use with the jsonschema library"""

def __init__(self, *args, **kw):
Expand All @@ -1178,44 +1182,29 @@ def __init__(self, *args, **kw):
# this is ignored when you supply a file
self.schema_url = kw.pop("schema_url", "")
self.config = kw.pop("config", "")
super().__init__(*args, **kw)

def resolve_remote(self, uri):
def retrieve(self, uri):
schema_name = uri.split("/")[-1]
if self.schema_file and self.file_schema_name == schema_name:
uri = self.schema_file
else:
uri = urljoin(self.schema_url, schema_name)

document = self.store.get(uri)

if document:
return document
if uri.startswith("http"):
# This branch of the if-statement in-lines `RefResolver.resolve_remote()`, but using `get_request()`.
# https://github.com/python-jsonschema/jsonschema/blob/dbc398245a583cb2366795dc529ae042d10c1577/jsonschema/validators.py#L1008-L1023
scheme = urlsplit(uri).scheme

if scheme in self.handlers:
result = self.handlers[scheme](uri)
elif scheme in ["http", "https"]:
# Requests has support for detecting the correct encoding of
# json over http
if scheme in ("http", "https"):
result = get_request(uri, config=self.config).json()
else:
# Otherwise, pass off to urllib and assume utf-8
with urlopen(uri) as url:
result = json.loads(url.read().decode("utf-8"))

if self.cache_remote:
self.store[uri] = result
return result
else:
with open(uri) as schema_file:
result = json.load(schema_file)

add_is_codelist(result)
self.store[uri] = result
return result
return Resource.from_contents(result)


def _get_schema_deprecated_paths(
Expand Down
3 changes: 2 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
long_description="A data review library",
install_requires=[
"jsonref",
"jsonschema>=4",
"jsonschema>=4.18",
"referencing",
"requests",
"cached-property;python_version<'3.8'",
"flattentool>=0.11.0",
Expand Down
11 changes: 4 additions & 7 deletions tests/lib/test_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import jsonschema
import pytest
from freezegun import freeze_time
from referencing.exceptions import CannotDetermineSpecification

from libcove.lib.common import (
SchemaJsonMixin,
Expand Down Expand Up @@ -766,7 +767,7 @@ def get_pkg_schema_obj(self):
assert "[Decimal('3.1')] is too short" in validation_error_json


def test_property_that_is_not_json_schema_doesnt_raise_exception(caplog, tmpdir):
def test_property_that_is_not_json_schema_does_raise_exception(tmpdir):
tmpdir.join("test.json").write(
json.dumps({"properties": {"bad_property": "not_a_json_schema"}})
)
Expand All @@ -778,12 +779,8 @@ class DummySchemaObj:
def get_pkg_schema_obj(self):
return {"$ref": "test.json"}

validation_errors = get_schema_validation_errors({}, DummySchemaObj(), "", {}, {})
assert validation_errors == {}
assert (
"A 'properties' object contains a 'bad_property' value that is not a JSON Schema: 'not_a_json_schema'"
in caplog.text
)
with pytest.raises(CannotDetermineSpecification):
get_schema_validation_errors({}, DummySchemaObj(), "", {}, {})


@pytest.mark.parametrize(
Expand Down

0 comments on commit d09c290

Please sign in to comment.