Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

upgrade-validator #375

Open
wants to merge 29 commits into
base: dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
0e3eaff
rdfjson-bump
keenangraham Feb 1, 2022
f80f5a5
Add some validator tests
keenangraham Feb 1, 2022
8a4e08a
Tests passing
keenangraham Feb 1, 2022
c686df4
Add default and serverDefault functionality
keenangraham Feb 1, 2022
e57c4f5
Hook up new validator
keenangraham Feb 2, 2022
ededd0c
Update test
keenangraham Feb 2, 2022
c49dac5
Update for tests
keenangraham Feb 2, 2022
e317714
Use validator is_type
keenangraham Feb 2, 2022
6074c36
Normalize linkTos in properties
keenangraham Feb 2, 2022
0133996
Normalize in properties
keenangraham Feb 2, 2022
91dc6bc
Use validator is_type
keenangraham Feb 2, 2022
cb1d521
Pass link back
keenangraham Feb 2, 2022
c3fbc30
Update
keenangraham Feb 2, 2022
916fc36
Update tests
keenangraham Feb 3, 2022
547dd07
Remove
keenangraham Feb 3, 2022
b1c33bc
Fix test
keenangraham Feb 3, 2022
41a3dac
Update
keenangraham Feb 3, 2022
a81844b
Filter out requestMethod and permission violations from server defaults
keenangraham Feb 4, 2022
8bf10ac
Change back schema
keenangraham Feb 4, 2022
5fddd4a
Update comment
keenangraham Feb 4, 2022
f0f3227
Update id attribute
keenangraham Feb 4, 2022
65934c4
Update $schema
keenangraham Feb 4, 2022
867d5df
Update
keenangraham Feb 4, 2022
acccb3e
Update dependencies keyword
keenangraham Feb 4, 2022
ad33291
Remove comment
keenangraham Feb 4, 2022
bffb292
Also install format checkers
keenangraham Feb 4, 2022
59839dc
Encode links like in utils
keenangraham Feb 4, 2022
e8156d4
Update requests version
keenangraham Feb 5, 2022
70d383b
Use $id
keenangraham Feb 8, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"elasticsearch-dsl==5.4.0",
"elasticsearch==5.4.0",
"future==0.18.2",
"jsonschema_serialize_fork==2.1.1",
"jsonschema[format]==4.4.0",
"lucenequery==0.1",
"passlib==1.7.2",
"psutil==5.6.7",
Expand All @@ -30,10 +30,10 @@
"pyramid_retry==2.1.1",
"python-magic==0.4.15",
"pytz==2019.3",
"rdflib-jsonld==0.4.0",
"rdflib-jsonld==0.6.0",
"rdflib==4.2.2",
"redis==3.5.3",
"requests==2.22.0",
"requests==2.27.1",
"simplejson==3.17.0",
"snovault-search==1.0.4",
"transaction==3.0.0",
Expand Down
67 changes: 27 additions & 40 deletions src/snovault/schema_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,10 @@
import codecs
import collections
import copy
from jsonschema_serialize_fork import (
Draft4Validator,
FormatChecker,
RefResolver,
)
from jsonschema_serialize_fork.exceptions import ValidationError
from snovault.schema_validation import SerializingSchemaValidator
from jsonschema import FormatChecker
from jsonschema import RefResolver
from jsonschema.exceptions import ValidationError
from uuid import UUID
from .util import ensurelist

Expand Down Expand Up @@ -95,7 +93,6 @@ def linkTo(validator, linkTo, instance, schema):
error = "%r is not of type %s" % (instance, ", ".join(reprs))
yield ValidationError(error)
return

linkEnum = schema.get('linkEnum')
if linkEnum is not None:
if not validator.is_type(linkEnum, "array"):
Expand Down Expand Up @@ -123,10 +120,6 @@ def linkTo(validator, linkTo, instance, schema):
yield ValidationError(error)
return

# And normalize the value to a uuid
if validator._serialize:
validator._validated[-1] = str(item.uuid)


def linkFrom(validator, linkFrom, instance, schema):
# avoid circular import
Expand Down Expand Up @@ -154,9 +147,6 @@ def linkFrom(validator, linkFrom, instance, schema):
yield ValidationError(error)
return
else:
if validator._serialize:
lv = len(validator._validated)

# Look for an existing item;
# if found use the schema for its type,
# which may be a subtype of an abstract linkType
Expand Down Expand Up @@ -210,16 +200,13 @@ def linkFrom(validator, linkFrom, instance, schema):
for error in validator.descend(instance, subschema):
yield error

if validator._serialize:
validated_instance = validator._validated[lv]
del validator._validated[lv:]
if uuid is not None:
validated_instance['uuid'] = uuid
elif 'uuid' in validated_instance: # where does this come from?
del validated_instance['uuid']
if new_type is not None:
validated_instance['@type'] = [new_type]
validator._validated[-1] = validated_instance
validated_instance = instance
if uuid is not None:
validated_instance['uuid'] = uuid
elif 'uuid' in validated_instance: # where does this come from?
del validated_instance['uuid']
if new_type is not None:
validated_instance['@type'] = [new_type]


class IgnoreUnchanged(ValidationError):
Expand Down Expand Up @@ -250,17 +237,6 @@ def permission(validator, permission, instance, schema):
yield IgnoreUnchanged(error)


orig_uniqueItems = Draft4Validator.VALIDATORS['uniqueItems']


def uniqueItems(validator, uI, instance, schema):
# Use serialized items if available
# (this gives the linkTo validator a chance to normalize paths into uuids)
if validator._serialize and validator._validated[-1]:
instance = validator._validated[-1]
yield from orig_uniqueItems(validator, uI, instance, schema)


VALIDATOR_REGISTRY = {}


Expand All @@ -281,16 +257,15 @@ def notSubmittable(validator, linkTo, instance, schema):
yield ValidationError('submission disallowed')


class SchemaValidator(Draft4Validator):
VALIDATORS = Draft4Validator.VALIDATORS.copy()
class SchemaValidator(SerializingSchemaValidator):
VALIDATORS = SerializingSchemaValidator.VALIDATORS.copy()
VALIDATORS['notSubmittable'] = notSubmittable
# for backwards-compatibility
VALIDATORS['calculatedProperty'] = notSubmittable
VALIDATORS['linkTo'] = linkTo
VALIDATORS['linkFrom'] = linkFrom
VALIDATORS['permission'] = permission
VALIDATORS['requestMethod'] = requestMethod
VALIDATORS['uniqueItems'] = uniqueItems
VALIDATORS['validators'] = validators
SERVER_DEFAULTS = SERVER_DEFAULTS

Expand All @@ -311,13 +286,13 @@ def load_schema(filename):
schema = mixinProperties(schema, resolver)

# SchemaValidator is not thread safe for now
SchemaValidator(schema, resolver=resolver, serialize=True)
SchemaValidator(schema, resolver=resolver)
return schema


def validate(schema, data, current=None):
resolver = NoRemoteResolver.from_schema(schema)
sv = SchemaValidator(schema, resolver=resolver, serialize=True, format_checker=format_checker)
sv = SchemaValidator(schema, resolver=resolver, format_checker=format_checker)
validated, errors = sv.serialize(data)

filtered_errors = []
Expand All @@ -336,6 +311,18 @@ def validate(schema, data, current=None):
validated_value = validated_value[key]
if validated_value == current_value:
continue
# Also ignore requestMethod and permission errors from defaults.
if isinstance(error, IgnoreUnchanged):
current_value = data
try:
for key in error.path:
# If it's in original data then either user passed it in
# or it's from PATCH object with unchanged data. If it's
# unchanged then it's already been skipped above.
current_value = current_value[key]
except KeyError:
# If it's not in original data then it's filled in by defaults.
continue
filtered_errors.append(error)

return validated, filtered_errors
Expand Down
143 changes: 143 additions & 0 deletions src/snovault/schema_validation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
from copy import deepcopy
from jsonschema import Draft202012Validator
from jsonschema import validators
from jsonschema.exceptions import ValidationError
from pyramid.threadlocal import get_current_request
from pyramid.traversal import find_resource


NO_DEFAULT = object()


def get_resource_base(validator, linkTo):
from snovault import COLLECTIONS
request = get_current_request()
collections = request.registry[COLLECTIONS]
if validator.is_type(linkTo, 'string'):
resource_base = collections.get(linkTo, request.root)
else:
resource_base = request.root
return resource_base


def normalize_links(validator, links, linkTo):
resource_base = get_resource_base(validator, linkTo)
normalized_links = []
errors = []
for link in links:
try:
normalized_links.append(
str(find_resource(resource_base, link.replace(':', '%3A')).uuid)
)
except KeyError:
errors.append(
ValidationError(f'Unable to resolve link: {link}')
)
normalized_links.append(
link
)
return normalized_links, errors


def should_mutate_properties(validator, instance):
if validator.is_type(instance, 'object'):
return True
return False


def get_items_or_empty_object(validator, subschema):
items = subschema.get('items', {})
if validator.is_type(items, 'object'):
return items
return {}


def maybe_normalize_links_to_uuids(validator, property, subschema, instance):
errors = []
if 'linkTo' in subschema:
link = instance.get(property)
if link:
normalized_links, errors = normalize_links(
validator,
[link],
subschema.get('linkTo'),
)
instance[property] = normalized_links[0]
if 'linkTo' in get_items_or_empty_object(validator, subschema):
links = instance.get(property, [])
if links:
normalized_links, errors = normalize_links(
validator,
links,
subschema.get('items').get('linkTo'),
)
instance[property] = normalized_links
for error in errors:
yield error


def set_defaults(validator, property, subschema, instance):
if 'default' in subschema:
instance.setdefault(
property,
deepcopy(subschema['default'])
)
if 'serverDefault' in subschema:
server_default = validator.server_default(
instance,
subschema
)
if server_default is not NO_DEFAULT:
instance.setdefault(
property,
server_default
)


def extend_with_default(validator_class):
validate_properties = validator_class.VALIDATORS['properties']

def mutate_properties(validator, properties, instance, schema):
for property, subschema in properties.items():
if not validator.is_type(subschema, 'object'):
continue
yield from maybe_normalize_links_to_uuids(validator, property, subschema, instance)
set_defaults(validator, property, subschema, instance)

def before_properties_validation_hook(validator, properties, instance, schema):
if should_mutate_properties(validator, instance):
yield from mutate_properties(validator, properties, instance, schema)
yield from validate_properties(validator, properties, instance, schema)

return validators.extend(
validator_class, {'properties': before_properties_validation_hook},
)


ExtendedValidator = extend_with_default(Draft202012Validator)


class SerializingSchemaValidator(ExtendedValidator):

SERVER_DEFAULTS = {}

def add_server_defaults(self, server_defaults):
self.SERVER_DEFAULTS.update(server_defaults)
return self

def serialize(self, instance):
self._original_instance = instance
self._mutated_instance = deepcopy(
self._original_instance
)
errors = list(
self.iter_errors(
self._mutated_instance
)
)
return self._mutated_instance, errors

def server_default(self, instance, subschema):
factory_name = subschema['serverDefault']
factory = self.SERVER_DEFAULTS[factory_name]
return factory(instance, subschema)
4 changes: 2 additions & 2 deletions src/snovault/schema_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ def schemas_map(context, request):
types = request.registry[TYPES]
profiles_map = {}
for type_info in types.by_item_type.values():
if 'id' in type_info.schema:
profiles_map[type_info.name] = type_info.schema['id']
if '$id' in type_info.schema:
profiles_map[type_info.name] = type_info.schema['$id']
profiles_map['@type'] = ['JSONSchemas']
return profiles_map

Expand Down
5 changes: 4 additions & 1 deletion src/snovault/tests/test_post_put_patch.py
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,10 @@ def test_put_object_child_validation(content_with_child, testapp):
}]
}
res = testapp.put_json(content_with_child['@id'], edit, status=422)
assert res.json['errors'][0]['name'] == [u'reverse', 0, u'target']
assert ['reverse', 0, 'target'] in [
x['name']
for x in res.json['errors']
]


def test_put_object_validates_child_references(content_with_child, testapp):
Expand Down
12 changes: 8 additions & 4 deletions src/snovault/tests/test_schema_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,21 @@ def content(testapp):

def test_uniqueItems_validates_normalized_links(content, threadlocals):
schema = {
'uniqueItems': True,
'items': {
'linkTo': 'TestingLinkTarget',
'properties': {
'some_links': {
'uniqueItems': True,
'items': {
'linkTo': 'TestingLinkTarget',
}
}
}
}
uuid = targets[0]['uuid']
data = [
uuid,
'/testing-link-targets/{}'.format(uuid),
]
validated, errors = validate(schema, data)
validated, errors = validate(schema, {'some_links': data})
assert len(errors) == 1
assert (
errors[0].message == "['{}', '{}'] has non-unique elements".format(
Expand Down
Loading