From a5d4fa0971092e16b121aad84dd343a46ea84e21 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Fri, 25 Sep 2020 18:39:12 +1000 Subject: [PATCH 01/10] wip Added first major functionality for shacl-js features --- pyproject.toml | 5 +- pyshacl/__init__.py | 2 +- pyshacl/extras/__init__.py | 32 +++ pyshacl/extras/js/__init__.py | 7 + pyshacl/extras/js/constraint.py | 194 +++++++++++++ pyshacl/extras/js/context.py | 468 +++++++++++++++++++++++++++++++ pyshacl/extras/js/loader.py | 40 +++ pyshacl/functions/__init__.py | 2 + pyshacl/shape.py | 13 +- pyshacl/shapes_graph.py | 8 + pyshacl/validate.py | 15 +- test/js/test_js_constraint.py | 38 +++ test/resources/js/germanLabel.js | 16 ++ 13 files changed, 833 insertions(+), 7 deletions(-) create mode 100644 pyshacl/extras/__init__.py create mode 100644 pyshacl/extras/js/__init__.py create mode 100644 pyshacl/extras/js/constraint.py create mode 100644 pyshacl/extras/js/context.py create mode 100644 pyshacl/extras/js/loader.py create mode 100644 test/js/test_js_constraint.py create mode 100644 test/resources/js/germanLabel.js diff --git a/pyproject.toml b/pyproject.toml index 9b0b9ae..564b880 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.masonry.api" [tool.poetry] name = "pyshacl" -version = "0.13.3" +version = "0.14.0" # Don't forget to change the version number in __init__.py along with this one description = "Python SHACL Validator" license = "Apache-2.0" @@ -47,6 +47,7 @@ python = "^3.6" # Compatible python versions must be declared here rdflib = "^5.0.0" rdflib-jsonld = "^0.5.0" owlrl = "^5.2.1" +pyduktape2 = {version="^0.3.1", optional=true} [tool.poetry.dev-dependencies] coverage = "^4.5" @@ -56,9 +57,11 @@ flake8 = "^3.7" isort = {version="^5.0.0", python=">=3.6"} black = {version=">=18.9b0,<=19.10b0", python=">=3.6"} mypy = {version="^0.770.0", python=">=3.6"} +mypy = {version="^0.770.0", python=">=3.6"} [tool.poetry.extras] +js = ["pyduktape2"] dev-lint = ["isort", "black", "flake8"] dev-type-checking = ["mypy"] diff --git a/pyshacl/__init__.py b/pyshacl/__init__.py index b472b73..d0f45fa 100644 --- a/pyshacl/__init__.py +++ b/pyshacl/__init__.py @@ -6,7 +6,7 @@ # version compliant with https://www.python.org/dev/peps/pep-0440/ -__version__ = '0.13.3' +__version__ = '0.14.0' # Don't forget to change the version number in pyproject.toml along with this one __all__ = ['validate', 'Validator', '__version__', 'Shape', 'ShapesGraph'] diff --git a/pyshacl/extras/__init__.py b/pyshacl/extras/__init__.py new file mode 100644 index 0000000..992bb29 --- /dev/null +++ b/pyshacl/extras/__init__.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# +import typing +from functools import lru_cache +from warnings import warn +import pkg_resources +from pkg_resources import UnknownExtra, DistributionNotFound + +dev_mode = True +@lru_cache() +def check_extra_installed(extra_name: str): + if dev_mode: + return True + check_name = "pyshacl["+extra_name+"]" + # first check if pyshacl is installed using the normal means + try: + res1 = pkg_resources.require("pyshacl") + except DistributionNotFound: + # Hmm, it thinks pyshacl isn't installed. Can't even check for extras + return None + try: + res2 = pkg_resources.require(check_name) + return True + except UnknownExtra: + # That extra doesn't exist in this version of pyshacl + warn(Warning("Extra \"{}\" doesn't exist in this version of pyshacl.".format(extra_name))) + return False + except DistributionNotFound: + # That extra is not installed right now + return False + except BaseException: + raise diff --git a/pyshacl/extras/js/__init__.py b/pyshacl/extras/js/__init__.py new file mode 100644 index 0000000..9d832f3 --- /dev/null +++ b/pyshacl/extras/js/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +# + +from .loader import load_into_context + + + diff --git a/pyshacl/extras/js/constraint.py b/pyshacl/extras/js/constraint.py new file mode 100644 index 0000000..9b23544 --- /dev/null +++ b/pyshacl/extras/js/constraint.py @@ -0,0 +1,194 @@ +import typing +from typing import List, Dict +from rdflib import Literal +if typing.TYPE_CHECKING: + from pyshacl.shapes_graph import ShapesGraph +from pyshacl.constraints import ConstraintComponent +from pyshacl.consts import SH +from pyshacl.errors import ConstraintLoadError +from pyshacl.rdfutil import stringify_node +from pyshacl.pytypes import GraphLike +from .context import SHACLJSContext + + +SH_js = SH.term('js') +SH_jsFunctionName = SH.term('jsFunctionName') +SH_jsLibrary = SH.term('jsLibrary') +SH_jsLibraryURL = SH.term('jsLibraryURL') +SH_JSConstraintComponent = SH.term('JSConstraintComponent') + + +class JSExecutable(object): + __slots__ = ("sg","node","fn_name","libraries") + + + def __init__(self, sg: 'ShapesGraph', node): + self.node = node + self.sg = sg + fn_names = set(sg.objects(node, SH_jsFunctionName)) + if len(fn_names) < 1: + raise ConstraintLoadError( + "At least one sh:jsFunctionName must be present on a JS Executable.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + elif len(fn_names) > 1: + raise ConstraintLoadError( + "At most one sh:jsFunctionName can be present on a JS Executable.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + fn_name = next(iter(fn_names)) + if not isinstance(fn_name, Literal): + raise ConstraintLoadError( + "sh:jsFunctionName must be an RDF Literal with type xsd:string.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + else: + fn_name = str(fn_name) + self.fn_name = fn_name + library_defs = sg.objects(node, SH_jsLibrary) + seen_library_defs = [] + libraries = {} + for libn in library_defs: + if libn in seen_library_defs: + continue + if isinstance(libn, Literal): + raise ConstraintLoadError( + "sh:jsLibrary must not have a value that is a Literal.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + seen_library_defs.append(libn) + jsLibraryURLs = list(sg.objects(libn, SH_jsLibraryURL)) + if len(jsLibraryURLs) > 0: + libraries[libn] = libraries.get(libn, []) + for u in jsLibraryURLs: + if not isinstance(u, Literal): + raise ConstraintLoadError( + "sh:jsLibraryURL must have a value that is a Literal.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + libraries[libn].append(str(u)) + library_defs2 = sg.objects(libn, SH_jsLibrary) + for libn2 in library_defs2: + if libn2 in seen_library_defs: + continue + if isinstance(libn2, Literal): + raise ConstraintLoadError( + "sh:jsLibrary must not have a value that is a Literal.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + seen_library_defs.append(libn2) + jsLibraryURLs2 = list(sg.objects(libn2, SH_jsLibraryURL)) + if len(jsLibraryURLs2) > 0: + libraries[libn2] = libraries.get(libn2, []) + for u2 in jsLibraryURLs2: + if not isinstance(u2, Literal): + raise ConstraintLoadError( + "sh:jsLibraryURL must have a value that is a Literal.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + libraries[libn2].append(str(u2)) + self.libraries = libraries + + def execute(self, datagraph, *args, **kwargs): + ctx = SHACLJSContext(self.sg, datagraph, **kwargs) + for lib_node, lib_urls in self.libraries.items(): + for lib_url in lib_urls: + ctx.load_js_library(lib_url) + return ctx.run_js_function(self.fn_name, args) + +class JSConstraintComponent(ConstraintComponent): + """ + sh:minCount specifies the minimum number of value nodes that satisfy the condition. If the minimum cardinality value is 0 then this constraint is always satisfied and so may be omitted. + Link: + https://www.w3.org/TR/shacl/#MinCountConstraintComponent + Textual Definition: + If the number of value nodes is less than $minCount, there is a validation result. + """ + + def __init__(self, shape): + super(JSConstraintComponent, self).__init__(shape) + js_decls = list(self.shape.objects(SH_js)) + if len(js_decls) < 1: + raise ConstraintLoadError( + "JSConstraintComponent must have at least one sh:js predicate.", + "https://www.w3.org/TR/shacl-js/#js-constraints", + ) + self.js_exes = [JSExecutable(shape.sg, j) for j in js_decls] + + @classmethod + def constraint_parameters(cls): + return [SH_js] + + @classmethod + def constraint_name(cls): + return "JSConstraintComponent" + + @classmethod + def shacl_constraint_class(cls): + return SH_JSConstraintComponent + + def make_generic_messages(self, datagraph: GraphLike, focus_node, value_node) -> List[Literal]: + return [Literal("Javascript Function generated constraint validation reports.")] + + def evaluate(self, data_graph: GraphLike, focus_value_nodes: Dict, _evaluation_path: List): + """ + :type data_graph: rdflib.Graph + :type focus_value_nodes: dict + :type _evaluation_path: list + """ + reports = [] + non_conformant = False + for c in self.js_exes: + _n, _r = self._evaluate_js_exe(data_graph, focus_value_nodes, c) + non_conformant = non_conformant or _n + reports.extend(_r) + return (not non_conformant), reports + + def _evaluate_js_exe(self, data_graph, f_v_dict, js_exe): + reports = [] + non_conformant = False + for f, value_nodes in f_v_dict.items(): + for v in value_nodes: + failed = False + try: + res = js_exe.execute(data_graph, f, v) + if isinstance(res, bool) and not res: + failed = True + reports.append(self.make_v_result(data_graph, f, value_node=v)) + elif isinstance(res, str): + failed = True + reports.append(self.make_v_result(data_graph, f, value_node=v, extra_messages=[res])) + elif isinstance(res, dict): + failed = True + val = res.get('value', None) + if val is None: + val = v + path = res.get('path', None) + msgs = [] + message = res.get('message', None) + if message is not None: + msgs.append(message) + reports.append(self.make_v_result(data_graph, f, value_node=val, result_path=path, extra_messages=msgs)) + elif isinstance(res, list): + for r in res: + failed = True + if isinstance(r, bool) and not r: + reports.append(self.make_v_result(data_graph, f, value_node=v)) + elif isinstance(r, str): + reports.append(self.make_v_result(data_graph, f, value_node=v, extra_messages=[r])) + elif isinstance(r, dict): + val = r.get('value', None) + if val is None: + val = v + path = r.get('path', None) + msgs = [] + message = r.get('message', None) + if message is not None: + msgs.append(message) + reports.append(self.make_v_result(data_graph, f, value_node=val, result_path=path, + extra_messages=msgs)) + except Exception as e: + raise + if failed: + non_conformant = True + return non_conformant, reports diff --git a/pyshacl/extras/js/context.py b/pyshacl/extras/js/context.py new file mode 100644 index 0000000..ce19e55 --- /dev/null +++ b/pyshacl/extras/js/context.py @@ -0,0 +1,468 @@ +import pprint +from rdflib import URIRef, BNode, Literal +import pyduktape2 +from pyduktape2 import JSProxy + +from . import load_into_context + +class URIRefNativeWrapper(object): + inner_type = "URIRef" + + def __init__(self, uri): + if isinstance(uri, URIRef): + self.inner = uri + else: + self.inner = URIRef(uri) + + @property + def uri(self): + return str(self.inner) + + def __eq__(self, other): + return self.inner.__eq__(other.inner) + + def __repr__(self): + inner_repr = repr(self.inner) + return "URIRefNativeWrapper({})".format(inner_repr) + + +class BNodeNativeWrapper(object): + inner_type = "BNode" + + def __init__(self, id_): + if isinstance(id_, BNode): + self.inner = id_ + else: + self.inner = BNode(id_ or None) + + @property + def identifier(self): + return str(self.inner) + + def __eq__(self, other): + return self.inner.__eq__(other.inner) + + def __repr__(self): + inner_repr = repr(self.inner) + return "BNodeNativeWrapper({})".format(inner_repr) + + +class LiteralNativeWrapper(object): + inner_type = "Literal" + + def __init__(self, lexical, dtype=None, lang=None): + if isinstance(lexical, Literal): + self.inner = lexical + else: + if isinstance(dtype, URIRefNativeWrapper): + dtype = dtype.inner + self.inner = Literal(lexical, lang=lang, datatype=dtype) + + @property + def lexical(self): + return self.inner.lexical_or_value + + @property + def language(self): + return self.inner.language + + @property + def datatype(self): + return self.inner.datatype + + def __eq__(self, other): + return self.inner.__eq__(other.inner) + + def __repr__(self): + inner_repr = repr(self.inner) + return "LiteralNativeWrapper({})".format(inner_repr) + + +class GraphNativeWrapper(object): + def __init__(self, g): + self.inner = g + +class IteratorNativeWrapper(object): + def __init__(self, it): + self.it = it + + def it_next(self): + return next(self.it) + +def _make_uriref(args): + uri = getattr(args, '0') + return URIRefNativeWrapper(uri) + +def _make_bnode(args): + id_ = getattr(args, '0') + return BNodeNativeWrapper(id_) + +def _make_literal(args): + lexical, dtype, lang = getattr(args, '0'), getattr(args, '1'), getattr(args, '2') + if isinstance(dtype, JSProxy): + as_native = getattr(dtype, '_native', None) + if as_native is not None: + dtype = as_native + return LiteralNativeWrapper(lexical, dtype, lang) + +def _native_node_equals(args): + this, other = getattr(args, '0'), getattr(args, '1') + if isinstance(this, (URIRefNativeWrapper, BNodeNativeWrapper, LiteralNativeWrapper)): + this = this.inner + if isinstance(other, (URIRefNativeWrapper, BNodeNativeWrapper, LiteralNativeWrapper)): + other = other.inner + return this == other + +def _native_graph_find(args): + #args are: g, s, p, o + triple = [getattr(args, '0'), getattr(args, '1'), getattr(args, '2'), getattr(args, '3')] + wrapped_triples = [] + for t in triple: + if isinstance(t, (GraphNativeWrapper, URIRefNativeWrapper, BNodeNativeWrapper, LiteralNativeWrapper)): + wrapped_triples.append(t.inner) + else: + wrapped_triples.append(t) + g, s, p, o = wrapped_triples[:4] + it = iter(g.triples((s, p, o))) + return IteratorNativeWrapper(it) + +def _native_iterator_next(args): + arg0 = getattr(args, "0") + if isinstance(arg0, IteratorNativeWrapper): + arg0 = arg0.it + try: + spo_list = next(arg0) + except StopIteration: + return None + wrapped_list = [] + for item in spo_list[:3]: + if isinstance(item, URIRef): + wrapped_list.append(URIRefNativeWrapper(item)) + elif isinstance(item, BNode): + wrapped_list.append(BNodeNativeWrapper(item)) + elif isinstance(item, Literal): + wrapped_list.append(LiteralNativeWrapper(item)) + else: + raise RuntimeError("Bad item returned from iterator!") + return wrapped_list + +def _print(args): + arg0 = getattr(args, '0') + pprint.pprint(arg0) + + +printJs = '''\ +function print(o) { + return _print({'0': o}); +} +''' + +namedNodeJs = '''\ +function NamedNode(uri, _native) { + this.uri = uri; + if (! _native) { + _native = _make_uriref({'0': uri}); + } + this._native = _native; +} +NamedNode.from_native = function(native) { + var uri = native.uri; + return new NamedNode(uri, native); +} +NamedNode.prototype.toPython = function() { return this._native; } +NamedNode.prototype.toString = function() { return "NamedNode("+this.uri+")"; } +NamedNode.prototype.isURI = function() { return true; } +NamedNode.prototype.isBlankNode = function() { return false; } +NamedNode.prototype.isLiteral = function() { return false; } +NamedNode.prototype.equals = function(other) { + if (other.constructor && other.constructor === NamedNode) { + return _native_node_equals({"0": this._native, "1": other._native}); + } + return false; +} +''' + +blankNodeJs = '''\ +function BlankNode(id, _native) { + this.id = id || null; + if (! _native) { + _native = _make_bnode({'0': id}); + } + this._native = _native; +} +BlankNode.from_native = function(native) { + var id = native.identifier; + return new BlankNode(id, native); +} +BlankNode.prototype.toPython = function() { return this._native; } +BlankNode.prototype.toString = function() { return "BlankNode("+this.id+")"; } +BlankNode.prototype.isURI = function() { return false; } +BlankNode.prototype.isBlankNode = function() { return true; } +BlankNode.prototype.isLiteral = function() { return false; } +BlankNode.prototype.equals = function(other) { + if (other.constructor && other.constructor === BlankNode) { + return _native_node_equals({"0": this._native, "1": other._native}); + } + return false; +} +''' + +literalJs = '''\ +function Literal(lex, languageOrDatatype, _native) { + this.lex = lex; + var ndict; + if (languageOrDatatype.isURI && languageOrDatatype.isURI()) { + this.language = ""; // Language cannot be null, be empty string + this.datatype = languageOrDatatype; + ndict = {'0': lex, '1': languageOrDatatype, '2': null}; + } else { + var _lang = ""+languageOrDatatype; + this.language = _lang; + this.datatype = new NamedNode("http://www.w3.org/1999/02/22-rdf-syntax-ns#langString"); + ndict = {'0': lex, '1': null, '2': languageOrDatatype}; + } + if (! _native) { + _native = _make_literal(ndict); + } + this._native = _native; +} +Literal.from_native = function(native) { + var lex = native.lexical; + var languageOrDatatype; + print("lang"); + print(native.lang); + print("dt"); + print(native.datatype); + var lang = native.language; + var dt = native.datatype; + if (lang) { + languageOrDatatype = ""+lang; + } else if (dt) { + languageOrDatatype = dt; + } else { + languageOrDatatype = new NamedNode("http://www.w3.org/2001/XMLSchema#"); + } + return new Literal(lex, languageOrDatatype, native); +} +Literal.prototype.toPython = function() { return this._native; } +Literal.prototype.toString = function() { return "Literal("+this.lexical+")"; } +Literal.prototype.isURI = function() { return false; } +Literal.prototype.isBlankNode = function() { return false; } +Literal.prototype.isLiteral = function() { return true; } +Literal.prototype.equals = function(other) { + if (other.constructor && other.constructor === Literal) { + return _native_node_equals({"0": this._native, "1": other._native}); + } + return false; +} +''' + +termFactoryJs = '''\ +function TermFactoryFactory() { +} +TermFactoryFactory.prototype.namedNode = function(uri) { return new NamedNode(uri); } +TermFactoryFactory.prototype.blankNode = function(id) { return new BlankNode(id); } +TermFactoryFactory.prototype.literal = function(lex, languageOrDatatype) { return new Literal(lex, languageOrDatatype); } +var TermFactory = new TermFactoryFactory(); +''' + +graphJs = '''\ +function Triple(s, p, o) { + this.subject = s; + this.predicate = p; + this.object = o; +} +function Iterator() { + this._native = null; + this.closed = false; +} +Iterator.from_native = function(native) { + var i = new Iterator(); + i._native = native; + return i; +} +Iterator.prototype.next = function() { + if (this.closed) { + 1/0; //exception + } + if (this._native === null) { + return null; + } + var bits = _native_iterator_next({"0": this._native}); + if (bits === null) { + //This is the end of the iteration + return null; + } + var converted_bits = []; + for (var i=0; i<3; i++) { + var b = bits[i]; + var inner_type = b.inner_type; + if (inner_type === "URIRef") { + converted_bits[i] = NamedNode.from_native(b); + } else if (inner_type === "BNode") { + converted_bits[i] = BlankNode.from_native(b); + } else if (inner_type === "Literal") { + converted_bits[i] = Literal.from_native(b); + } else { + 1/0; //exception + } + } + return new Triple(converted_bits[0], converted_bits[1], converted_bits[2]); +} +Iterator.prototype.close = function() { + if (this.closed) { return; } + this.closed = true; + this._native = null; +} +Iterator.prototype.toPython = function() { return this._native; } +function Graph(_native) { + this._native = _native; +} +Graph.prototype.toPython = function() { return this._native; } +Graph.prototype.find = function(s, p, o) { + if (this._native === null) { + return null; + } + if (s && s.hasOwnProperty('_native')) { s = s._native; } + if (p && p.hasOwnProperty('_native')) { p = p._native; } + if (o && o.hasOwnProperty('_native')) { o = o._native; } + var native_it = _native_graph_find({"0": this._native, "1": s, "2": p, "3": o}); + var it = Iterator.from_native(native_it); + return it; +} +''' + + + +class SHACLJSContext(object): + __slots__ = ("context",) + + def __init__(self, shapes_graph, data_graph, *args, **kwargs): + context = pyduktape2.DuktapeContext() + context.set_globals( + _print=_print, _make_uriref=_make_uriref, _make_bnode=_make_bnode, _make_literal=_make_literal, + _native_node_equals=_native_node_equals, _native_iterator_next=_native_iterator_next, + _native_graph_find=_native_graph_find, + ) + context.eval_js(printJs) + context.eval_js(termFactoryJs) + context.eval_js(namedNodeJs) + context.eval_js(blankNodeJs) + context.eval_js(literalJs) + context.eval_js(graphJs) + context.set_globals(_native_shapes_graph=GraphNativeWrapper(shapes_graph)) + context.set_globals(_native_data_graph=GraphNativeWrapper(data_graph)) + context.eval_js('''\ + var $data = new Graph(_native_data_graph); + var $shapes = new Graph(_native_shapes_graph); + ''') + context.set_globals(*args, **kwargs) + + self.context = context + + def load_js_library(self, library: str): + load_into_context(self.context, library) + + @classmethod + def build_results(cls, res): + if isinstance(res, JSProxy): + try: + return res.toPython() + except AttributeError: + pass + # this means its a JS Array or Object + keys = list(iter(res)) + if len(keys) < 1: + res = [] + else: + first_key = keys[0] + if isinstance(first_key, JSProxy): + # res is an array of objects + new_res = [] + for k in keys: + try: + new_res.append(k.toPython()) + continue + except AttributeError: + pass + + v = getattr(k, 'value', None) + m = getattr(k, 'message', None) + p = getattr(k, 'path', None) + + if v is not None: + try: + v = v.toPython() + except AttributeError: + pass + if v is not None and hasattr(v, 'inner'): + v = v.inner + + if p is not None: + try: + p = p.toPython() + except AttributeError: + pass + r = {'value': v, 'message': m, 'path': p} + new_res.append(r) + return new_res + try: + getattr(res, first_key) + new_res = {} + for k in keys: + v = getattr(res, k, None) + if v is not None: + try: + v = v.toPython() + except AttributeError: + pass + if v is not None and hasattr(v, 'inner'): + v = v.inner + new_res[k] = v + return new_res + except AttributeError: + # This must be an array of something else + res = keys + return res + + def run_js_function(self, fn_name, args, returns: list = None): + if returns is None: + returns = [] + c = self.context + args_string = "" + bind_dict = {} + preamble = "" + for i, a in enumerate(args): + arg_name = "fn_arg_"+str(i+1) + if isinstance(a, URIRef): + wrapped_a = URIRefNativeWrapper(a) + native_name = "_{}_native".format(arg_name) + preamble += "var {} = NamedNode.from_native({})\n".format(arg_name, native_name) + bind_dict[native_name] = wrapped_a + elif isinstance(a, BNode): + wrapped_a = BNodeNativeWrapper(a) + native_name = "_{}_native".format(arg_name) + preamble += "var {} = BlankNode.from_native({})\n".format(arg_name, native_name) + bind_dict[native_name] = wrapped_a + elif isinstance(a, Literal): + wrapped_a = LiteralNativeWrapper(a) + native_name = "_{}_native".format(arg_name) + preamble += "var {} = Literal.from_native({})\n".format(arg_name, native_name) + bind_dict[native_name] = wrapped_a + else: + bind_dict[arg_name] = a + + args_string = args_string+arg_name+"," + c.set_globals(**bind_dict) + args_string = args_string.rstrip(',') + c.eval_js(preamble) + res = c.eval_js("\n{}({});".format(fn_name, args_string)) + returns_dict = {} + for r in returns: + try: + returns_dict[r] = c.get_global(r) + except BaseException as e: + print(e) + returns_dict[r] = None + res = self.build_results(res) + return res diff --git a/pyshacl/extras/js/loader.py b/pyshacl/extras/js/loader.py new file mode 100644 index 0000000..a1f6836 --- /dev/null +++ b/pyshacl/extras/js/loader.py @@ -0,0 +1,40 @@ +import typing +from urllib import request +if typing.TYPE_CHECKING: + from pyduktape2 import DuktapeContext + +def get_js_from_web(url: str): + """ + + :param url: + :type url: str + :return: + """ + headers = {'Accept': 'application/javascript, text/javascript, application/ecmascript, text/ecmascript,' + 'text/plain'} + r = request.Request(url, headers=headers) + resp = request.urlopen(r) + code = resp.getcode() + if not (200 <= code <= 210): + raise RuntimeError("Cannot pull JS Library URL from the web: {}, code: {}".format(url, str(code))) + return resp + +def get_js_from_file(filepath: str): + if filepath.startswith("file://"): + filepath = filepath[7:] + f = open(filepath, "rb") + return f + +def load_into_context(context: 'DuktapeContext', location: str): + f = None + try: + if location.startswith("http:") or location.startswith("https:"): + f = get_js_from_web(location) + else: + f = get_js_from_file(location) + contents = f.read() + finally: + if f: + f.close() + context.eval_js(contents) + return diff --git a/pyshacl/functions/__init__.py b/pyshacl/functions/__init__.py index b668454..b21bd72 100644 --- a/pyshacl/functions/__init__.py +++ b/pyshacl/functions/__init__.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +# import typing from typing import List, Sequence, Union diff --git a/pyshacl/shape.py b/pyshacl/shape.py index ad4e394..21d102d 100644 --- a/pyshacl/shape.py +++ b/pyshacl/shape.py @@ -487,14 +487,23 @@ def validate( # Lazy import here to avoid an import loop from .constraints import ALL_CONSTRAINT_PARAMETERS, CONSTRAINT_PARAMETERS_MAP - parameters = (p for p, v in self.sg.predicate_objects(self.node) if p in ALL_CONSTRAINT_PARAMETERS) + if self.sg.js_enabled: + search_parameters = ALL_CONSTRAINT_PARAMETERS.copy() + constraint_map = CONSTRAINT_PARAMETERS_MAP.copy() + from pyshacl.extras.js.constraint import JSConstraintComponent, SH_js + search_parameters.append(SH_js) + constraint_map[SH_js] = JSConstraintComponent + else: + search_parameters = ALL_CONSTRAINT_PARAMETERS + constraint_map = CONSTRAINT_PARAMETERS_MAP + parameters = (p for p, v in self.sg.predicate_objects(self.node) if p in search_parameters) reports = [] focus_value_nodes = self.value_nodes(target_graph, focus) non_conformant = False done_constraints = set() run_count = 0 _evaluation_path.append(self) - constraint_components = [CONSTRAINT_PARAMETERS_MAP[p] for p in iter(parameters)] + constraint_components = [constraint_map[p] for p in iter(parameters)] for constraint_component in constraint_components: if constraint_component in done_constraints: continue diff --git a/pyshacl/shapes_graph.py b/pyshacl/shapes_graph.py index 8c96709..b9fdfd2 100644 --- a/pyshacl/shapes_graph.py +++ b/pyshacl/shapes_graph.py @@ -55,8 +55,16 @@ def __init__(self, graph, logger=None): self._custom_constraints = None self._shacl_functions = {} self._shacl_target_types = {} + self._use_js = False self._add_system_triples() + def enable_js(self): + self._use_js = True + + @property + def js_enabled(self): + return bool(self._use_js) + def _add_system_triples(self): if isinstance(self.graph, (rdflib.Dataset, rdflib.ConjunctiveGraph)): g = next(iter(self.graph.contexts())) diff --git a/pyshacl/validate.py b/pyshacl/validate.py index a3d4488..f41550d 100644 --- a/pyshacl/validate.py +++ b/pyshacl/validate.py @@ -13,6 +13,7 @@ from rdflib.util import from_n3 +from .extras import check_extra_installed from .pytypes import GraphLike from .target import apply_target_types, gather_target_types @@ -179,7 +180,10 @@ def __init__( shacl_graph = clone_graph(data_graph, identifier='shacl') assert isinstance(shacl_graph, rdflib.Graph), "shacl_graph must be a rdflib Graph object" self.shacl_graph = ShapesGraph(shacl_graph, self.logger) - + if self.options.get('use_js', None): + is_js_installed = check_extra_installed('js') + if is_js_installed: + self.shacl_graph.enable_js() @property def target_graph(self): return self._target_graph @@ -205,6 +209,7 @@ def run(self): self._target_graph = the_target_graph shapes = self.shacl_graph.shapes # This property getter triggers shapes harvest. + if self.options['advanced']: target_types = gather_target_types(self.shacl_graph) advanced = {'functions': gather_functions(self.shacl_graph), 'rules': gather_rules(self.shacl_graph)} @@ -346,18 +351,22 @@ def validate( ) shacl_graph_format = kwargs.pop('shacl_graph_format', None) if shacl_graph is not None: - # SHACL spec requires rdf BOOL literals to operate in a very specific way rdflib_bool_patch() shacl_graph = load_from_source( shacl_graph, rdf_format=shacl_graph_format, multigraph=True, do_owl_imports=do_owl_imports ) rdflib_bool_unpatch() + use_js = kwargs.pop('js', None) + validator = None try: validator = Validator( data_graph, shacl_graph=shacl_graph, ont_graph=ont_graph, - options={'inference': inference, 'abort_on_error': abort_on_error, 'advanced': advanced, 'logger': log}, + options={ + 'inference': inference, 'abort_on_error': abort_on_error, 'advanced': advanced, + 'use_js': use_js, 'logger': log + }, ) conforms, report_graph, report_text = validator.run() except ValidationFailure as e: diff --git a/test/js/test_js_constraint.py b/test/js/test_js_constraint.py new file mode 100644 index 0000000..b09a4ba --- /dev/null +++ b/test/js/test_js_constraint.py @@ -0,0 +1,38 @@ +from rdflib import Graph +from pyshacl import validate +shapes_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix xsd: . +@prefix ex: . + +ex:LanguageExampleShape + a sh:NodeShape ; + sh:targetClass ex:Country ; + sh:js [ + a sh:JSConstraint ; + sh:message "Values are literals with German language tag." ; + sh:jsLibrary [ sh:jsLibraryURL "file://resources/js/germanLabel.js" ] ; + sh:jsFunctionName "validateGermanLabel" ; + ] . +''' + +data_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix ex: . + +ex:ValidCountry a ex:Country ; + ex:germanLabel "Spanien"@de . + +ex:InvalidCountry a ex:Country ; + ex:germanLabel "Spain"@en . +''' + +s1 = Graph().parse(data=shapes_graph, format="turtle") +g1 = Graph().parse(data=data_graph, format="turtle") + + +res = validate(g1, shacl_graph=s1, advanced=True, debug=True, js=True) diff --git a/test/resources/js/germanLabel.js b/test/resources/js/germanLabel.js new file mode 100644 index 0000000..b208f65 --- /dev/null +++ b/test/resources/js/germanLabel.js @@ -0,0 +1,16 @@ +// From https://www.w3.org/TR/shacl-js/#js-constraints +function validateGermanLabel($this) { + var results = []; + print($this); + var p = TermFactory.namedNode("http://example.com/ex#germanLabel"); + var s = $data.find($this, p, null); + for(var t = s.next(); t; t = s.next()) { + var object = t.object; + if(!object.isLiteral() || !object.language.startsWith("de")) { + results.push({ + value : object + }); + } + } + return results; +} From a04c55dbc8ccfdb5a2b4b4920ec7673f2fcc2cd7 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Fri, 25 Sep 2020 22:54:03 +1000 Subject: [PATCH 02/10] wip Got parameterizable JSConstraintComponent almost finished --- pyproject.toml | 1 - pyshacl/constraints/constraint_component.py | 132 ++++++++- .../sparql_based_constraint_components.py | 113 +------- pyshacl/consts.py | 4 + pyshacl/extras/js/constraint.py | 269 ++++++++++++++++-- pyshacl/extras/js/context.py | 16 +- pyshacl/shape.py | 4 +- pyshacl/shapes_graph.py | 6 +- test/js/test_js_constraint.py | 9 +- test/js/test_js_constraint_component.py | 62 ++++ test/resources/js/germanLabel.js | 5 +- test/resources/js/hasMaxLength.js | 11 + 12 files changed, 477 insertions(+), 155 deletions(-) create mode 100644 test/js/test_js_constraint_component.py create mode 100644 test/resources/js/hasMaxLength.js diff --git a/pyproject.toml b/pyproject.toml index 564b880..825f2f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,6 @@ flake8 = "^3.7" isort = {version="^5.0.0", python=">=3.6"} black = {version=">=18.9b0,<=19.10b0", python=">=3.6"} mypy = {version="^0.770.0", python=">=3.6"} -mypy = {version="^0.770.0", python=">=3.6"} [tool.poetry.extras] diff --git a/pyshacl/constraints/constraint_component.py b/pyshacl/constraints/constraint_component.py index cb92cc2..a07e6fb 100644 --- a/pyshacl/constraints/constraint_component.py +++ b/pyshacl/constraints/constraint_component.py @@ -4,8 +4,8 @@ https://www.w3.org/TR/shacl/#core-components-value-type """ import abc - -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional +import typing +from typing import TYPE_CHECKING, Dict, Iterable, Tuple, List, Set, Optional, Any import rdflib @@ -22,14 +22,17 @@ SH_sourceShape, SH_ValidationResult, SH_value, - SH_Violation, + SH_Violation, SH_parameter, SH_path, SH, SH_ask, SH_select, SH_jsFunctionName, ) +from pyshacl.errors import ConstraintLoadError +from pyshacl.parameter import SHACLParameter from pyshacl.pytypes import GraphLike from pyshacl.rdfutil import stringify_node if TYPE_CHECKING: from pyshacl.shape import Shape + from pyshacl.shapes_graph import ShapesGraph class ConstraintComponent(object, metaclass=abc.ABCMeta): @@ -236,3 +239,126 @@ def make_v_result( ) self.shape.logger.debug(desc) return desc, r_node, r_triples + + +SH_nodeValidator = SH.term('nodeValidator') +SH_propertyValidator = SH.term('propertyValidator') +SH_validator = SH.term('validator') +SH_optional = SH.term('optional') +SH_SPARQLSelectValidator = SH.term('SPARQLSelectValidator') +SH_SPARQLAskValidator = SH.term('SPARQLAskValidator') +SH_JSValidator = SH.term('JSValidator') + +class CustomConstraintComponentFactory(object): + __slots__: Tuple = tuple() + + def __new__(cls, shacl_graph: 'ShapesGraph', node): + self: List[Any] = list() + self.append(shacl_graph) + self.append(node) + optional_params = [] + mandatory_params = [] + param_nodes = set(shacl_graph.objects(node, SH_parameter)) + if len(param_nodes) < 1: + # TODO:coverage: we don't have any tests for invalid constraints + raise ConstraintLoadError( + "A sh:ConstraintComponent must have at least one value for sh:parameter", + "https://www.w3.org/TR/shacl/#constraint-components-parameters", + ) + for param_node in iter(param_nodes): + path_nodes = set(shacl_graph.objects(param_node, SH_path)) + if len(path_nodes) < 1: + # TODO:coverage: we don't have any tests for invalid constraints + raise ConstraintLoadError( + "A sh:ConstraintComponent parameter value must have at least one value for sh:path", + "https://www.w3.org/TR/shacl/#constraint-components-parameters", + ) + elif len(path_nodes) > 1: + # TODO:coverage: we don't have any tests for invalid constraints + raise ConstraintLoadError( + "A sh:ConstraintComponent parameter value must have at most one value for sh:path", + "https://www.w3.org/TR/shacl/#constraint-components-parameters", + ) + path = next(iter(path_nodes)) + parameter = SHACLParameter(shacl_graph, param_node, path=path, logger=None) # pass in logger? + if parameter.optional: + optional_params.append(parameter) + else: + mandatory_params.append(parameter) + if len(mandatory_params) < 1: + # TODO:coverage: we don't have any tests for invalid constraint components + raise ConstraintLoadError( + "A sh:ConstraintComponent must have at least one non-optional parameter.", + "https://www.w3.org/TR/shacl/#constraint-components-parameters", + ) + self.append(mandatory_params + optional_params) + + validator_node_set = set(shacl_graph.graph.objects(node, SH_validator)) + node_val_node_set = set(shacl_graph.graph.objects(node, SH_nodeValidator)) + prop_val_node_set = set(shacl_graph.graph.objects(node, SH_propertyValidator)) + validator_node_set = validator_node_set.difference(node_val_node_set) + validator_node_set = validator_node_set.difference(prop_val_node_set) + self.append(validator_node_set) + self.append(node_val_node_set) + self.append(prop_val_node_set) + is_sparql_constraint_component = False + is_js_constraint_component = False + for s in (validator_node_set, node_val_node_set, prop_val_node_set): + for v in s: + v_types = set(shacl_graph.graph.objects(v, RDF_type)) + if SH_SPARQLAskValidator in v_types or SH_SPARQLSelectValidator in v_types: + is_sparql_constraint_component = True + break + elif SH_JSValidator in v_types: + is_js_constraint_component = True + break + v_props = set(p[0] for p in shacl_graph.graph.predicate_objects(v)) + if SH_ask in v_props or SH_select in v_props: + is_sparql_constraint_component = True + break + elif SH_jsFunctionName in v_props: + is_js_constraint_component = True + break + if is_sparql_constraint_component: + raise ConstraintLoadError( + "Found a mix of SPARQL-based validators and non-SPARQL validators on a SPARQLConstraintComponent.", + 'https://www.w3.org/TR/shacl/#constraint-components-validators', + ) + elif is_js_constraint_component: + raise ConstraintLoadError( + "Found a mix of JS-based validators and non-JS validators on a JSConstraintComponent.", + 'https://www.w3.org/TR/shacl/#constraint-components-validators', + ) + if is_sparql_constraint_component: + from pyshacl.constraints.sparql.sparql_based_constraint_components import SPARQLConstraintComponent + return SPARQLConstraintComponent(*self) + elif is_js_constraint_component and shacl_graph.js_enabled: + from pyshacl.extras.js.constraint import JSConstraintComponent + return JSConstraintComponent(*self) + else: + return CustomConstraintComponent(*self) + + +class CustomConstraintComponent(object): + __slots__: Tuple = ('sg', 'node', 'parameters', 'validators', 'node_validators', 'property_validators') + + if typing.TYPE_CHECKING: + sg: ShapesGraph + node: Any + parameters: List[SHACLParameter] + validators: Set + node_validators: Set + property_validators: Set + + def __new__(cls, shacl_graph: 'ShapesGraph', node, parameters, validators, node_validators, property_validators): + self = super(CustomConstraintComponent, cls).__new__(cls) + self.sg = shacl_graph + self.node = node + self.parameters = parameters + self.validators = validators + self.node_validators = node_validators + self.property_validators = property_validators + return self + + def make_validator_for_shape(self, shape: 'Shape'): + raise NotImplementedError() diff --git a/pyshacl/constraints/sparql/sparql_based_constraint_components.py b/pyshacl/constraints/sparql/sparql_based_constraint_components.py index 986792a..5c827d8 100644 --- a/pyshacl/constraints/sparql/sparql_based_constraint_components.py +++ b/pyshacl/constraints/sparql/sparql_based_constraint_components.py @@ -8,11 +8,10 @@ import rdflib -from pyshacl.constraints.constraint_component import ConstraintComponent +from pyshacl.constraints.constraint_component import ConstraintComponent, CustomConstraintComponent from pyshacl.constraints.sparql.sparql_based_constraints import SPARQLQueryHelper -from pyshacl.consts import SH, RDF_type, SH_ask, SH_message, SH_parameter, SH_path, SH_select +from pyshacl.consts import SH, RDF_type, SH_ask, SH_message, SH_select, SH_ConstraintComponent from pyshacl.errors import ConstraintLoadError, ValidationFailure -from pyshacl.parameter import SHACLParameter from pyshacl.pytypes import GraphLike @@ -21,12 +20,6 @@ from pyshacl.shapes_graph import ShapesGraph -SH_nodeValidator = SH.term('nodeValidator') -SH_propertyValidator = SH.term('propertyValidator') -SH_validator = SH.term('validator') -SH_optional = SH.term('optional') - -SH_ConstraintComponent = SH.term('ConstraintComponent') SH_SPARQLSelectValidator = SH.term('SPARQLSelectValidator') SH_SPARQLAskValidator = SH.term('SPARQLAskValidator') @@ -55,7 +48,7 @@ def __init__(self, constraint, shape: 'Shape', validator): @classmethod def constraint_parameters(cls): # TODO:coverage: this is never used for this constraint? - return [SH_validator, SH_nodeValidator, SH_propertyValidator] + return [] @classmethod def constraint_name(cls): @@ -330,106 +323,6 @@ def validate(self, focus, value_nodes, target_graph, query_helper=None, new_bind pass return violations - -class CustomConstraintComponentFactory(object): - __slots__: Tuple = tuple() - - def __new__(cls, shacl_graph: 'ShapesGraph', node): - self: List[Any] = list() - self.append(shacl_graph) - self.append(node) - optional_params = [] - mandatory_params = [] - param_nodes = set(shacl_graph.objects(node, SH_parameter)) - if len(param_nodes) < 1: - # TODO:coverage: we don't have any tests for invalid constraints - raise ConstraintLoadError( - "A sh:ConstraintComponent must have at least one value for sh:parameter", - "https://www.w3.org/TR/shacl/#constraint-components-parameters", - ) - for param_node in iter(param_nodes): - path_nodes = set(shacl_graph.objects(param_node, SH_path)) - if len(path_nodes) < 1: - # TODO:coverage: we don't have any tests for invalid constraints - raise ConstraintLoadError( - "A sh:ConstraintComponent parameter value must have at least one value for sh:path", - "https://www.w3.org/TR/shacl/#constraint-components-parameters", - ) - elif len(path_nodes) > 1: - # TODO:coverage: we don't have any tests for invalid constraints - raise ConstraintLoadError( - "A sh:ConstraintComponent parameter value must have at most one value for sh:path", - "https://www.w3.org/TR/shacl/#constraint-components-parameters", - ) - path = next(iter(path_nodes)) - parameter = SHACLParameter(shacl_graph, param_node, path=path, logger=None) # pass in logger? - if parameter.optional: - optional_params.append(parameter) - else: - mandatory_params.append(parameter) - if len(mandatory_params) < 1: - # TODO:coverage: we don't have any tests for invalid constraint components - raise ConstraintLoadError( - "A sh:ConstraintComponent must have at least one non-optional parameter.", - "https://www.w3.org/TR/shacl/#constraint-components-parameters", - ) - self.append(mandatory_params + optional_params) - - validator_node_set = set(shacl_graph.graph.objects(node, SH_validator)) - node_val_node_set = set(shacl_graph.graph.objects(node, SH_nodeValidator)) - prop_val_node_set = set(shacl_graph.graph.objects(node, SH_propertyValidator)) - validator_node_set = validator_node_set.difference(node_val_node_set) - validator_node_set = validator_node_set.difference(prop_val_node_set) - self.append(validator_node_set) - self.append(node_val_node_set) - self.append(prop_val_node_set) - is_sparql_constraint_component = False - for s in (validator_node_set, node_val_node_set, prop_val_node_set): - for v in s: - v_types = set(shacl_graph.graph.objects(v, RDF_type)) - if SH_SPARQLAskValidator in v_types or SH_SPARQLSelectValidator in v_types: - is_sparql_constraint_component = True - break - v_props = set(p[0] for p in shacl_graph.graph.predicate_objects(v)) - if SH_ask in v_props or SH_select in v_props: - is_sparql_constraint_component = True - break - if is_sparql_constraint_component: - raise ConstraintLoadError( - "Found a mix of SPARQL-based validators and non-SPARQL validators on a SPARQLConstraintComponent.", - 'https://www.w3.org/TR/shacl/#constraint-components-validators', - ) - if is_sparql_constraint_component: - return SPARQLConstraintComponent(*self) - else: - return CustomConstraintComponent(*self) - - -class CustomConstraintComponent(object): - __slots__: Tuple = ('sg', 'node', 'parameters', 'validators', 'node_validators', 'property_validators') - - if typing.TYPE_CHECKING: - sg: ShapesGraph - node: Any - parameters: List[SHACLParameter] - validators: Set - node_validators: Set - property_validators: Set - - def __new__(cls, shacl_graph: 'ShapesGraph', node, parameters, validators, node_validators, property_validators): - self = super(CustomConstraintComponent, cls).__new__(cls) - self.sg = shacl_graph - self.node = node - self.parameters = parameters - self.validators = validators - self.node_validators = node_validators - self.property_validators = property_validators - return self - - def make_validator_for_shape(self, shape: 'Shape'): - raise NotImplementedError() - - class SPARQLConstraintComponent(CustomConstraintComponent): """ SPARQL-based constraints provide a lot of flexibility but may be hard to understand for some people or lead to repetition. This section introduces SPARQL-based constraint components as a way to abstract the complexity of SPARQL and to declare high-level reusable components similar to the Core constraint components. Such constraint components can be declared using the SHACL RDF vocabulary and thus shared and reused. diff --git a/pyshacl/consts.py b/pyshacl/consts.py index 148f363..af1ad33 100644 --- a/pyshacl/consts.py +++ b/pyshacl/consts.py @@ -26,6 +26,7 @@ SH_BlankNodeOrIRI = SH.term('BlankNodeOrIRI') SH_BlankNodeORLiteral = SH.term('BlankNodeOrLiteral') SH_IRIOrLiteral = SH.term('IRIOrLiteral') +SH_ConstraintComponent = SH.term('ConstraintComponent') SH_SHACLFunction = SH.term('SHACLFunction') SH_SPARQLFunction = SH.term('SPARQLFunction') SH_SPARQLRule = SH.term('SPARQLRule') @@ -90,3 +91,6 @@ SH_intersection = SH.term('intersection') SH_datatype = SH.term('datatype') SH_optional = SH.term('optional') +SH_js = SH.term('js') +SH_jsFunctionName = SH.term('jsFunctionName') +SH_jsLibrary = SH.term('jsLibrary') diff --git a/pyshacl/extras/js/constraint.py b/pyshacl/extras/js/constraint.py index 9b23544..6136138 100644 --- a/pyshacl/extras/js/constraint.py +++ b/pyshacl/extras/js/constraint.py @@ -1,20 +1,19 @@ import typing -from typing import List, Dict +from typing import List, Dict, Tuple from rdflib import Literal -if typing.TYPE_CHECKING: - from pyshacl.shapes_graph import ShapesGraph from pyshacl.constraints import ConstraintComponent -from pyshacl.consts import SH -from pyshacl.errors import ConstraintLoadError -from pyshacl.rdfutil import stringify_node +from pyshacl.constraints.constraint_component import CustomConstraintComponent +from pyshacl.consts import SH, SH_message, SH_js, SH_jsLibrary, SH_jsFunctionName, SH_ConstraintComponent +from pyshacl.errors import ConstraintLoadError, ValidationFailure, ReportableRuntimeError from pyshacl.pytypes import GraphLike from .context import SHACLJSContext +if typing.TYPE_CHECKING: + from pyshacl.shapes_graph import ShapesGraph + from pyshacl.shape import Shape -SH_js = SH.term('js') -SH_jsFunctionName = SH.term('jsFunctionName') -SH_jsLibrary = SH.term('jsLibrary') SH_jsLibraryURL = SH.term('jsLibraryURL') +SH_JSConstraint = SH.term('JSConstraint') SH_JSConstraintComponent = SH.term('JSConstraintComponent') @@ -96,7 +95,7 @@ def execute(self, datagraph, *args, **kwargs): ctx.load_js_library(lib_url) return ctx.run_js_function(self.fn_name, args) -class JSConstraintComponent(ConstraintComponent): +class JSConstraint(ConstraintComponent): """ sh:minCount specifies the minimum number of value nodes that satisfy the condition. If the minimum cardinality value is 0 then this constraint is always satisfied and so may be omitted. Link: @@ -106,11 +105,11 @@ class JSConstraintComponent(ConstraintComponent): """ def __init__(self, shape): - super(JSConstraintComponent, self).__init__(shape) + super(JSConstraint, self).__init__(shape) js_decls = list(self.shape.objects(SH_js)) if len(js_decls) < 1: raise ConstraintLoadError( - "JSConstraintComponent must have at least one sh:js predicate.", + "JSConstraint must have at least one sh:js predicate.", "https://www.w3.org/TR/shacl-js/#js-constraints", ) self.js_exes = [JSExecutable(shape.sg, j) for j in js_decls] @@ -121,11 +120,11 @@ def constraint_parameters(cls): @classmethod def constraint_name(cls): - return "JSConstraintComponent" + return "JSConstraint" @classmethod def shacl_constraint_class(cls): - return SH_JSConstraintComponent + return SH_JSConstraint def make_generic_messages(self, datagraph: GraphLike, focus_node, value_node) -> List[Literal]: return [Literal("Javascript Function generated constraint validation reports.")] @@ -157,17 +156,18 @@ def _evaluate_js_exe(self, data_graph, f_v_dict, js_exe): reports.append(self.make_v_result(data_graph, f, value_node=v)) elif isinstance(res, str): failed = True - reports.append(self.make_v_result(data_graph, f, value_node=v, extra_messages=[res])) + m = Literal(res) + reports.append(self.make_v_result(data_graph, f, value_node=v, extra_messages=[m])) elif isinstance(res, dict): failed = True val = res.get('value', None) if val is None: val = v - path = res.get('path', None) + path = res.get('path', None) if not self.shape.is_property_shape else None msgs = [] message = res.get('message', None) if message is not None: - msgs.append(message) + msgs.append(Literal(message)) reports.append(self.make_v_result(data_graph, f, value_node=val, result_path=path, extra_messages=msgs)) elif isinstance(res, list): for r in res: @@ -175,16 +175,17 @@ def _evaluate_js_exe(self, data_graph, f_v_dict, js_exe): if isinstance(r, bool) and not r: reports.append(self.make_v_result(data_graph, f, value_node=v)) elif isinstance(r, str): - reports.append(self.make_v_result(data_graph, f, value_node=v, extra_messages=[r])) + m = Literal(r) + reports.append(self.make_v_result(data_graph, f, value_node=v, extra_messages=[m])) elif isinstance(r, dict): val = r.get('value', None) if val is None: val = v - path = r.get('path', None) + path = r.get('path', None) if not self.shape.is_property_shape else None msgs = [] message = r.get('message', None) if message is not None: - msgs.append(message) + msgs.append(Literal(message)) reports.append(self.make_v_result(data_graph, f, value_node=val, result_path=path, extra_messages=msgs)) except Exception as e: @@ -192,3 +193,231 @@ def _evaluate_js_exe(self, data_graph, f_v_dict, js_exe): if failed: non_conformant = True return non_conformant, reports + + +class BoundShapeJSValidatorComponent(ConstraintComponent): + invalid_parameter_names = {'this', 'shapesGraph', 'currentShape', 'path', 'PATH', 'value'} + def __init__(self, constraint, shape: 'Shape', validator): + """ + Create a new custom constraint, by applying a ConstraintComponent and a Validator to a Shape + :param constraint: The source ConstraintComponent, this is needed to bind the parameters in the query_helper + :type constraint: SPARQLConstraintComponent + :param shape: + :type shape: Shape + :param validator: + :type validator: AskConstraintValidator | SelectConstraintValidator + """ + super(BoundShapeJSValidatorComponent, self).__init__(shape) + self.constraint = constraint + self.validator = validator + self.param_bind_map = {} + self.messages = [] + self.bind_params() + + def bind_params(self): + bind_map = {} + shape = self.shape + for p in self.constraint.parameters: + name = p.localname + if name in self.invalid_parameter_names: + # TODO:coverage: No test for this case + raise ReportableRuntimeError("Parameter name {} cannot be used.".format(name)) + shape_params = set(shape.objects(p.path())) + if len(shape_params) < 1: + if not p.optional: + # TODO:coverage: No test for this case + raise ReportableRuntimeError( + "Shape does not have mandatory parameter {}.".format(str(p.path()))) + continue + # TODO: Can shapes have more than one value for the predicate? + # Just use one for now. + # TODO: Check for sh:class and sh:nodeKind on the found param value + bind_map[name] = next(iter(shape_params)) + self.param_bind_map = bind_map + + + @classmethod + def constraint_parameters(cls): + # TODO:coverage: this is never used for this constraint? + return [] + + @classmethod + def constraint_name(cls): + return "ConstraintComponent" + + @classmethod + def shacl_constraint_class(cls): + # TODO:coverage: this is never used for this constraint? + return SH_ConstraintComponent + + def evaluate(self, target_graph: GraphLike, focus_value_nodes: Dict, _evaluation_path: List): + """ + :type focus_value_nodes: dict + :type target_graph: rdflib.Graph + """ + reports = [] + non_conformant = False + extra_messages = self.messages or None + rept_kwargs = { + # TODO, determine if we need sourceConstraint here + # 'source_constraint': self.validator.node, + 'constraint_component': self.constraint.node, + 'extra_messages': extra_messages, + } + for f, value_nodes in focus_value_nodes.items(): + # we don't use value_nodes in the sparql constraint + # All queries are done on the corresponding focus node. + try: + violations = self.validator.validate(f, value_nodes, target_graph, self.param_bind_map) + except ValidationFailure as e: + raise e + if not self.shape.is_property_shape: + result_val = f + else: + result_val = None + for v in violations: + non_conformant = True + if isinstance(v, bool) and v is True: + # TODO:coverage: No test for when violation is `True` + rept = self.make_v_result(target_graph, f, value_node=result_val, **rept_kwargs) + elif isinstance(v, tuple): + t, p, v = v + if v is None: + v = result_val + rept = self.make_v_result(target_graph, t or f, value_node=v, result_path=p, **rept_kwargs) + else: + rept = self.make_v_result(target_graph, f, value_node=v, **rept_kwargs) + reports.append(rept) + return (not non_conformant), reports + +class JSConstraintComponent(CustomConstraintComponent): + """ + SPARQL-based constraints provide a lot of flexibility but may be hard to understand for some people or lead to repetition. This section introduces SPARQL-based constraint components as a way to abstract the complexity of SPARQL and to declare high-level reusable components similar to the Core constraint components. Such constraint components can be declared using the SHACL RDF vocabulary and thus shared and reused. + Link: + https://www.w3.org/TR/shacl-js/#js-components + """ + + __slots__: Tuple = tuple() + + def __new__(cls, shacl_graph, node, parameters, validators, node_validators, property_validators): + return super(JSConstraintComponent, cls).__new__( + cls, shacl_graph, node, parameters, validators, node_validators, property_validators + ) + + def make_validator_for_shape(self, shape: 'Shape'): + """ + :param shape: + :type shape: Shape + :return: + """ + val_count = len(self.validators) + node_val_count = len(self.node_validators) + prop_val_count = len(self.property_validators) + if shape.is_property_shape and prop_val_count > 0: + validator_node = next(iter(self.property_validators)) + elif (not shape.is_property_shape) and node_val_count > 0: + validator_node = next(iter(self.node_validators)) + elif val_count > 0: + validator_node = next(iter(self.validators)) + else: + raise ConstraintLoadError( + "Cannot select a validator to use, according to the rules.", + "https://www.w3.org/TR/shacl/#constraint-components-validators", + ) + + validator = JSConstraintComponentValidator(self.sg, validator_node) + applied_validator = validator.apply_to_shape_via_constraint(self, shape) + return applied_validator + +class JSConstraintComponentValidator(object): + validator_cache: Dict[Tuple[int, str], 'JSConstraintComponentValidator'] = {} + + def __new__(cls, shacl_graph: 'ShapesGraph', node, *args, **kwargs): + cache_key = (id(shacl_graph.graph), str(node)) + found_in_cache = cls.validator_cache.get(cache_key, False) + if found_in_cache: + return found_in_cache + self = super(JSConstraintComponentValidator, cls).__new__(cls) + cls.validator_cache[cache_key] = self + return self + + def __init__(self, shacl_graph: 'ShapesGraph', node, *args, **kwargs): + initialised = getattr(self, 'initialised', False) + if initialised: + return + self.shacl_graph = shacl_graph + self.node = node + sg = shacl_graph.graph + message_nodes = set(sg.objects(node, SH_message)) + for m in message_nodes: + if not (isinstance(m, Literal) and isinstance(m.value, str)): + # TODO:coverage: No test for when SPARQL-based constraint is RDF Literal is is not of type string + raise ConstraintLoadError( + "Validator sh:message must be an RDF Literal of type xsd:string.", + "https://www.w3.org/TR/shacl/#ConstraintComponent", + ) + self.messages = message_nodes + self.js_exe = JSExecutable(shacl_graph, node) + self.initialised = True + + def validate(self, focus, value_nodes, target_graph, param_bind_vals, new_bind_vals=None): + """ + + :param focus: + :param value_nodes: + :param query_helper: + :param target_graph: + :type target_graph: rdflib.Graph + :param new_bind_vals: + :return: + """ + new_bind_vals = new_bind_vals or {} + bind_vals = param_bind_vals.copy() + bind_vals.update(new_bind_vals) + for v in value_nodes: + # bind v, and bind_vals + args = [v, bind_vals['maxLength']] + # TODO: Determine which variables the fn takes, and determine the order it takes them + results = self.js_exe.execute(target_graph, *args) + if not results or len(results.bindings) < 1: + return [] + violations = set() + for r in results: + try: + p = r['path'] + except KeyError: + p = None + try: + v = r['value'] + except KeyError: + v = None + try: + t = r['this'] + except KeyError: + # TODO:coverage: No test for when result has no 'this' key + t = None + if p or v or t: + violations.add((t, p, v)) + else: + # TODO:coverage: No test for generic failure, when + # 'path' and 'value' and 'this' are not returned. + # here 'failure' must exist + try: + _ = r['failure'] + violations.add(True) + except KeyError: + pass + return violations + + def apply_to_shape_via_constraint(self, constraint, shape, **kwargs) -> BoundShapeJSValidatorComponent: + """ + Create a new Custom Constraint (BoundShapeValidatorComponent) + :param constraint: + :type constraint: SPARQLConstraintComponent + :param shape: + :type shape: pyshacl.shape.Shape + :param kwargs: + :return: + """ + return BoundShapeJSValidatorComponent(constraint, shape, self) + diff --git a/pyshacl/extras/js/context.py b/pyshacl/extras/js/context.py index ce19e55..bafab15 100644 --- a/pyshacl/extras/js/context.py +++ b/pyshacl/extras/js/context.py @@ -60,7 +60,7 @@ def __init__(self, lexical, dtype=None, lang=None): @property def lexical(self): - return self.inner.lexical_or_value + return self.inner.value @property def language(self): @@ -146,14 +146,14 @@ def _native_iterator_next(args): raise RuntimeError("Bad item returned from iterator!") return wrapped_list -def _print(args): +def _pprint(args): arg0 = getattr(args, '0') pprint.pprint(arg0) printJs = '''\ -function print(o) { - return _print({'0': o}); +function pprint(o) { + return _pprint({'0': o}); } ''' @@ -229,10 +229,6 @@ def _print(args): Literal.from_native = function(native) { var lex = native.lexical; var languageOrDatatype; - print("lang"); - print(native.lang); - print("dt"); - print(native.datatype); var lang = native.language; var dt = native.datatype; if (lang) { @@ -340,7 +336,7 @@ class SHACLJSContext(object): def __init__(self, shapes_graph, data_graph, *args, **kwargs): context = pyduktape2.DuktapeContext() context.set_globals( - _print=_print, _make_uriref=_make_uriref, _make_bnode=_make_bnode, _make_literal=_make_literal, + _pprint=_pprint, _make_uriref=_make_uriref, _make_bnode=_make_bnode, _make_literal=_make_literal, _native_node_equals=_native_node_equals, _native_iterator_next=_native_iterator_next, _native_graph_find=_native_graph_find, ) @@ -403,6 +399,8 @@ def build_results(cls, res): p = p.toPython() except AttributeError: pass + if p is not None and hasattr(p, 'inner'): + p = p.inner r = {'value': v, 'message': m, 'path': p} new_res.append(r) return new_res diff --git a/pyshacl/shape.py b/pyshacl/shape.py index 21d102d..aebbede 100644 --- a/pyshacl/shape.py +++ b/pyshacl/shape.py @@ -490,9 +490,9 @@ def validate( if self.sg.js_enabled: search_parameters = ALL_CONSTRAINT_PARAMETERS.copy() constraint_map = CONSTRAINT_PARAMETERS_MAP.copy() - from pyshacl.extras.js.constraint import JSConstraintComponent, SH_js + from pyshacl.extras.js.constraint import JSConstraint, SH_js search_parameters.append(SH_js) - constraint_map[SH_js] = JSConstraintComponent + constraint_map[SH_js] = JSConstraint else: search_parameters = ALL_CONSTRAINT_PARAMETERS constraint_map = CONSTRAINT_PARAMETERS_MAP diff --git a/pyshacl/shapes_graph.py b/pyshacl/shapes_graph.py index b9fdfd2..c8af745 100644 --- a/pyshacl/shapes_graph.py +++ b/pyshacl/shapes_graph.py @@ -6,10 +6,7 @@ from .constraints.core.logical_constraints import SH_and, SH_not, SH_or, SH_xone from .constraints.core.shape_based_constraints import SH_qualifiedValueShape -from .constraints.sparql.sparql_based_constraint_components import ( - CustomConstraintComponentFactory, - SH_ConstraintComponent, -) +from .constraints.constraint_component import CustomConstraintComponentFactory from .consts import ( SH, OWL_Class, @@ -27,6 +24,7 @@ SH_targetNode, SH_targetObjectsOf, SH_targetSubjectsOf, + SH_ConstraintComponent ) from .errors import ShapeLoadError from .shape import Shape diff --git a/test/js/test_js_constraint.py b/test/js/test_js_constraint.py index b09a4ba..2f46325 100644 --- a/test/js/test_js_constraint.py +++ b/test/js/test_js_constraint.py @@ -31,8 +31,9 @@ ex:germanLabel "Spain"@en . ''' -s1 = Graph().parse(data=shapes_graph, format="turtle") -g1 = Graph().parse(data=data_graph, format="turtle") +def test_js_constraint(): + s1 = Graph().parse(data=shapes_graph, format="turtle") + g1 = Graph().parse(data=data_graph, format="turtle") + conforms, result_graph, result_text = validate(g1, shacl_graph=s1, advanced=True, debug=True, js=True) + assert not conforms - -res = validate(g1, shacl_graph=s1, advanced=True, debug=True, js=True) diff --git a/test/js/test_js_constraint_component.py b/test/js/test_js_constraint_component.py new file mode 100644 index 0000000..2ae2b16 --- /dev/null +++ b/test/js/test_js_constraint_component.py @@ -0,0 +1,62 @@ +from rdflib import Graph +from pyshacl import validate +shapes_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix xsd: . +@prefix ex: . + +ex:MaxLengthConstraintComponent + a sh:ConstraintComponent ; + sh:parameter [ + sh:path ex:maxLength ; + sh:datatype xsd:integer ; + ] ; + sh:validator ex:hasMaxLength . + +ex:hasMaxLength + a sh:JSValidator ; + sh:message "Value has more than {$maxLength} characters" ; + rdfs:comment """ + Note that $value and $maxLength are RDF nodes expressed in JavaScript Objects. + Their string value is accessed via the .lex and .uri attributes. + The function returns true if no violation has been found. + """ ; + sh:jsLibrary [ sh:jsLibraryURL "file://resources/js/hasMaxLength.js"^^xsd:anyURI ] ; + sh:jsFunctionName "hasMaxLength" . + +ex:TestShape + rdf:type sh:NodeShape ; + rdfs:label "Test shape" ; + sh:property [ + sh:path ex:postcode ; + ex:maxLength 4 ; + ] ; + sh:targetNode ex:InvalidResource1 ; + sh:targetNode ex:ValidResource1 ; + . + +''' + +data_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix ex: . + +ex:InvalidResource1 a rdf:Resource ; + ex:postcode "12345" . + +ex:ValidResource1 a rdf:Resource ; + ex:postcode "1234" . +''' + +def test_js_constraint_component(): + s1 = Graph().parse(data=shapes_graph, format="turtle") + g1 = Graph().parse(data=data_graph, format="turtle") + conforms, result_graph, result_text = validate(g1, shacl_graph=s1, advanced=True, debug=True, js=True) + assert not conforms + +if __name__ == "__main__": + test_js_constraint_component() diff --git a/test/resources/js/germanLabel.js b/test/resources/js/germanLabel.js index b208f65..876b04d 100644 --- a/test/resources/js/germanLabel.js +++ b/test/resources/js/germanLabel.js @@ -1,14 +1,15 @@ // From https://www.w3.org/TR/shacl-js/#js-constraints function validateGermanLabel($this) { var results = []; - print($this); var p = TermFactory.namedNode("http://example.com/ex#germanLabel"); var s = $data.find($this, p, null); for(var t = s.next(); t; t = s.next()) { var object = t.object; if(!object.isLiteral() || !object.language.startsWith("de")) { results.push({ - value : object + value : object, + message : "Hello World.", + path : p, }); } } diff --git a/test/resources/js/hasMaxLength.js b/test/resources/js/hasMaxLength.js new file mode 100644 index 0000000..54e8485 --- /dev/null +++ b/test/resources/js/hasMaxLength.js @@ -0,0 +1,11 @@ +function hasMaxLength($value, $maxLength) { + if($value.isLiteral()) { + return $value.lex.length <= $maxLength.lex; + } + else if($value.isURI()) { + return $value.uri.length <= $maxLength.lex; + } + else { // Blank node + return false; + } +} From e24b86ab51fcccfb1965620e6a6b92ffe523200b Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Mon, 12 Oct 2020 21:21:09 +1000 Subject: [PATCH 03/10] More changes for SHACL-JS support * Detect functions and signature from loaded JS files, and map SHACL arguments into the function appropriately according to the SHACL spec * Tidy up the JSConstraint and JSConstraintComponent * Property generate Validation results from JSConstraintComponent * Move sparql query helper into a new helpers directory --- pyproject.toml | 3 +- .../sparql_based_constraint_components.py | 10 +- .../sparql/sparql_based_constraints.py | 3 +- pyshacl/extras/js/constraint.py | 191 +++++++++++------- pyshacl/extras/js/context.py | 65 +++++- pyshacl/extras/js/loader.py | 24 ++- pyshacl/functions/shacl_function.py | 4 +- pyshacl/helper/__init__.py | 15 ++ pyshacl/{ => helper}/sparql_query_helper.py | 4 +- pyshacl/rules/__init__.py | 3 - pyshacl/rules/sparql/__init__.py | 9 +- pyshacl/shape.py | 9 +- pyshacl/target.py | 4 +- test/js/test_js_constraint.py | 2 + test/js/test_js_constraint_path_component.py | 60 ++++++ test/resources/js/hasMaxCount.js | 12 ++ 16 files changed, 306 insertions(+), 112 deletions(-) create mode 100644 pyshacl/helper/__init__.py rename pyshacl/{ => helper}/sparql_query_helper.py (99%) create mode 100644 test/js/test_js_constraint_path_component.py create mode 100644 test/resources/js/hasMaxCount.js diff --git a/pyproject.toml b/pyproject.toml index 825f2f5..d762804 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ python = "^3.6" # Compatible python versions must be declared here rdflib = "^5.0.0" rdflib-jsonld = "^0.5.0" owlrl = "^5.2.1" -pyduktape2 = {version="^0.3.1", optional=true} +pyduktape2 = {version="^0.4.1", optional=true} [tool.poetry.dev-dependencies] coverage = "^4.5" @@ -58,7 +58,6 @@ isort = {version="^5.0.0", python=">=3.6"} black = {version=">=18.9b0,<=19.10b0", python=">=3.6"} mypy = {version="^0.770.0", python=">=3.6"} - [tool.poetry.extras] js = ["pyduktape2"] dev-lint = ["isort", "black", "flake8"] diff --git a/pyshacl/constraints/sparql/sparql_based_constraint_components.py b/pyshacl/constraints/sparql/sparql_based_constraint_components.py index 5c827d8..95f2934 100644 --- a/pyshacl/constraints/sparql/sparql_based_constraint_components.py +++ b/pyshacl/constraints/sparql/sparql_based_constraint_components.py @@ -9,9 +9,9 @@ import rdflib from pyshacl.constraints.constraint_component import ConstraintComponent, CustomConstraintComponent -from pyshacl.constraints.sparql.sparql_based_constraints import SPARQLQueryHelper from pyshacl.consts import SH, RDF_type, SH_ask, SH_message, SH_select, SH_ConstraintComponent from pyshacl.errors import ConstraintLoadError, ValidationFailure +from pyshacl.helper import get_query_helper_cls from pyshacl.pytypes import GraphLike @@ -39,6 +39,7 @@ def __init__(self, constraint, shape: 'Shape', validator): self.constraint = constraint self.validator = validator params = constraint.parameters + SPARQLQueryHelper = get_query_helper_cls() self.query_helper = SPARQLQueryHelper( self.shape, validator.node, validator.query_text, params, messages=validator.messages ) @@ -281,6 +282,7 @@ def validate(self, focus, value_nodes, target_graph, query_helper=None, new_bind new_bind_vals = new_bind_vals or {} bind_vals = param_bind_vals.copy() bind_vals.update(new_bind_vals) + violations = set() for v in value_nodes: if query_helper is None: # TODO:coverage: No test for this case when query_helper is None @@ -294,8 +296,7 @@ def validate(self, focus, value_nodes, target_graph, query_helper=None, new_bind init_binds.update(bind_vals) results = target_graph.query(sparql_text, initBindings=init_binds) if not results or len(results.bindings) < 1: - return [] - violations = set() + continue for r in results: try: p = r['path'] @@ -321,7 +322,7 @@ def validate(self, focus, value_nodes, target_graph, query_helper=None, new_bind violations.add(True) except KeyError: pass - return violations + return violations class SPARQLConstraintComponent(CustomConstraintComponent): """ @@ -368,3 +369,4 @@ def make_validator_for_shape(self, shape: 'Shape'): self, shape, must_be_ask_val=must_be_ask_val, must_be_select_val=must_be_select_val ) return applied_validator + diff --git a/pyshacl/constraints/sparql/sparql_based_constraints.py b/pyshacl/constraints/sparql/sparql_based_constraints.py index 4e33216..e0b1168 100644 --- a/pyshacl/constraints/sparql/sparql_based_constraints.py +++ b/pyshacl/constraints/sparql/sparql_based_constraints.py @@ -9,8 +9,8 @@ from pyshacl.constraints.constraint_component import ConstraintComponent from pyshacl.consts import SH, SH_deactivated, SH_message, SH_select from pyshacl.errors import ConstraintLoadError, ValidationFailure +from pyshacl.helper import get_query_helper_cls from pyshacl.pytypes import GraphLike -from pyshacl.sparql_query_helper import SPARQLQueryHelper SH_sparql = SH.term('sparql') @@ -73,6 +73,7 @@ def __init__(self, shape): "https://www.w3.org/TR/shacl/#SPARQLConstraintComponent", ) deact = bool(deactivated.value) + SPARQLQueryHelper = get_query_helper_cls() query_helper = SPARQLQueryHelper(self.shape, s, select_node.value, messages=msgs, deactivated=deact) query_helper.collect_prefixes() sparql_constraints.add(query_helper) diff --git a/pyshacl/extras/js/constraint.py b/pyshacl/extras/js/constraint.py index 6136138..56c8efa 100644 --- a/pyshacl/extras/js/constraint.py +++ b/pyshacl/extras/js/constraint.py @@ -88,22 +88,15 @@ def __init__(self, sg: 'ShapesGraph', node): libraries[libn2].append(str(u2)) self.libraries = libraries - def execute(self, datagraph, *args, **kwargs): + def execute(self, datagraph, args_map, *args, **kwargs): ctx = SHACLJSContext(self.sg, datagraph, **kwargs) for lib_node, lib_urls in self.libraries.items(): for lib_url in lib_urls: ctx.load_js_library(lib_url) - return ctx.run_js_function(self.fn_name, args) + fn_args = ctx.get_fn_args(self.fn_name, args_map) + return ctx.run_js_function(self.fn_name, fn_args) class JSConstraint(ConstraintComponent): - """ - sh:minCount specifies the minimum number of value nodes that satisfy the condition. If the minimum cardinality value is 0 then this constraint is always satisfied and so may be omitted. - Link: - https://www.w3.org/TR/shacl/#MinCountConstraintComponent - Textual Definition: - If the number of value nodes is less than $minCount, there is a validation result. - """ - def __init__(self, shape): super(JSConstraint, self).__init__(shape) js_decls = list(self.shape.objects(SH_js)) @@ -150,7 +143,9 @@ def _evaluate_js_exe(self, data_graph, f_v_dict, js_exe): for v in value_nodes: failed = False try: - res = js_exe.execute(data_graph, f, v) + args_map = {'this': f, 'value': v} + res_dict = js_exe.execute(data_graph, args_map) + res = res_dict['_result'] if isinstance(res, bool) and not res: failed = True reports.append(self.make_v_result(data_graph, f, value_node=v)) @@ -250,10 +245,10 @@ def shacl_constraint_class(cls): # TODO:coverage: this is never used for this constraint? return SH_ConstraintComponent - def evaluate(self, target_graph: GraphLike, focus_value_nodes: Dict, _evaluation_path: List): + def evaluate(self, data_graph: GraphLike, focus_value_nodes: Dict, _evaluation_path: List): """ :type focus_value_nodes: dict - :type target_graph: rdflib.Graph + :type data_graph: rdflib.Graph """ reports = [] non_conformant = False @@ -265,29 +260,60 @@ def evaluate(self, target_graph: GraphLike, focus_value_nodes: Dict, _evaluation 'extra_messages': extra_messages, } for f, value_nodes in focus_value_nodes.items(): - # we don't use value_nodes in the sparql constraint - # All queries are done on the corresponding focus node. try: - violations = self.validator.validate(f, value_nodes, target_graph, self.param_bind_map) + p = self.shape.path() + results = self.validator.validate(f, value_nodes, p, data_graph, self.param_bind_map) except ValidationFailure as e: raise e - if not self.shape.is_property_shape: - result_val = f - else: - result_val = None - for v in violations: - non_conformant = True - if isinstance(v, bool) and v is True: - # TODO:coverage: No test for when violation is `True` - rept = self.make_v_result(target_graph, f, value_node=result_val, **rept_kwargs) - elif isinstance(v, tuple): - t, p, v = v - if v is None: - v = result_val - rept = self.make_v_result(target_graph, t or f, value_node=v, result_path=p, **rept_kwargs) - else: - rept = self.make_v_result(target_graph, f, value_node=v, **rept_kwargs) - reports.append(rept) + for (v, res) in results: + if isinstance(res, bool) and not res: + non_conformant = True + reports.append(self.make_v_result(data_graph, f, value_node=v, **rept_kwargs)) + elif isinstance(res, str): + non_conformant = True + m = Literal(res) + new_kwargs = rept_kwargs.copy() + new_kwargs['extra_messages'] = [m] + reports.append(self.make_v_result(data_graph, f, value_node=v, **rept_kwargs)) + elif isinstance(res, dict): + non_conformant = True + val = res.get('value', None) + if val is None: + val = v + path = res.get('path', None) if not self.shape.is_property_shape else None + msgs = [] + message = res.get('message', None) + if message is not None: + msgs.append(Literal(message)) + new_kwargs = rept_kwargs.copy() + new_kwargs['extra_messages'] = msgs + reports.append( + self.make_v_result(data_graph, f, value_node=val, result_path=path, **rept_kwargs)) + elif isinstance(res, list): + for r in res: + if isinstance(r, bool) and not r: + non_conformant = True + reports.append(self.make_v_result(data_graph, f, value_node=v, **rept_kwargs)) + elif isinstance(r, str): + non_conformant = True + m = Literal(r) + new_kwargs = rept_kwargs.copy() + new_kwargs['extra_messages'] = [m] + reports.append(self.make_v_result(data_graph, f, value_node=v, **rept_kwargs)) + elif isinstance(r, dict): + non_conformant = True + val = r.get('value', None) + if val is None: + val = v + path = r.get('path', None) if not self.shape.is_property_shape else None + msgs = [] + message = r.get('message', None) + if message is not None: + msgs.append(Literal(message)) + new_kwargs = rept_kwargs.copy() + new_kwargs['extra_messages'] = msgs + reports.append(self.make_v_result(data_graph, f, value_node=val, result_path=path, + **rept_kwargs)) return (not non_conformant), reports class JSConstraintComponent(CustomConstraintComponent): @@ -313,8 +339,10 @@ def make_validator_for_shape(self, shape: 'Shape'): val_count = len(self.validators) node_val_count = len(self.node_validators) prop_val_count = len(self.property_validators) + is_property_val = False if shape.is_property_shape and prop_val_count > 0: validator_node = next(iter(self.property_validators)) + is_property_val = True elif (not shape.is_property_shape) and node_val_count > 0: validator_node = next(iter(self.node_validators)) elif val_count > 0: @@ -324,8 +352,10 @@ def make_validator_for_shape(self, shape: 'Shape'): "Cannot select a validator to use, according to the rules.", "https://www.w3.org/TR/shacl/#constraint-components-validators", ) - - validator = JSConstraintComponentValidator(self.sg, validator_node) + if is_property_val: + validator = JSConstraintComponentPathValidator(self.sg, validator_node) + else: + validator = JSConstraintComponentValidator(self.sg, validator_node) applied_validator = validator.apply_to_shape_via_constraint(self, shape) return applied_validator @@ -360,64 +390,73 @@ def __init__(self, shacl_graph: 'ShapesGraph', node, *args, **kwargs): self.js_exe = JSExecutable(shacl_graph, node) self.initialised = True - def validate(self, focus, value_nodes, target_graph, param_bind_vals, new_bind_vals=None): + def validate(self, f, value_nodes, path, data_graph, param_bind_vals, new_bind_vals=None): """ - :param focus: + :param f: :param value_nodes: - :param query_helper: - :param target_graph: - :type target_graph: rdflib.Graph + :param path: + :param data_graph: + :type data_graph: rdflib.Graph :param new_bind_vals: :return: """ new_bind_vals = new_bind_vals or {} bind_vals = param_bind_vals.copy() bind_vals.update(new_bind_vals) + results = [] for v in value_nodes: - # bind v, and bind_vals - args = [v, bind_vals['maxLength']] - # TODO: Determine which variables the fn takes, and determine the order it takes them - results = self.js_exe.execute(target_graph, *args) - if not results or len(results.bindings) < 1: - return [] - violations = set() - for r in results: - try: - p = r['path'] - except KeyError: - p = None - try: - v = r['value'] - except KeyError: - v = None - try: - t = r['this'] - except KeyError: - # TODO:coverage: No test for when result has no 'this' key - t = None - if p or v or t: - violations.add((t, p, v)) - else: - # TODO:coverage: No test for generic failure, when - # 'path' and 'value' and 'this' are not returned. - # here 'failure' must exist - try: - _ = r['failure'] - violations.add(True) - except KeyError: - pass - return violations - - def apply_to_shape_via_constraint(self, constraint, shape, **kwargs) -> BoundShapeJSValidatorComponent: + args_map = bind_vals.copy() + args_map.update({ + 'this': f, + 'value': v + }) + try: + result_dict = self.js_exe.execute(data_graph, args_map) + results.append((v, result_dict['_result'])) + except Exception as e: + raise + return results + + def apply_to_shape_via_constraint(self, constraint, shape, **kwargs)\ + -> BoundShapeJSValidatorComponent: """ Create a new Custom Constraint (BoundShapeValidatorComponent) :param constraint: - :type constraint: SPARQLConstraintComponent + :type constraint: JSConstraintComponent :param shape: :type shape: pyshacl.shape.Shape :param kwargs: :return: + :rtype: BoundShapeJSValidatorComponent """ return BoundShapeJSValidatorComponent(constraint, shape, self) + +class JSConstraintComponentPathValidator(JSConstraintComponentValidator): + + def validate(self, f, value_nodes, path, data_graph, param_bind_vals, new_bind_vals=None): + """ + + :param f: + :param value_nodes: + :param path: + :param data_graph: + :type data_graph: rdflib.Graph + :param new_bind_vals: + :return: + """ + new_bind_vals = new_bind_vals or {} + args_map = param_bind_vals.copy() + args_map.update(new_bind_vals) + args_map.update({ + 'this': f, + 'path': path + }) + results = [] + try: + result_dict = self.js_exe.execute(data_graph, args_map) + results.append((f, result_dict['_result'])) + except Exception as e: + raise + return results diff --git a/pyshacl/extras/js/context.py b/pyshacl/extras/js/context.py index bafab15..007207b 100644 --- a/pyshacl/extras/js/context.py +++ b/pyshacl/extras/js/context.py @@ -4,6 +4,8 @@ from pyduktape2 import JSProxy from . import load_into_context +from ...errors import ReportableRuntimeError + class URIRefNativeWrapper(object): inner_type = "URIRef" @@ -234,20 +236,20 @@ def _pprint(args): if (lang) { languageOrDatatype = ""+lang; } else if (dt) { - languageOrDatatype = dt; + languageOrDatatype = new NamedNode(dt); } else { - languageOrDatatype = new NamedNode("http://www.w3.org/2001/XMLSchema#"); + languageOrDatatype = new NamedNode("http://www.w3.org/2001/XMLSchema#string"); } return new Literal(lex, languageOrDatatype, native); } Literal.prototype.toPython = function() { return this._native; } -Literal.prototype.toString = function() { return "Literal("+this.lexical+")"; } +Literal.prototype.toString = function() { return "Literal("+this.lex+", dt="+this.datatype+", lang='"+this.language+"')"; } Literal.prototype.isURI = function() { return false; } Literal.prototype.isBlankNode = function() { return false; } Literal.prototype.isLiteral = function() { return true; } Literal.prototype.equals = function(other) { if (other.constructor && other.constructor === Literal) { - return _native_node_equals({"0": this._native, "1": other._native}); + return _native_node_equals({'0': this._native, '1': other._native}); } return false; } @@ -322,7 +324,7 @@ def _pprint(args): if (s && s.hasOwnProperty('_native')) { s = s._native; } if (p && p.hasOwnProperty('_native')) { p = p._native; } if (o && o.hasOwnProperty('_native')) { o = o._native; } - var native_it = _native_graph_find({"0": this._native, "1": s, "2": p, "3": o}); + var native_it = _native_graph_find({'0': this._native, '1': s, '2': p, '3': o}); var it = Iterator.from_native(native_it); return it; } @@ -331,7 +333,7 @@ def _pprint(args): class SHACLJSContext(object): - __slots__ = ("context",) + __slots__ = ("context", "fns") def __init__(self, shapes_graph, data_graph, *args, **kwargs): context = pyduktape2.DuktapeContext() @@ -355,9 +357,11 @@ def __init__(self, shapes_graph, data_graph, *args, **kwargs): context.set_globals(*args, **kwargs) self.context = context + self.fns = {} def load_js_library(self, library: str): - load_into_context(self.context, library) + fns = load_into_context(self.context, library) + self.fns.update(fns) @classmethod def build_results(cls, res): @@ -423,6 +427,42 @@ def build_results(cls, res): res = keys return res + def get_fn_args(self, fn_name, args_map): + c = self.context + try: + fn = c.get_global(fn_name) + except BaseException as e: + print(e) + raise + if not fn: + raise ReportableRuntimeError("JS Function {} cannot be found in the loaded files.".format(fn_name)) + if fn_name not in self.fns: + raise ReportableRuntimeError("JS Function {} args cannot be determined. Bad JS structure?".format(fn_name)) + known_fn_args = self.fns[fn_name] # type: tuple + needed_args = [] + for k, v in args_map.items(): + look_for = "$"+str(k) + positions = [] + start = 0 + while True: + try: + pos = known_fn_args.index(look_for, start) + positions.append(pos) + except ValueError: + break + start = pos+1 + if not positions: + continue + for p in positions: + needed_args.append((p, k, v)) + for i, a in enumerate(known_fn_args): + if a.startswith("$"): + a = a[1:] + if a not in args_map: + needed_args.append((i, a, None)) + needed_args = [v for p,k,v in sorted(needed_args)] + return needed_args + def run_js_function(self, fn_name, args, returns: list = None): if returns is None: returns = [] @@ -435,18 +475,20 @@ def run_js_function(self, fn_name, args, returns: list = None): if isinstance(a, URIRef): wrapped_a = URIRefNativeWrapper(a) native_name = "_{}_native".format(arg_name) - preamble += "var {} = NamedNode.from_native({})\n".format(arg_name, native_name) + preamble += "var {} = NamedNode.from_native({});\n".format(arg_name, native_name) bind_dict[native_name] = wrapped_a elif isinstance(a, BNode): wrapped_a = BNodeNativeWrapper(a) native_name = "_{}_native".format(arg_name) - preamble += "var {} = BlankNode.from_native({})\n".format(arg_name, native_name) + preamble += "var {} = BlankNode.from_native({});\n".format(arg_name, native_name) bind_dict[native_name] = wrapped_a elif isinstance(a, Literal): wrapped_a = LiteralNativeWrapper(a) native_name = "_{}_native".format(arg_name) - preamble += "var {} = Literal.from_native({})\n".format(arg_name, native_name) + preamble += "var {} = Literal.from_native({});\n".format(arg_name, native_name) bind_dict[native_name] = wrapped_a + elif a is None: # this is how we set an undefined variable + preamble += "var {};\n".format(arg_name) else: bind_dict[arg_name] = a @@ -463,4 +505,5 @@ def run_js_function(self, fn_name, args, returns: list = None): print(e) returns_dict[r] = None res = self.build_results(res) - return res + returns_dict['_result'] = res + return returns_dict diff --git a/pyshacl/extras/js/loader.py b/pyshacl/extras/js/loader.py index a1f6836..e69fa08 100644 --- a/pyshacl/extras/js/loader.py +++ b/pyshacl/extras/js/loader.py @@ -1,8 +1,14 @@ +# +# import typing +import regex from urllib import request if typing.TYPE_CHECKING: from pyduktape2 import DuktapeContext +JS_FN_RE1 = regex.compile(rb'function\s+([^ \n]+)\s*\((.*)\)\s*\{', regex.MULTILINE, regex.IGNORECASE) +JS_FN_RE2 = regex.compile(rb'(?:let|const|var)\s+([^ \n]+)\s*=\s*function\s*\((.*)\)\s*\{', regex.MULTILINE, regex.IGNORECASE) + def get_js_from_web(url: str): """ @@ -25,6 +31,21 @@ def get_js_from_file(filepath: str): f = open(filepath, "rb") return f +def extract_functions(content): + fns = {} + matches1 = regex.findall(JS_FN_RE1, content) + for m in matches1: + name = m[0].decode('utf-8') + params = tuple(p.strip().decode('utf-8') for p in m[1].split(b',') if p) + fns[name] = params + matches2 = regex.findall(JS_FN_RE2, content) + for m in matches2: + name = m[0].decode('utf-8') + params = tuple(p.strip().decode('utf-8') for p in m[1].split(b',') if p) + fns[name] = params + return fns + + def load_into_context(context: 'DuktapeContext', location: str): f = None try: @@ -36,5 +57,6 @@ def load_into_context(context: 'DuktapeContext', location: str): finally: if f: f.close() + fns = extract_functions(contents) context.eval_js(contents) - return + return fns diff --git a/pyshacl/functions/shacl_function.py b/pyshacl/functions/shacl_function.py index 334989c..c568a02 100644 --- a/pyshacl/functions/shacl_function.py +++ b/pyshacl/functions/shacl_function.py @@ -10,8 +10,8 @@ from ..consts import SH, RDFS_comment, SH_ask, SH_parameter, SH_select from ..errors import ConstraintLoadError, ReportableRuntimeError +from ..helper import get_query_helper_cls from ..parameter import SHACLParameter -from ..sparql_query_helper import SPARQLQueryHelper if typing.TYPE_CHECKING: @@ -126,6 +126,7 @@ def __init__(self, fn_node, sg): self.select = selects[0] if num_selects else None # deliberately not passing in Parameters to queryHelper here, because we can't bind them to this function # (this function is not a Shape, and Function Params don't get bound to it) + SPARQLQueryHelper = get_query_helper_cls() query_helper = SPARQLQueryHelper(self, self.node, None, None, deactivated=False) query_helper.collect_prefixes() self._qh = query_helper @@ -194,3 +195,4 @@ def apply(self, g): def unapply(self, g): super(SPARQLFunction, self).unapply(g) unregister_custom_function(self.node, self.execute_from_sparql) + diff --git a/pyshacl/helper/__init__.py b/pyshacl/helper/__init__.py new file mode 100644 index 0000000..d1ed7da --- /dev/null +++ b/pyshacl/helper/__init__.py @@ -0,0 +1,15 @@ +import sys + +mod = sys.modules[__name__] +setattr(mod, 'SPARQLQueryHelperCls', None) + +def get_query_helper_cls(): + # The SPARQLQueryHelper file is expensive to load due to regex compilation steps + # so we do it this way so its only loaded when something actually needs to use + # a SPARQLQueryHelper + SPARQLQueryHelperCls = getattr(mod, 'SPARQLQueryHelperCls', None) + if SPARQLQueryHelperCls is None: + from .sparql_query_helper import SPARQLQueryHelper + SPARQLQueryHelperCls = SPARQLQueryHelper + setattr(mod, 'SPARQLQueryHelperCls', SPARQLQueryHelperCls) + return SPARQLQueryHelperCls diff --git a/pyshacl/sparql_query_helper.py b/pyshacl/helper/sparql_query_helper.py similarity index 99% rename from pyshacl/sparql_query_helper.py rename to pyshacl/helper/sparql_query_helper.py index d0efafd..bb5598a 100644 --- a/pyshacl/sparql_query_helper.py +++ b/pyshacl/helper/sparql_query_helper.py @@ -10,7 +10,7 @@ from rdflib import RDF, XSD -from .consts import ( +from ..consts import ( SH, OWL_Ontology, RDF_type, @@ -23,7 +23,7 @@ SH_zeroOrMorePath, SH_zeroOrOnePath, ) -from .errors import ConstraintLoadError, ReportableRuntimeError, ValidationFailure +from ..errors import ConstraintLoadError, ReportableRuntimeError, ValidationFailure SH_declare = SH.term('declare') diff --git a/pyshacl/rules/__init__.py b/pyshacl/rules/__init__.py index 4300bd8..0e875d0 100644 --- a/pyshacl/rules/__init__.py +++ b/pyshacl/rules/__init__.py @@ -15,9 +15,6 @@ from .shacl_rule import SHACLRule -ALL_SPARQL_RULES = [TripleRule, SPARQLRule] - - def gather_rules(shacl_graph: 'ShapesGraph') -> Dict['Shape', List['SHACLRule']]: """ diff --git a/pyshacl/rules/sparql/__init__.py b/pyshacl/rules/sparql/__init__.py index 811cf52..916aa3f 100644 --- a/pyshacl/rules/sparql/__init__.py +++ b/pyshacl/rules/sparql/__init__.py @@ -9,9 +9,8 @@ from pyshacl.consts import SH_construct from pyshacl.errors import ReportableRuntimeError, RuleLoadError from pyshacl.rdfutil import clone_graph -from pyshacl.rules.shacl_rule import SHACLRule -from pyshacl.sparql_query_helper import SPARQLQueryHelper - +from pyshacl.helper import get_query_helper_cls +from ..shacl_rule import SHACLRule if TYPE_CHECKING: from pyshacl.shape import Shape @@ -43,6 +42,7 @@ def __init__(self, shape: 'Shape', rule_node: 'rdflib.term.Identifier'): "SPARQLRule sh:construct must be an xsd:string", "https://www.w3.org/TR/shacl-af/#SPARQLRule" ) self._constructs.append(str(c.value)) + SPARQLQueryHelper = get_query_helper_cls() query_helper = SPARQLQueryHelper(self.shape, self.node, None, deactivated=self._deactivated) query_helper.collect_prefixes() self._qh = query_helper @@ -51,6 +51,7 @@ def apply(self, data_graph): focus_nodes = self.shape.focus_nodes(data_graph) # uses target nodes to find focus nodes applicable_nodes = self.filter_conditions(focus_nodes, data_graph) construct_graphs = set() + SPARQLQueryHelper = get_query_helper_cls() for a in applicable_nodes: for c in self._constructs: init_bindings = {} @@ -64,3 +65,5 @@ def apply(self, data_graph): construct_graphs.add(results.graph) for g in construct_graphs: data_graph = clone_graph(g, target_graph=data_graph) + + diff --git a/pyshacl/shape.py b/pyshacl/shape.py index aebbede..54a7730 100644 --- a/pyshacl/shape.py +++ b/pyshacl/shape.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # import logging - from decimal import Decimal from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Union @@ -34,14 +33,12 @@ SH_zeroOrOnePath, ) from .errors import ConstraintLoadError, ConstraintLoadWarning, ReportableRuntimeError, ShapeLoadError +from .helper import get_query_helper_cls from .pytypes import GraphLike -from .sparql_query_helper import SPARQLQueryHelper - if TYPE_CHECKING: from pyshacl.shapes_graph import ShapesGraph - class Shape(object): __slots__ = ( @@ -240,6 +237,7 @@ def advanced_target(self): is_types = set(self.sg.objects(c, RDF_type)) if has_select or (SH_SPARQLTarget in is_types): ct['type'] = SH_SPARQLTarget + SPARQLQueryHelper = get_query_helper_cls() qh = SPARQLQueryHelper(self, c, selects[0], deactivated=self._deactivated) qh.collect_prefixes() ct['qh'] = qh @@ -535,7 +533,4 @@ def validate( non_conformant = non_conformant or (not _is_conform) reports.extend(_r) run_count += 1 - # TODO: Can these two lines be completely removed? - # if run_count < 1: - # raise RuntimeError("A SHACL Shape should have at least one parameter or attached property shape.") return (not non_conformant), reports diff --git a/pyshacl/target.py b/pyshacl/target.py index 1fe5024..a964346 100644 --- a/pyshacl/target.py +++ b/pyshacl/target.py @@ -7,8 +7,8 @@ from .consts import SH, RDF_type, RDFS_subClassOf, SH_parameter, SH_select, SH_SPARQLTargetType from .errors import ConstraintLoadError, ShapeLoadError from .parameter import SHACLParameter +from .helper import get_query_helper_cls from .pytypes import GraphLike -from .sparql_query_helper import SPARQLQueryHelper if typing.TYPE_CHECKING: @@ -129,6 +129,7 @@ class BoundSPARQLTargetType(BoundSHACLTargetType): def __init__(self, target_type, target_declaration, shape): super(BoundSPARQLTargetType, self).__init__(target_type, target_declaration, shape) params = self.target_type.parameters + SPARQLQueryHelper = get_query_helper_cls() self.query_helper = SPARQLQueryHelper( self.target_declaration, self.target_type.node, self.target_type.select, params ) @@ -201,3 +202,4 @@ def gather_target_types(shacl_graph: 'ShapesGraph') -> Sequence[Union['SHACLTarg def apply_target_types(tts: Sequence): for t in tts: t.apply() + diff --git a/test/js/test_js_constraint.py b/test/js/test_js_constraint.py index 2f46325..96e2d1b 100644 --- a/test/js/test_js_constraint.py +++ b/test/js/test_js_constraint.py @@ -37,3 +37,5 @@ def test_js_constraint(): conforms, result_graph, result_text = validate(g1, shacl_graph=s1, advanced=True, debug=True, js=True) assert not conforms +if __name__ == "__main__": + test_js_constraint() diff --git a/test/js/test_js_constraint_path_component.py b/test/js/test_js_constraint_path_component.py new file mode 100644 index 0000000..ada61bd --- /dev/null +++ b/test/js/test_js_constraint_path_component.py @@ -0,0 +1,60 @@ +from rdflib import Graph +from pyshacl import validate +shapes_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix xsd: . +@prefix ex: . + +ex:MaxCountConstraintComponent + a sh:ConstraintComponent ; + sh:parameter [ + sh:path ex:maxCount ; + sh:datatype xsd:integer ; + ] ; + sh:propertyValidator ex:hasMaxCount . + +ex:hasMaxCount + a sh:JSValidator ; + sh:message "Path has more than {$maxCount} values." ; + sh:jsLibrary [ sh:jsLibraryURL "file://resources/js/hasMaxCount.js"^^xsd:anyURI ] ; + sh:jsFunctionName "hasMaxCount" . + +ex:TestShape + rdf:type sh:NodeShape ; + rdfs:label "Test shape" ; + sh:property [ + sh:path ex:parent ; + ex:maxCount 2 ; + ] ; + sh:targetNode ex:InvalidResource1 ; + sh:targetNode ex:ValidResource1 ; + . + +''' + +data_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix ex: . + +ex:InvalidResource1 a rdf:Resource ; + ex:parent ex:Parent1 ; + ex:parent ex:Parent2 ; + ex:parent ex:Parent3 . + +ex:ValidResource1 a rdf:Resource ; + ex:parent ex:Parent1 ; + ex:parent ex:Parent2 . +''' + +def test_js_constraint_path_component(): + s1 = Graph().parse(data=shapes_graph, format="turtle") + g1 = Graph().parse(data=data_graph, format="turtle") + conforms, result_graph, result_text = validate(g1, shacl_graph=s1, advanced=True, debug=True, js=True) + assert not conforms + +if __name__ == "__main__": + test_js_constraint_path_component() diff --git a/test/resources/js/hasMaxCount.js b/test/resources/js/hasMaxCount.js new file mode 100644 index 0000000..f29a507 --- /dev/null +++ b/test/resources/js/hasMaxCount.js @@ -0,0 +1,12 @@ +function hasMaxCount($this, $path, $maxCount) { + var spo = $data.find($this, $path, null); + var accum = []; + for(var t = spo.next(); t; t = spo.next()) { + var object = t.object; + accum.push(object); + } + if (accum.length > $maxCount.lex) { + return false; + } + return true; +} From 530223543f703167791482795a330a6dccc74dc1 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Tue, 13 Oct 2020 20:42:28 +1000 Subject: [PATCH 04/10] Finalizing SHACL-JS JSConstraint and JSConstraintComponent Enable functionality to map JSConstraintComponent parameters into the sh:message string Pull out JSExecutable into its own file Pull out JSConstraintComponent and related classes into their own file Fix bug in SPARQLConstraintComponent parameters mapping into sh:message --- pyshacl/constraints/constraint_component.py | 2 +- .../sparql_based_constraint_components.py | 88 +++- pyshacl/extras/js/constraint.py | 471 +++--------------- pyshacl/extras/js/constraint_component.py | 317 ++++++++++++ pyshacl/extras/js/js_executable.py | 96 ++++ pyshacl/helper/sparql_query_helper.py | 15 +- test/resources/js/germanLabel.js | 1 - 7 files changed, 566 insertions(+), 424 deletions(-) create mode 100644 pyshacl/extras/js/constraint_component.py create mode 100644 pyshacl/extras/js/js_executable.py diff --git a/pyshacl/constraints/constraint_component.py b/pyshacl/constraints/constraint_component.py index a07e6fb..ac5fadd 100644 --- a/pyshacl/constraints/constraint_component.py +++ b/pyshacl/constraints/constraint_component.py @@ -333,7 +333,7 @@ def __new__(cls, shacl_graph: 'ShapesGraph', node): from pyshacl.constraints.sparql.sparql_based_constraint_components import SPARQLConstraintComponent return SPARQLConstraintComponent(*self) elif is_js_constraint_component and shacl_graph.js_enabled: - from pyshacl.extras.js.constraint import JSConstraintComponent + from pyshacl.extras.js.constraint_component import JSConstraintComponent return JSConstraintComponent(*self) else: return CustomConstraintComponent(*self) diff --git a/pyshacl/constraints/sparql/sparql_based_constraint_components.py b/pyshacl/constraints/sparql/sparql_based_constraint_components.py index 95f2934..0b9d98d 100644 --- a/pyshacl/constraints/sparql/sparql_based_constraint_components.py +++ b/pyshacl/constraints/sparql/sparql_based_constraint_components.py @@ -60,6 +60,9 @@ def shacl_constraint_class(cls): # TODO:coverage: this is never used for this constraint? return SH_ConstraintComponent + def make_generic_messages(self, datagraph: GraphLike, focus_node, value_node) -> List[rdflib.Literal]: + return [rdflib.Literal("Parameterised SHACL Query generated constraint validation reports.")] + def evaluate(self, target_graph: GraphLike, focus_value_nodes: Dict, _evaluation_path: List): """ :type focus_value_nodes: dict @@ -67,7 +70,7 @@ def evaluate(self, target_graph: GraphLike, focus_value_nodes: Dict, _evaluation """ reports = [] non_conformant = False - extra_messages = self.query_helper.messages or None + extra_messages = self.constraint.messages or [] rept_kwargs = { # TODO, determine if we need sourceConstraint here # 'source_constraint': self.validator.node, @@ -81,22 +84,43 @@ def evaluate(self, target_graph: GraphLike, focus_value_nodes: Dict, _evaluation violations = self.validator.validate(f, value_nodes, target_graph, self.query_helper) except ValidationFailure as e: raise e - if not self.shape.is_property_shape: - result_val = f - else: - result_val = None - for v in violations: + for val, vio in violations: non_conformant = True - if isinstance(v, bool) and v is True: - # TODO:coverage: No test for when violation is `True` - rept = self.make_v_result(target_graph, f, value_node=result_val, **rept_kwargs) - elif isinstance(v, tuple): - t, p, v = v + msg_args_map = self.query_helper.param_bind_map.copy() + msg_args_map.update({"this": f, "value": val}) + if self.shape.is_property_shape: + msg_args_map['path'] = self.shape.path() + self.query_helper.bind_messages(msg_args_map) + bound_messages = self.query_helper.bound_messages + # The DASH test suite likes _no_ value entry in the report if we're on a Property Shape. + report_val = val if not self.shape.is_property_shape else None + if isinstance(vio, bool): + if vio is False: # ASKValidator Result + new_kwargs = rept_kwargs.copy() + new_kwargs['extra_messages'].extend(bound_messages) + rept = self.make_v_result(target_graph, f, value_node=report_val, **new_kwargs) + else: # SELECTValidator Failure + raise ValidationFailure("Validation Failure generated by SPARQLConstraint.") + elif isinstance(vio, tuple): + t, p, v = vio + new_msg_args_map = msg_args_map.copy() if v is None: - v = result_val - rept = self.make_v_result(target_graph, t or f, value_node=v, result_path=p, **rept_kwargs) + v = report_val + else: + new_msg_args_map['value'] = v + if p is not None: + new_msg_args_map['path'] = p + if t is not None: + new_msg_args_map['this'] = t + self.query_helper.bind_messages(new_msg_args_map) + new_bound_msgs = self.query_helper.bound_messages + new_kwargs = rept_kwargs.copy() + new_kwargs['extra_messages'].extend(new_bound_msgs) + rept = self.make_v_result(target_graph, t or f, value_node=v, result_path=p, **new_kwargs) else: - rept = self.make_v_result(target_graph, f, value_node=v, **rept_kwargs) + new_kwargs = rept_kwargs.copy() + new_kwargs['extra_messages'].extend(bound_messages) + rept = self.make_v_result(target_graph, f, value_node=report_val, **new_kwargs) reports.append(rept) return (not non_conformant), reports @@ -182,6 +206,20 @@ def __init__(self, shacl_graph: 'ShapesGraph', node, **kwargs): self.messages = message_nodes self.initialised = True + def make_messages(self, params_map=None): + if params_map is None: + return self.messages + ret_msgs = [] + for m in self.messages: + this_m = m.value[:] + for a, v in params_map.items(): + replace_me = "{$" + str(a) + "}" + if isinstance(v, rdflib.Literal): + v = v.value + this_m = this_m.replace(replace_me, str(v)) + ret_msgs.append(rdflib.Literal(this_m)) + return ret_msgs + class AskConstraintValidator(SPARQLConstraintComponentValidator): def __new__(cls, shacl_graph: 'ShapesGraph', node, *args, **kwargs): @@ -221,7 +259,7 @@ def validate(self, focus, value_nodes, target_graph, query_helper=None, new_bind new_bind_vals = new_bind_vals or {} bind_vals = param_bind_vals.copy() bind_vals.update(new_bind_vals) - violations = [] + violations = set() for v in value_nodes: if query_helper is None: # TODO:coverage: No test for this case when query_helper is None @@ -240,7 +278,7 @@ def validate(self, focus, value_nodes, target_graph, query_helper=None, new_bind # TODO:coverage: Can this ever actually happen? raise ValidationFailure("ASK Query did not return an askAnswer.") if answer is False: - violations.append(v) + violations.add((v, False)) return violations @@ -303,23 +341,24 @@ def validate(self, focus, value_nodes, target_graph, query_helper=None, new_bind except KeyError: p = None try: - v = r['value'] + v2 = r['value'] except KeyError: - v = None + v2 = None try: t = r['this'] except KeyError: # TODO:coverage: No test for when result has no 'this' key t = None - if p or v or t: - violations.add((t, p, v)) + if p or v2 or t: + violations.add((v, (t, p, v2))) else: # TODO:coverage: No test for generic failure, when # 'path' and 'value' and 'this' are not returned. # here 'failure' must exist try: - _ = r['failure'] - violations.add(True) + f = r['failure'] + if f is True or (isinstance(f, rdflib.Literal) and f.value): + violations.add((v, True)) except KeyError: pass return violations @@ -338,6 +377,11 @@ def __new__(cls, shacl_graph, node, parameters, validators, node_validators, pro cls, shacl_graph, node, parameters, validators, node_validators, property_validators ) + @property + def messages(self): + # TODO: allow messages at this SPARQLConstraintComponent level + return [] + def make_validator_for_shape(self, shape: 'Shape'): """ :param shape: diff --git a/pyshacl/extras/js/constraint.py b/pyshacl/extras/js/constraint.py index 56c8efa..b1b1829 100644 --- a/pyshacl/extras/js/constraint.py +++ b/pyshacl/extras/js/constraint.py @@ -1,103 +1,60 @@ +# +# import typing -from typing import List, Dict, Tuple +from typing import List, Dict from rdflib import Literal from pyshacl.constraints import ConstraintComponent -from pyshacl.constraints.constraint_component import CustomConstraintComponent -from pyshacl.consts import SH, SH_message, SH_js, SH_jsLibrary, SH_jsFunctionName, SH_ConstraintComponent -from pyshacl.errors import ConstraintLoadError, ValidationFailure, ReportableRuntimeError +from pyshacl.consts import SH, SH_js, SH_message +from pyshacl.errors import ConstraintLoadError from pyshacl.pytypes import GraphLike -from .context import SHACLJSContext +from .js_executable import JSExecutable + if typing.TYPE_CHECKING: - from pyshacl.shapes_graph import ShapesGraph from pyshacl.shape import Shape + from pyshacl.shapes_graph import ShapesGraph -SH_jsLibraryURL = SH.term('jsLibraryURL') SH_JSConstraint = SH.term('JSConstraint') SH_JSConstraintComponent = SH.term('JSConstraintComponent') -class JSExecutable(object): - __slots__ = ("sg","node","fn_name","libraries") +class JSConstraintImpl(JSExecutable): + __slots__ = ("messages",) - - def __init__(self, sg: 'ShapesGraph', node): - self.node = node - self.sg = sg - fn_names = set(sg.objects(node, SH_jsFunctionName)) - if len(fn_names) < 1: - raise ConstraintLoadError( - "At least one sh:jsFunctionName must be present on a JS Executable.", - "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", - ) - elif len(fn_names) > 1: - raise ConstraintLoadError( - "At most one sh:jsFunctionName can be present on a JS Executable.", - "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", - ) - fn_name = next(iter(fn_names)) - if not isinstance(fn_name, Literal): - raise ConstraintLoadError( - "sh:jsFunctionName must be an RDF Literal with type xsd:string.", - "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", - ) - else: - fn_name = str(fn_name) - self.fn_name = fn_name - library_defs = sg.objects(node, SH_jsLibrary) - seen_library_defs = [] - libraries = {} - for libn in library_defs: - if libn in seen_library_defs: - continue - if isinstance(libn, Literal): + def __init__(self, shapes_graph: 'ShapesGraph', node): + super(JSConstraintImpl, self).__init__(shapes_graph, node) + msgs_iter = shapes_graph.objects(node, SH_message) + self.messages = [] + for m in msgs_iter: + if not isinstance(m, Literal): + raise ConstraintLoadError( + "JSConstraint sh:message must be a RDF Literal.", + "https://www.w3.org/TR/shacl-js/#js-constraints", + ) + if not isinstance(m.value, str): raise ConstraintLoadError( - "sh:jsLibrary must not have a value that is a Literal.", - "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + "JSConstraint sh:message must be a RDF Literal with type string.", + "https://www.w3.org/TR/shacl-js/#js-constraints", ) - seen_library_defs.append(libn) - jsLibraryURLs = list(sg.objects(libn, SH_jsLibraryURL)) - if len(jsLibraryURLs) > 0: - libraries[libn] = libraries.get(libn, []) - for u in jsLibraryURLs: - if not isinstance(u, Literal): - raise ConstraintLoadError( - "sh:jsLibraryURL must have a value that is a Literal.", - "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", - ) - libraries[libn].append(str(u)) - library_defs2 = sg.objects(libn, SH_jsLibrary) - for libn2 in library_defs2: - if libn2 in seen_library_defs: - continue - if isinstance(libn2, Literal): - raise ConstraintLoadError( - "sh:jsLibrary must not have a value that is a Literal.", - "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", - ) - seen_library_defs.append(libn2) - jsLibraryURLs2 = list(sg.objects(libn2, SH_jsLibraryURL)) - if len(jsLibraryURLs2) > 0: - libraries[libn2] = libraries.get(libn2, []) - for u2 in jsLibraryURLs2: - if not isinstance(u2, Literal): - raise ConstraintLoadError( - "sh:jsLibraryURL must have a value that is a Literal.", - "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", - ) - libraries[libn2].append(str(u2)) - self.libraries = libraries + self.messages.append(m) + + def make_messages(self, args_map=None): + if args_map is None: + return self.messages + ret_msgs = [] + for m in self.messages: + this_m = m.value[:] + for a, v in args_map.items(): + replace_me = "{$" + str(a) + "}" + if isinstance(v, Literal): + v = v.value + this_m = this_m.replace(replace_me, str(v)) + ret_msgs.append(Literal(this_m)) + return ret_msgs - def execute(self, datagraph, args_map, *args, **kwargs): - ctx = SHACLJSContext(self.sg, datagraph, **kwargs) - for lib_node, lib_urls in self.libraries.items(): - for lib_url in lib_urls: - ctx.load_js_library(lib_url) - fn_args = ctx.get_fn_args(self.fn_name, args_map) - return ctx.run_js_function(self.fn_name, fn_args) class JSConstraint(ConstraintComponent): - def __init__(self, shape): + def __init__(self, shape: 'Shape'): super(JSConstraint, self).__init__(shape) js_decls = list(self.shape.objects(SH_js)) if len(js_decls) < 1: @@ -105,7 +62,7 @@ def __init__(self, shape): "JSConstraint must have at least one sh:js predicate.", "https://www.w3.org/TR/shacl-js/#js-constraints", ) - self.js_exes = [JSExecutable(shape.sg, j) for j in js_decls] + self.js_impls = [JSConstraintImpl(shape.sg, j) for j in js_decls] @classmethod def constraint_parameters(cls): @@ -130,13 +87,13 @@ def evaluate(self, data_graph: GraphLike, focus_value_nodes: Dict, _evaluation_p """ reports = [] non_conformant = False - for c in self.js_exes: + for c in self.js_impls: _n, _r = self._evaluate_js_exe(data_graph, focus_value_nodes, c) non_conformant = non_conformant or _n reports.extend(_r) return (not non_conformant), reports - def _evaluate_js_exe(self, data_graph, f_v_dict, js_exe): + def _evaluate_js_exe(self, data_graph, f_v_dict, js_impl: JSConstraintImpl): reports = [] non_conformant = False for f, value_nodes in f_v_dict.items(): @@ -144,45 +101,43 @@ def _evaluate_js_exe(self, data_graph, f_v_dict, js_exe): failed = False try: args_map = {'this': f, 'value': v} - res_dict = js_exe.execute(data_graph, args_map) - res = res_dict['_result'] - if isinstance(res, bool) and not res: - failed = True - reports.append(self.make_v_result(data_graph, f, value_node=v)) - elif isinstance(res, str): - failed = True - m = Literal(res) - reports.append(self.make_v_result(data_graph, f, value_node=v, extra_messages=[m])) - elif isinstance(res, dict): - failed = True - val = res.get('value', None) - if val is None: - val = v - path = res.get('path', None) if not self.shape.is_property_shape else None - msgs = [] - message = res.get('message', None) - if message is not None: - msgs.append(Literal(message)) - reports.append(self.make_v_result(data_graph, f, value_node=val, result_path=path, extra_messages=msgs)) - elif isinstance(res, list): - for r in res: + if self.shape.is_property_shape: + args_map['path'] = self.shape.path() + res_dict = js_impl.execute(data_graph, args_map) + result = res_dict['_result'] + if result is True: + continue + msgs = js_impl.make_messages(args_map) + if isinstance(result, list): + pass + else: + result = [result] + for res in result: + if isinstance(res, bool): + if res: + continue + else: + failed = True + reports.append(self.make_v_result(data_graph, f, value_node=v, extra_messages=msgs)) + elif isinstance(res, str): failed = True - if isinstance(r, bool) and not r: - reports.append(self.make_v_result(data_graph, f, value_node=v)) - elif isinstance(r, str): - m = Literal(r) - reports.append(self.make_v_result(data_graph, f, value_node=v, extra_messages=[m])) - elif isinstance(r, dict): - val = r.get('value', None) - if val is None: - val = v - path = r.get('path', None) if not self.shape.is_property_shape else None - msgs = [] - message = r.get('message', None) - if message is not None: - msgs.append(Literal(message)) - reports.append(self.make_v_result(data_graph, f, value_node=val, result_path=path, - extra_messages=msgs)) + msgs.append(Literal(res)) + reports.append(self.make_v_result(data_graph, f, value_node=v, extra_messages=msgs)) + elif isinstance(res, dict): + failed = True + args_map2 = args_map.copy() + val = res.get('value', None) + if val is None: + val = v + args_map2['value'] = val + path = res.get('path', None) if not self.shape.is_property_shape else None + if path is not None: + args_map2['value'] = path + msgs = js_impl.make_messages(args_map2) + message = res.get('message', None) + if message is not None: + msgs.append(Literal(message)) + reports.append(self.make_v_result(data_graph, f, value_node=val, result_path=path, extra_messages=msgs)) except Exception as e: raise if failed: @@ -190,273 +145,3 @@ def _evaluate_js_exe(self, data_graph, f_v_dict, js_exe): return non_conformant, reports -class BoundShapeJSValidatorComponent(ConstraintComponent): - invalid_parameter_names = {'this', 'shapesGraph', 'currentShape', 'path', 'PATH', 'value'} - def __init__(self, constraint, shape: 'Shape', validator): - """ - Create a new custom constraint, by applying a ConstraintComponent and a Validator to a Shape - :param constraint: The source ConstraintComponent, this is needed to bind the parameters in the query_helper - :type constraint: SPARQLConstraintComponent - :param shape: - :type shape: Shape - :param validator: - :type validator: AskConstraintValidator | SelectConstraintValidator - """ - super(BoundShapeJSValidatorComponent, self).__init__(shape) - self.constraint = constraint - self.validator = validator - self.param_bind_map = {} - self.messages = [] - self.bind_params() - - def bind_params(self): - bind_map = {} - shape = self.shape - for p in self.constraint.parameters: - name = p.localname - if name in self.invalid_parameter_names: - # TODO:coverage: No test for this case - raise ReportableRuntimeError("Parameter name {} cannot be used.".format(name)) - shape_params = set(shape.objects(p.path())) - if len(shape_params) < 1: - if not p.optional: - # TODO:coverage: No test for this case - raise ReportableRuntimeError( - "Shape does not have mandatory parameter {}.".format(str(p.path()))) - continue - # TODO: Can shapes have more than one value for the predicate? - # Just use one for now. - # TODO: Check for sh:class and sh:nodeKind on the found param value - bind_map[name] = next(iter(shape_params)) - self.param_bind_map = bind_map - - - @classmethod - def constraint_parameters(cls): - # TODO:coverage: this is never used for this constraint? - return [] - - @classmethod - def constraint_name(cls): - return "ConstraintComponent" - - @classmethod - def shacl_constraint_class(cls): - # TODO:coverage: this is never used for this constraint? - return SH_ConstraintComponent - - def evaluate(self, data_graph: GraphLike, focus_value_nodes: Dict, _evaluation_path: List): - """ - :type focus_value_nodes: dict - :type data_graph: rdflib.Graph - """ - reports = [] - non_conformant = False - extra_messages = self.messages or None - rept_kwargs = { - # TODO, determine if we need sourceConstraint here - # 'source_constraint': self.validator.node, - 'constraint_component': self.constraint.node, - 'extra_messages': extra_messages, - } - for f, value_nodes in focus_value_nodes.items(): - try: - p = self.shape.path() - results = self.validator.validate(f, value_nodes, p, data_graph, self.param_bind_map) - except ValidationFailure as e: - raise e - for (v, res) in results: - if isinstance(res, bool) and not res: - non_conformant = True - reports.append(self.make_v_result(data_graph, f, value_node=v, **rept_kwargs)) - elif isinstance(res, str): - non_conformant = True - m = Literal(res) - new_kwargs = rept_kwargs.copy() - new_kwargs['extra_messages'] = [m] - reports.append(self.make_v_result(data_graph, f, value_node=v, **rept_kwargs)) - elif isinstance(res, dict): - non_conformant = True - val = res.get('value', None) - if val is None: - val = v - path = res.get('path', None) if not self.shape.is_property_shape else None - msgs = [] - message = res.get('message', None) - if message is not None: - msgs.append(Literal(message)) - new_kwargs = rept_kwargs.copy() - new_kwargs['extra_messages'] = msgs - reports.append( - self.make_v_result(data_graph, f, value_node=val, result_path=path, **rept_kwargs)) - elif isinstance(res, list): - for r in res: - if isinstance(r, bool) and not r: - non_conformant = True - reports.append(self.make_v_result(data_graph, f, value_node=v, **rept_kwargs)) - elif isinstance(r, str): - non_conformant = True - m = Literal(r) - new_kwargs = rept_kwargs.copy() - new_kwargs['extra_messages'] = [m] - reports.append(self.make_v_result(data_graph, f, value_node=v, **rept_kwargs)) - elif isinstance(r, dict): - non_conformant = True - val = r.get('value', None) - if val is None: - val = v - path = r.get('path', None) if not self.shape.is_property_shape else None - msgs = [] - message = r.get('message', None) - if message is not None: - msgs.append(Literal(message)) - new_kwargs = rept_kwargs.copy() - new_kwargs['extra_messages'] = msgs - reports.append(self.make_v_result(data_graph, f, value_node=val, result_path=path, - **rept_kwargs)) - return (not non_conformant), reports - -class JSConstraintComponent(CustomConstraintComponent): - """ - SPARQL-based constraints provide a lot of flexibility but may be hard to understand for some people or lead to repetition. This section introduces SPARQL-based constraint components as a way to abstract the complexity of SPARQL and to declare high-level reusable components similar to the Core constraint components. Such constraint components can be declared using the SHACL RDF vocabulary and thus shared and reused. - Link: - https://www.w3.org/TR/shacl-js/#js-components - """ - - __slots__: Tuple = tuple() - - def __new__(cls, shacl_graph, node, parameters, validators, node_validators, property_validators): - return super(JSConstraintComponent, cls).__new__( - cls, shacl_graph, node, parameters, validators, node_validators, property_validators - ) - - def make_validator_for_shape(self, shape: 'Shape'): - """ - :param shape: - :type shape: Shape - :return: - """ - val_count = len(self.validators) - node_val_count = len(self.node_validators) - prop_val_count = len(self.property_validators) - is_property_val = False - if shape.is_property_shape and prop_val_count > 0: - validator_node = next(iter(self.property_validators)) - is_property_val = True - elif (not shape.is_property_shape) and node_val_count > 0: - validator_node = next(iter(self.node_validators)) - elif val_count > 0: - validator_node = next(iter(self.validators)) - else: - raise ConstraintLoadError( - "Cannot select a validator to use, according to the rules.", - "https://www.w3.org/TR/shacl/#constraint-components-validators", - ) - if is_property_val: - validator = JSConstraintComponentPathValidator(self.sg, validator_node) - else: - validator = JSConstraintComponentValidator(self.sg, validator_node) - applied_validator = validator.apply_to_shape_via_constraint(self, shape) - return applied_validator - -class JSConstraintComponentValidator(object): - validator_cache: Dict[Tuple[int, str], 'JSConstraintComponentValidator'] = {} - - def __new__(cls, shacl_graph: 'ShapesGraph', node, *args, **kwargs): - cache_key = (id(shacl_graph.graph), str(node)) - found_in_cache = cls.validator_cache.get(cache_key, False) - if found_in_cache: - return found_in_cache - self = super(JSConstraintComponentValidator, cls).__new__(cls) - cls.validator_cache[cache_key] = self - return self - - def __init__(self, shacl_graph: 'ShapesGraph', node, *args, **kwargs): - initialised = getattr(self, 'initialised', False) - if initialised: - return - self.shacl_graph = shacl_graph - self.node = node - sg = shacl_graph.graph - message_nodes = set(sg.objects(node, SH_message)) - for m in message_nodes: - if not (isinstance(m, Literal) and isinstance(m.value, str)): - # TODO:coverage: No test for when SPARQL-based constraint is RDF Literal is is not of type string - raise ConstraintLoadError( - "Validator sh:message must be an RDF Literal of type xsd:string.", - "https://www.w3.org/TR/shacl/#ConstraintComponent", - ) - self.messages = message_nodes - self.js_exe = JSExecutable(shacl_graph, node) - self.initialised = True - - def validate(self, f, value_nodes, path, data_graph, param_bind_vals, new_bind_vals=None): - """ - - :param f: - :param value_nodes: - :param path: - :param data_graph: - :type data_graph: rdflib.Graph - :param new_bind_vals: - :return: - """ - new_bind_vals = new_bind_vals or {} - bind_vals = param_bind_vals.copy() - bind_vals.update(new_bind_vals) - results = [] - for v in value_nodes: - args_map = bind_vals.copy() - args_map.update({ - 'this': f, - 'value': v - }) - try: - result_dict = self.js_exe.execute(data_graph, args_map) - results.append((v, result_dict['_result'])) - except Exception as e: - raise - return results - - def apply_to_shape_via_constraint(self, constraint, shape, **kwargs)\ - -> BoundShapeJSValidatorComponent: - """ - Create a new Custom Constraint (BoundShapeValidatorComponent) - :param constraint: - :type constraint: JSConstraintComponent - :param shape: - :type shape: pyshacl.shape.Shape - :param kwargs: - :return: - :rtype: BoundShapeJSValidatorComponent - """ - return BoundShapeJSValidatorComponent(constraint, shape, self) - - -class JSConstraintComponentPathValidator(JSConstraintComponentValidator): - - def validate(self, f, value_nodes, path, data_graph, param_bind_vals, new_bind_vals=None): - """ - - :param f: - :param value_nodes: - :param path: - :param data_graph: - :type data_graph: rdflib.Graph - :param new_bind_vals: - :return: - """ - new_bind_vals = new_bind_vals or {} - args_map = param_bind_vals.copy() - args_map.update(new_bind_vals) - args_map.update({ - 'this': f, - 'path': path - }) - results = [] - try: - result_dict = self.js_exe.execute(data_graph, args_map) - results.append((f, result_dict['_result'])) - except Exception as e: - raise - return results diff --git a/pyshacl/extras/js/constraint_component.py b/pyshacl/extras/js/constraint_component.py new file mode 100644 index 0000000..33a06b0 --- /dev/null +++ b/pyshacl/extras/js/constraint_component.py @@ -0,0 +1,317 @@ +# +# +import typing +from typing import List, Dict, Tuple +from rdflib import Literal +from pyshacl.constraints import ConstraintComponent +from pyshacl.constraints.constraint_component import CustomConstraintComponent +from pyshacl.consts import SH, SH_message, SH_js, SH_jsLibrary, SH_jsFunctionName, SH_ConstraintComponent +from pyshacl.errors import ConstraintLoadError, ValidationFailure, ReportableRuntimeError +from pyshacl.pytypes import GraphLike +from .js_executable import JSExecutable + +if typing.TYPE_CHECKING: + from pyshacl.shapes_graph import ShapesGraph + from pyshacl.shape import Shape + + +SH_JSConstraint = SH.term('JSConstraint') +SH_JSConstraintComponent = SH.term('JSConstraintComponent') + +class BoundShapeJSValidatorComponent(ConstraintComponent): + invalid_parameter_names = {'this', 'shapesGraph', 'currentShape', 'path', 'PATH', 'value'} + def __init__(self, constraint, shape: 'Shape', validator): + """ + Create a new custom constraint, by applying a ConstraintComponent and a Validator to a Shape + :param constraint: The source ConstraintComponent, this is needed to bind the parameters in the query_helper + :type constraint: SPARQLConstraintComponent + :param shape: + :type shape: Shape + :param validator: + :type validator: AskConstraintValidator | SelectConstraintValidator + """ + super(BoundShapeJSValidatorComponent, self).__init__(shape) + self.constraint = constraint + self.validator = validator + self.param_bind_map = {} + self.messages = [] + self.bind_params() + + def bind_params(self): + bind_map = {} + shape = self.shape + for p in self.constraint.parameters: + name = p.localname + if name in self.invalid_parameter_names: + # TODO:coverage: No test for this case + raise ReportableRuntimeError("Parameter name {} cannot be used.".format(name)) + shape_params = set(shape.objects(p.path())) + if len(shape_params) < 1: + if not p.optional: + # TODO:coverage: No test for this case + raise ReportableRuntimeError( + "Shape does not have mandatory parameter {}.".format(str(p.path()))) + continue + # TODO: Can shapes have more than one value for the predicate? + # Just use one for now. + # TODO: Check for sh:class and sh:nodeKind on the found param value + bind_map[name] = next(iter(shape_params)) + self.param_bind_map = bind_map + + + @classmethod + def constraint_parameters(cls): + # TODO:coverage: this is never used for this constraint? + return [] + + @classmethod + def constraint_name(cls): + return "ConstraintComponent" + + @classmethod + def shacl_constraint_class(cls): + # TODO:coverage: this is never used for this constraint? + return SH_ConstraintComponent + + def make_generic_messages(self, datagraph: GraphLike, focus_node, value_node) -> List[Literal]: + return [Literal("Parameterised Javascript Function generated constraint validation reports.")] + + def evaluate(self, data_graph: GraphLike, focus_value_nodes: Dict, _evaluation_path: List): + """ + :type focus_value_nodes: dict + :type data_graph: rdflib.Graph + """ + reports = [] + non_conformant = False + extra_messages = self.messages or [] + rept_kwargs = { + 'constraint_component': self.constraint.node, + 'extra_messages': extra_messages, + } + for f, value_nodes in focus_value_nodes.items(): + try: + p = self.shape.path() + results = self.validator.validate(f, value_nodes, p, data_graph, self.param_bind_map) + except ValidationFailure as e: + raise e + for (v, result) in results: + if result is True: + continue + args_map = self.param_bind_map.copy() + args_map.update({"this": f, "value": v}) + if self.shape.is_property_shape: + args_map['path'] = self.shape.path() + bound_messages = self.validator.make_messages(args_map) + failed = False + if isinstance(result, list): + pass + else: + result = [result] + for res in result: + if isinstance(res, bool): + if res: + continue + else: + failed = True + new_kwargs = rept_kwargs.copy() + new_kwargs['extra_messages'].extend(bound_messages) + reports.append(self.make_v_result(data_graph, f, value_node=v, **new_kwargs)) + elif isinstance(res, str): + failed = True + m = Literal(res) + new_kwargs = rept_kwargs.copy() + new_kwargs['extra_messages'].append(m) + new_kwargs['extra_messages'].extend(bound_messages) + reports.append(self.make_v_result(data_graph, f, value_node=v, **new_kwargs)) + elif isinstance(res, dict): + failed = True + args_map2 = args_map.copy() + val = res.get('value', None) + if val is None: + val = v + args_map2['value'] = val + path = res.get('path', None) if not self.shape.is_property_shape else None + if path is not None: + args_map2['value'] = path + msgs = self.validator.make_messages(args_map2) + message = res.get('message', None) + if message is not None: + msgs.append(Literal(message)) + new_kwargs = rept_kwargs.copy() + new_kwargs['extra_messages'].extend(msgs) + reports.append( + self.make_v_result(data_graph, f, value_node=val, result_path=path, **new_kwargs)) + if failed: + non_conformant = True + return (not non_conformant), reports + +class JSConstraintComponent(CustomConstraintComponent): + """ + SPARQL-based constraints provide a lot of flexibility but may be hard to understand for some people or lead to repetition. This section introduces SPARQL-based constraint components as a way to abstract the complexity of SPARQL and to declare high-level reusable components similar to the Core constraint components. Such constraint components can be declared using the SHACL RDF vocabulary and thus shared and reused. + Link: + https://www.w3.org/TR/shacl-js/#js-components + """ + + __slots__: Tuple = tuple() + + def __new__(cls, shacl_graph, node, parameters, validators, node_validators, property_validators): + return super(JSConstraintComponent, cls).__new__( + cls, shacl_graph, node, parameters, validators, node_validators, property_validators + ) + + def make_validator_for_shape(self, shape: 'Shape'): + """ + :param shape: + :type shape: Shape + :return: + """ + val_count = len(self.validators) + node_val_count = len(self.node_validators) + prop_val_count = len(self.property_validators) + is_property_val = False + if shape.is_property_shape and prop_val_count > 0: + validator_node = next(iter(self.property_validators)) + is_property_val = True + elif (not shape.is_property_shape) and node_val_count > 0: + validator_node = next(iter(self.node_validators)) + elif val_count > 0: + validator_node = next(iter(self.validators)) + else: + raise ConstraintLoadError( + "Cannot select a validator to use, according to the rules.", + "https://www.w3.org/TR/shacl/#constraint-components-validators", + ) + if is_property_val: + validator = JSConstraintComponentPathValidator(self.sg, validator_node) + else: + validator = JSConstraintComponentValidator(self.sg, validator_node) + applied_validator = validator.apply_to_shape_via_constraint(self, shape) + return applied_validator + + +class JSConstraintComponentValidator(JSExecutable): + __slots__ = ("messages", "initialised") + + validator_cache: Dict[Tuple[int, str], 'JSConstraintComponentValidator'] = {} + + def __new__(cls, shacl_graph: 'ShapesGraph', node, *args, **kwargs): + cache_key = (id(shacl_graph.graph), str(node)) + found_in_cache = cls.validator_cache.get(cache_key, False) + if found_in_cache: + return found_in_cache + self = super(JSConstraintComponentValidator, cls).__new__(cls, shacl_graph, node) + cls.validator_cache[cache_key] = self + return self + + def __init__(self, shacl_graph: 'ShapesGraph', node, *args, **kwargs): + initialised = getattr(self, 'initialised', False) + if initialised: + return + super(JSConstraintComponentValidator, self).__init__(shacl_graph, node) + sg = shacl_graph.graph + message_nodes = set(sg.objects(node, SH_message)) + for m in message_nodes: + if not (isinstance(m, Literal) and isinstance(m.value, str)): + # TODO:coverage: No test for when SPARQL-based constraint is RDF Literal is is not of type string + raise ConstraintLoadError( + "Validator sh:message must be an RDF Literal of type xsd:string.", + "https://www.w3.org/TR/shacl/#ConstraintComponent", + ) + self.messages = message_nodes + self.initialised = True + + def make_messages(self, args_map=None): + if args_map is None: + return self.messages + ret_msgs = [] + for m in self.messages: + this_m = m.value[:] + for a, v in args_map.items(): + replace_me = "{$" + str(a) + "}" + if isinstance(v, Literal): + v = v.value + this_m = this_m.replace(replace_me, str(v)) + ret_msgs.append(Literal(this_m)) + return ret_msgs + + def validate(self, f, value_nodes, path, data_graph, param_bind_vals, new_bind_vals=None): + """ + + :param f: + :param value_nodes: + :param path: + :param data_graph: + :type data_graph: rdflib.Graph + :param new_bind_vals: + :return: + """ + new_bind_vals = new_bind_vals or {} + bind_vals = param_bind_vals.copy() + bind_vals.update(new_bind_vals) + results = [] + for v in value_nodes: + args_map = bind_vals.copy() + args_map.update({ + 'this': f, + 'value': v + }) + try: + result_dict = self.execute(data_graph, args_map) + results.append((v, result_dict['_result'])) + except Exception as e: + raise + return results + + def apply_to_shape_via_constraint(self, constraint, shape, **kwargs)\ + -> BoundShapeJSValidatorComponent: + """ + Create a new Custom Constraint (BoundShapeValidatorComponent) + :param constraint: + :type constraint: JSConstraintComponent + :param shape: + :type shape: pyshacl.shape.Shape + :param kwargs: + :return: + :rtype: BoundShapeJSValidatorComponent + """ + return BoundShapeJSValidatorComponent(constraint, shape, self) + + +class JSConstraintComponentPathValidator(JSConstraintComponentValidator): + validator_cache: Dict[Tuple[int, str], 'JSConstraintComponentPathValidator'] = {} + + def __new__(cls, shacl_graph: 'ShapesGraph', node, *args, **kwargs): + cache_key = (id(shacl_graph.graph), str(node)) + found_in_cache = cls.validator_cache.get(cache_key, False) + if found_in_cache: + return found_in_cache + self = super(JSConstraintComponentPathValidator, cls).__new__(cls, shacl_graph, node) + cls.validator_cache[cache_key] = self + return self + + def validate(self, f, value_nodes, path, data_graph, param_bind_vals, new_bind_vals=None): + """ + + :param f: + :param value_nodes: + :param path: + :param data_graph: + :type data_graph: rdflib.Graph + :param new_bind_vals: + :return: + """ + new_bind_vals = new_bind_vals or {} + args_map = param_bind_vals.copy() + args_map.update(new_bind_vals) + args_map.update({ + 'this': f, + 'path': path + }) + results = [] + try: + result_dict = self.execute(data_graph, args_map) + results.append((f, result_dict['_result'])) + except Exception as e: + raise + return results + diff --git a/pyshacl/extras/js/js_executable.py b/pyshacl/extras/js/js_executable.py new file mode 100644 index 0000000..551254f --- /dev/null +++ b/pyshacl/extras/js/js_executable.py @@ -0,0 +1,96 @@ +# +# +import typing +from rdflib import Literal +from pyshacl.consts import SH, SH_jsLibrary, SH_jsFunctionName +from pyshacl.errors import ConstraintLoadError +from .context import SHACLJSContext + +if typing.TYPE_CHECKING: + from pyshacl.shapes_graph import ShapesGraph + +SH_jsLibraryURL = SH.term('jsLibraryURL') + + +class JSExecutable(object): + __slots__ = ("sg", "node", "fn_name", "libraries") + + def __new__(cls, shapes_graph: 'ShapesGraph', node): + return super(JSExecutable, cls).__new__(cls) + + def __init__(self, shapes_graph: 'ShapesGraph', node): + self.node = node + self.sg = shapes_graph + fn_names = set(shapes_graph.objects(node, SH_jsFunctionName)) + if len(fn_names) < 1: + raise ConstraintLoadError( + "At least one sh:jsFunctionName must be present on a JS Executable.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + elif len(fn_names) > 1: + raise ConstraintLoadError( + "At most one sh:jsFunctionName can be present on a JS Executable.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + fn_name = next(iter(fn_names)) + if not isinstance(fn_name, Literal): + raise ConstraintLoadError( + "sh:jsFunctionName must be an RDF Literal with type xsd:string.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + else: + fn_name = str(fn_name) + self.fn_name = fn_name + library_defs = shapes_graph.objects(node, SH_jsLibrary) + seen_library_defs = [] + libraries = {} + for libn in library_defs: + # Library defs can only do two levels deep for now. + # TODO: Make this recursive somehow to some further depth + if libn in seen_library_defs: + continue + if isinstance(libn, Literal): + raise ConstraintLoadError( + "sh:jsLibrary must not have a value that is a Literal.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + seen_library_defs.append(libn) + jsLibraryURLs = list(shapes_graph.objects(libn, SH_jsLibraryURL)) + if len(jsLibraryURLs) > 0: + libraries[libn] = libraries.get(libn, []) + for u in jsLibraryURLs: + if not isinstance(u, Literal): + raise ConstraintLoadError( + "sh:jsLibraryURL must have a value that is a Literal.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + libraries[libn].append(str(u)) + library_defs2 = shapes_graph.objects(libn, SH_jsLibrary) + for libn2 in library_defs2: + if libn2 in seen_library_defs: + continue + if isinstance(libn2, Literal): + raise ConstraintLoadError( + "sh:jsLibrary must not have a value that is a Literal.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + seen_library_defs.append(libn2) + jsLibraryURLs2 = list(shapes_graph.objects(libn2, SH_jsLibraryURL)) + if len(jsLibraryURLs2) > 0: + libraries[libn2] = libraries.get(libn2, []) + for u2 in jsLibraryURLs2: + if not isinstance(u2, Literal): + raise ConstraintLoadError( + "sh:jsLibraryURL must have a value that is a Literal.", + "https://www.w3.org/TR/shacl-js/#dfn-javascript-executables", + ) + libraries[libn2].append(str(u2)) + self.libraries = libraries + + def execute(self, datagraph, args_map, *args, **kwargs): + ctx = SHACLJSContext(self.sg, datagraph, **kwargs) + for lib_node, lib_urls in self.libraries.items(): + for lib_url in lib_urls: + ctx.load_js_library(lib_url) + fn_args = ctx.get_fn_args(self.fn_name, args_map) + return ctx.run_js_function(self.fn_name, fn_args) diff --git a/pyshacl/helper/sparql_query_helper.py b/pyshacl/helper/sparql_query_helper.py index bb5598a..fa87511 100644 --- a/pyshacl/helper/sparql_query_helper.py +++ b/pyshacl/helper/sparql_query_helper.py @@ -43,6 +43,7 @@ class SPARQLQueryHelper(object): r"SELECT[\s\(\)\$\?\a-z]*\{[^\}]*SELECT\s+((?:(?:[\?\$]\w+\s+)|(?:\*\s+))+)", flags=re.M | re.I ) has_as_var_regex = re.compile(r"[^\w]+AS[\s]+[\$\?](\w+)", flags=re.M | re.I) + find_msg_subs = re.compile(r"({[\$\?](.+)})", flags=re.M) def __init__(self, shape, node, select_text, parameters=None, messages=None, deactivated=False): self._shape = None @@ -98,28 +99,28 @@ def bind_params(self): bind_map[name] = next(iter(shape_params)) self.param_bind_map = bind_map - def bind_messages(self): + def bind_messages(self, param_map=None): # must call bind_params _before_ bind_messages - message_var_finder = re.compile(r"([\s()\"\'])\{[\$\?](\w+)\}", flags=re.M) - param_bind_map = self.param_bind_map + if param_map is None: + param_map = self.param_bind_map var_replacers = {} bound_messages = set() for m in self.unbound_messages: m_val = str(m.value) - finds = message_var_finder.findall(m_val) + finds = self.find_msg_subs.findall(m_val) if len(finds) < 1: bound_messages.add(m) continue for f in finds: variable = f[1] - if variable not in param_bind_map.keys(): + if variable not in param_map.keys(): continue try: replacer = var_replacers[variable] except KeyError: - replacer = re.compile(r"([\s()\"\'])\{[\$\?]" + f[1] + r"\}", flags=re.M) + replacer = re.compile(r"{[\$\?]" + variable + r"}", flags=re.M) var_replacers[variable] = replacer - m_val = replacer.sub(r"\g<1>{}".format(param_bind_map[variable].value), m_val, 1) + m_val = replacer.sub(str(param_map[variable].value), m_val, 1) bound_messages.add(rdflib.Literal(m_val, lang=m.language, datatype=m.datatype)) self.bound_messages = bound_messages diff --git a/test/resources/js/germanLabel.js b/test/resources/js/germanLabel.js index 876b04d..57d6231 100644 --- a/test/resources/js/germanLabel.js +++ b/test/resources/js/germanLabel.js @@ -8,7 +8,6 @@ function validateGermanLabel($this) { if(!object.isLiteral() || !object.language.startsWith("de")) { results.push({ value : object, - message : "Hello World.", path : p, }); } From 19f75998c2af281fc93ca68e6653be4656790199 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Wed, 14 Oct 2020 00:03:57 +1000 Subject: [PATCH 05/10] Add JSFunction functionality. Javascript fns can now be called from SHACL Constraints. Add WIP JSRule functionality, still needs to be further tested. Allow the JSExecutable to wrangle its results in different ways depending on the reason it was run (eg, JSConstraint vs JSFunction vs JSRule all need different interpretation and wrangling of the JS fn output). Refactor SHACL-JS tests, enable them to run like the other test suites Add new SHACL-JS tests --- pyshacl/extras/js/__init__.py | 1 - pyshacl/extras/js/constraint.py | 2 + pyshacl/extras/js/constraint_component.py | 2 + pyshacl/extras/js/context.py | 113 ++++++++++++++++-- pyshacl/extras/js/function.py | 71 +++++++++++ pyshacl/extras/js/js_executable.py | 28 ++++- pyshacl/extras/js/rules.py | 48 ++++++++ pyshacl/functions/__init__.py | 38 ++++-- pyshacl/rules/__init__.py | 22 ++++ test/resources/js/multiply.js | 3 + test/resources/js/rectangle.js | 18 +++ test/test_js/__init__.py | 0 test/{js => test_js}/test_js_constraint.py | 0 .../test_js_constraint_component.py | 0 .../test_js_constraint_path_component.py | 0 test/test_js/test_js_function.py | 106 ++++++++++++++++ test/test_js/test_js_rules.py | 40 +++++++ 17 files changed, 472 insertions(+), 20 deletions(-) create mode 100644 pyshacl/extras/js/function.py create mode 100644 pyshacl/extras/js/rules.py create mode 100644 test/resources/js/multiply.js create mode 100644 test/resources/js/rectangle.js create mode 100644 test/test_js/__init__.py rename test/{js => test_js}/test_js_constraint.py (100%) rename test/{js => test_js}/test_js_constraint_component.py (100%) rename test/{js => test_js}/test_js_constraint_path_component.py (100%) create mode 100644 test/test_js/test_js_function.py create mode 100644 test/test_js/test_js_rules.py diff --git a/pyshacl/extras/js/__init__.py b/pyshacl/extras/js/__init__.py index 9d832f3..64d9000 100644 --- a/pyshacl/extras/js/__init__.py +++ b/pyshacl/extras/js/__init__.py @@ -4,4 +4,3 @@ from .loader import load_into_context - diff --git a/pyshacl/extras/js/constraint.py b/pyshacl/extras/js/constraint.py index b1b1829..7f9ab56 100644 --- a/pyshacl/extras/js/constraint.py +++ b/pyshacl/extras/js/constraint.py @@ -113,6 +113,8 @@ def _evaluate_js_exe(self, data_graph, f_v_dict, js_impl: JSConstraintImpl): else: result = [result] for res in result: + if isinstance(res, Literal): + res = res.value if isinstance(res, bool): if res: continue diff --git a/pyshacl/extras/js/constraint_component.py b/pyshacl/extras/js/constraint_component.py index 33a06b0..d1d0375 100644 --- a/pyshacl/extras/js/constraint_component.py +++ b/pyshacl/extras/js/constraint_component.py @@ -108,6 +108,8 @@ def evaluate(self, data_graph: GraphLike, focus_value_nodes: Dict, _evaluation_p else: result = [result] for res in result: + if isinstance(res, Literal): + res = res.value if isinstance(res, bool): if res: continue diff --git a/pyshacl/extras/js/context.py b/pyshacl/extras/js/context.py index 007207b..1783c52 100644 --- a/pyshacl/extras/js/context.py +++ b/pyshacl/extras/js/context.py @@ -1,5 +1,7 @@ import pprint from rdflib import URIRef, BNode, Literal +from rdflib.namespace import XSD +from decimal import Decimal import pyduktape2 from pyduktape2 import JSProxy @@ -335,7 +337,7 @@ def _pprint(args): class SHACLJSContext(object): __slots__ = ("context", "fns") - def __init__(self, shapes_graph, data_graph, *args, **kwargs): + def __init__(self, data_graph, *args, shapes_graph=None, **kwargs): context = pyduktape2.DuktapeContext() context.set_globals( _pprint=_pprint, _make_uriref=_make_uriref, _make_bnode=_make_bnode, _make_literal=_make_literal, @@ -348,12 +350,17 @@ def __init__(self, shapes_graph, data_graph, *args, **kwargs): context.eval_js(blankNodeJs) context.eval_js(literalJs) context.eval_js(graphJs) - context.set_globals(_native_shapes_graph=GraphNativeWrapper(shapes_graph)) + context.set_globals(_native_data_graph=GraphNativeWrapper(data_graph)) - context.eval_js('''\ - var $data = new Graph(_native_data_graph); - var $shapes = new Graph(_native_shapes_graph); - ''') + if shapes_graph is not None: + context.set_globals(_native_shapes_graph=GraphNativeWrapper(shapes_graph)) + else: + context.set_globals(_native_shapes_graph=None) + context.eval_js('''var $data = new Graph(_native_data_graph);\n''') + if shapes_graph is not None: + context.eval_js('''var $shapes = new Graph(_native_shapes_graph);\n''') + else: + context.eval_js('''var $shapes;\n''') # leave $shapes undefined context.set_globals(*args, **kwargs) self.context = context @@ -364,7 +371,7 @@ def load_js_library(self, library: str): self.fns.update(fns) @classmethod - def build_results(cls, res): + def build_results_as_constraint(cls, res): if isinstance(res, JSProxy): try: return res.toPython() @@ -425,6 +432,97 @@ def build_results(cls, res): except AttributeError: # This must be an array of something else res = keys + elif isinstance(res, (Literal, URIRef, BNode)): + pass + elif isinstance(res, Decimal): + res = Literal(res, datatype=XSD['decimal']) + elif isinstance(res, (int, float, str, bytes, bool)): + res = Literal(res) + return res + + @classmethod + def build_results_as_construct(cls, res): + if isinstance(res, JSProxy): + try: + return res.toPython() + except AttributeError: + pass + # this means its a JS Array or Object + keys = list(iter(res)) + if len(keys) < 1: + res = [] + else: + first_key = keys[0] + if isinstance(first_key, JSProxy): + # res is an array of objects or array of arrays + new_res = [] + for k in keys: + try: + new_res.append(k.toPython()) + continue + except AttributeError: + pass + subkeys = list(iter(k)) + if len(subkeys) < 1: + new_res.append([]) + else: + first_subkey = subkeys[0] + if isinstance(first_subkey, JSProxy): + if len(subkeys) >= 3: + # res is an array of arrays + this_s = subkeys[0] + this_p = subkeys[1] + this_o = subkeys[2] + else: + raise ReportableRuntimeError("JS Function returned incorrect number of items in the array.") + else: + this_s = getattr(k, 'subject', None) + this_p = getattr(k, 'predicate', None) + this_o = getattr(k, 'object', None) + try: + this_s = this_s.toPython() + except AttributeError: + pass + if this_s is not None and hasattr(this_s, 'inner'): + this_s = this_s.inner + try: + this_p = this_p.toPython() + except AttributeError: + pass + if this_p is not None and hasattr(this_p, 'inner'): + this_p = this_p.inner + try: + this_o = this_o.toPython() + except AttributeError: + pass + if this_o is not None and hasattr(this_o, 'inner'): + this_o = this_o.inner + new_res.append((this_s, this_p, this_o)) + return new_res + else: + # a JS Rules function must return an array of arrays, or array of objects + # otherwise, it does nothing! + return [] + return res + + @classmethod + def build_results_as_shacl_function(cls, res, return_type=None): + if isinstance(res, JSProxy): + try: + return res.toPython() + except AttributeError: + return None + elif isinstance(res, (Literal, URIRef, BNode)): + pass + elif return_type is not None and isinstance(res, (int, float)): + lex = str(res) + res = Literal(lex, datatype=return_type, normalize=False) + elif isinstance(res, (float, Decimal)): + res = Literal(str(res), datatype=XSD['decimal']) + elif isinstance(res, bool): + res = Literal(res, datatype=XSD['boolean']) + elif isinstance(res, (int, float, str, bytes, bool)): + res = Literal(res, datatype=return_type, normalize=False) return res def get_fn_args(self, fn_name, args_map): @@ -504,6 +602,5 @@ def run_js_function(self, fn_name, args, returns: list = None): except BaseException as e: print(e) returns_dict[r] = None - res = self.build_results(res) returns_dict['_result'] = res return returns_dict diff --git a/pyshacl/extras/js/function.py b/pyshacl/extras/js/function.py new file mode 100644 index 0000000..48db7c3 --- /dev/null +++ b/pyshacl/extras/js/function.py @@ -0,0 +1,71 @@ +# +# +import typing +from rdflib.plugins.sparql.operators import register_custom_function, unregister_custom_function +from rdflib.plugins.sparql.sparql import SPARQLError + +from pyshacl.consts import SH +from pyshacl.functions import SHACLFunction +from pyshacl.errors import ReportableRuntimeError +from .js_executable import JSExecutable + +if typing.TYPE_CHECKING: + from pyshacl.pytypes import GraphLike + from pyshacl.shapes_graph import ShapesGraph + +SH_JSFunction = SH.term('JSFunction') + +class JSFunction(SHACLFunction): + __slots__ = ('js_exe',) + + def __init__(self, fn_node, shapes_graph: 'ShapesGraph'): + super(JSFunction, self).__init__(fn_node, shapes_graph) + self.js_exe = JSExecutable(shapes_graph, fn_node) + + def execute(self, g, *args): + params = self.get_params_in_order() + if len(args) != len(params): + raise ReportableRuntimeError("Got incorrect number of arguments for JSFunction {}.".format(self.node)) + args_map = {} + for i, p in enumerate(params): + arg = args[i] + ln = p.localname + if arg is None and p.optional is False: + raise ReportableRuntimeError("Got NoneType for Non-optional argument {}.".format(ln)) + args_map[ln] = arg + results = self.js_exe.execute(g, args_map, mode="function", return_type=self.rtype) + res = results['_result'] + return res + + def execute_from_sparql(self, e, ctx): + if not e.expr: + raise SPARQLError("Nothing given to SPARQLFunction.") + params = self.get_params_in_order() + num_params = len(params) + if len(e.expr) > num_params: + raise SPARQLError("Too many parameters passed to SPARQLFunction.") + elif len(e.expr) < num_params: + raise SPARQLError("Too few parameters passed to SPARQLFunction.") + args_map = {str(var): val for var, val in ctx.ctx.initBindings.items()} + args_map.update({str(var): val for var, val in ctx.ctx.bindings.items()}) + g = ctx.ctx.graph + for i, var in enumerate(e.expr): + var_val = ctx[var] + param_name = params[i].localname + args_map[param_name] = var_val + results = self.js_exe.execute(g, args_map, mode="function", return_type=self.rtype) + res = results['_result'] + return res + + def apply(self, g): + super(JSFunction, self).apply(g) + register_custom_function(self.node, self.execute_from_sparql, True, True) + + def unapply(self, g): + super(JSFunction, self).unapply(g) + unregister_custom_function(self.node, self.execute_from_sparql) + + + + + diff --git a/pyshacl/extras/js/js_executable.py b/pyshacl/extras/js/js_executable.py index 551254f..da22c2c 100644 --- a/pyshacl/extras/js/js_executable.py +++ b/pyshacl/extras/js/js_executable.py @@ -87,10 +87,32 @@ def __init__(self, shapes_graph: 'ShapesGraph', node): libraries[libn2].append(str(u2)) self.libraries = libraries - def execute(self, datagraph, args_map, *args, **kwargs): - ctx = SHACLJSContext(self.sg, datagraph, **kwargs) + def execute(self, data_graph, args_map, *args, mode=None, return_type=None, **kwargs): + """ + :param data_graph: + :param args_map: + :param args: + :param mode: + :param return_type: + :param kwargs: + :return: + :rtype: dict + """ + if mode == "function": + ctx = SHACLJSContext(data_graph, shapes_graph=None, **kwargs) + else: + ctx = SHACLJSContext(data_graph, shapes_graph=self.sg, **kwargs) + for lib_node, lib_urls in self.libraries.items(): for lib_url in lib_urls: ctx.load_js_library(lib_url) fn_args = ctx.get_fn_args(self.fn_name, args_map) - return ctx.run_js_function(self.fn_name, fn_args) + rvals = ctx.run_js_function(self.fn_name, fn_args) + res = rvals['_result'] + if mode == "function": + rvals['_result'] = ctx.build_results_as_shacl_function(res, return_type) + elif mode == "construct": + rvals['_result'] = ctx.build_results_as_construct(res) + else: + rvals['_result'] = ctx.build_results_as_constraint(res) + return rvals diff --git a/pyshacl/extras/js/rules.py b/pyshacl/extras/js/rules.py new file mode 100644 index 0000000..d8f46fa --- /dev/null +++ b/pyshacl/extras/js/rules.py @@ -0,0 +1,48 @@ +# +# +import typing +from rdflib.plugins.sparql.operators import register_custom_function, unregister_custom_function +from rdflib.plugins.sparql.sparql import SPARQLError + +from pyshacl.consts import SH +from pyshacl.rules.shacl_rule import SHACLRule +from pyshacl.errors import ReportableRuntimeError +from .js_executable import JSExecutable + +if typing.TYPE_CHECKING: + from pyshacl.shapes_graph import ShapesGraph + from pyshacl.shape import Shape + +SH_JSRule = SH.term('JSRule') + +class JSRule(SHACLRule): + __slots__ = ('js_exe',) + + def __init__(self, shape: 'Shape', rule_node): + super(JSRule, self).__init__(shape, rule_node) + shapes_graph = shape.sg # type: ShapesGraph + self.js_exe = JSExecutable(shapes_graph, rule_node) + + def apply(self, data_graph): + focus_nodes = self.shape.focus_nodes(data_graph) # uses target nodes to find focus nodes + applicable_nodes = self.filter_conditions(focus_nodes, data_graph) + sets_to_add = [] + for a in applicable_nodes: + args_map = {"this": a} + results = self.js_exe.execute(data_graph, args_map, mode="construct") + triples = results['_result'] + if triples is not None and isinstance(triples, (list, tuple)): + set_to_add = set() + for t in triples: + s,p,o = t[:3] + set_to_add.add((s,p,o)) + sets_to_add.append(set_to_add) + for s in sets_to_add: + for t in s: + data_graph.add(t) + return + + + + + diff --git a/pyshacl/functions/__init__.py b/pyshacl/functions/__init__.py index b21bd72..a2b8d94 100644 --- a/pyshacl/functions/__init__.py +++ b/pyshacl/functions/__init__.py @@ -4,12 +4,11 @@ from typing import List, Sequence, Union -from pyshacl.consts import RDF_type, SH_ask, SH_select, SH_SHACLFunction, SH_SPARQLFunction +from pyshacl.consts import RDF_type, SH_ask, SH_select, SH_SHACLFunction, SH_SPARQLFunction, SH_jsLibrary, \ + SH_jsFunctionName from pyshacl.pytypes import GraphLike - from .shacl_function import SHACLFunction, SPARQLFunction - if typing.TYPE_CHECKING: from pyshacl.shapes_graph import ShapesGraph @@ -22,23 +21,46 @@ def gather_functions(shacl_graph: 'ShapesGraph') -> Sequence[Union['SHACLFunctio :return: :rtype: [SHACLRule] """ + + spq_nodes = set(shacl_graph.subjects(RDF_type, SH_SPARQLFunction)) - scl_nodes = set(shacl_graph.subjects(RDF_type, SH_SHACLFunction)).difference(spq_nodes) - to_swap = set() + if shacl_graph.js_enabled: + use_js = True + from pyshacl.extras.js.function import JSFunction, SH_JSFunction + js_nodes = set(shacl_graph.subjects(RDF_type, SH_JSFunction)) + else: + use_js = False + JSFunction = object # for error checking purposes, needs to be defined + js_nodes = set() + scl_nodes = set(shacl_graph.subjects(RDF_type, SH_SHACLFunction)).difference(spq_nodes).difference(js_nodes) + to_swap_spq = set() + to_swap_js = set() for n in scl_nodes: has_select = len(shacl_graph.objects(n, SH_select)) > 0 has_ask = len(shacl_graph.objects(n, SH_ask)) > 0 if has_ask or has_select: - to_swap.add(n) - for n in to_swap: + to_swap_spq.add(n) + continue + if use_js: + has_jslibrary = len(shacl_graph.objects(n, SH_jsLibrary)) > 0 + has_jsfuncitonnname = len(shacl_graph.objects(n, SH_jsFunctionName)) > 0 + if has_jslibrary or has_jsfuncitonnname: + to_swap_js.add(n) + for n in to_swap_spq: scl_nodes.remove(n) spq_nodes.add(n) + for n in to_swap_js: + scl_nodes.remove(n) + js_nodes.add(n) - all_fns: List[Union['SHACLFunction', 'SPARQLFunction']] = [] + all_fns: List[Union['SHACLFunction', 'SPARQLFunction', 'JSFunction']] = [] for n in spq_nodes: all_fns.append(SPARQLFunction(n, shacl_graph)) for n in scl_nodes: all_fns.append(SHACLFunction(n, shacl_graph)) + if use_js: + for n in js_nodes: + all_fns.append(JSFunction(n, shacl_graph)) return all_fns diff --git a/pyshacl/rules/__init__.py b/pyshacl/rules/__init__.py index 0e875d0..00b021b 100644 --- a/pyshacl/rules/__init__.py +++ b/pyshacl/rules/__init__.py @@ -25,12 +25,32 @@ def gather_rules(shacl_graph: 'ShapesGraph') -> Dict['Shape', List['SHACLRule']] """ triple_rule_nodes = set(shacl_graph.subjects(RDF_type, SH_TripleRule)) sparql_rule_nodes = set(shacl_graph.subjects(RDF_type, SH_SPARQLRule)) + if shacl_graph.js_enabled: + use_js = True + from pyshacl.extras.js.rules import JSRule, SH_JSRule + js_rule_nodes = set(shacl_graph.subjects(RDF_type, SH_JSRule)) + else: + use_js = False + js_rule_nodes = set() + JSRule = object # to keep the linter happy overlaps = triple_rule_nodes.intersection(sparql_rule_nodes) if len(overlaps) > 0: raise RuleLoadError( "A SHACL Rule cannot be both a TripleRule and a SPARQLRule.", "https://www.w3.org/TR/shacl-af/#rules-syntax", ) + overlaps = triple_rule_nodes.intersection(js_rule_nodes) + if len(overlaps) > 0: + raise RuleLoadError( + "A SHACL Rule cannot be both a TripleRule and a JSRule.", + "https://www.w3.org/TR/shacl-af/#rules-syntax", + ) + overlaps = sparql_rule_nodes.intersection(js_rule_nodes) + if len(overlaps) > 0: + raise RuleLoadError( + "A SHACL Rule cannot be both a SPARQLRule and a JSRule.", + "https://www.w3.org/TR/shacl-af/#rules-syntax", + ) used_rules = shacl_graph.subject_objects(SH_rule) ret_rules = defaultdict(list) for sub, obj in used_rules: @@ -45,6 +65,8 @@ def gather_rules(shacl_graph: 'ShapesGraph') -> Dict['Shape', List['SHACLRule']] rule: SHACLRule = TripleRule(shape, obj) elif obj in sparql_rule_nodes: rule = SPARQLRule(shape, obj) + elif use_js and obj in js_rule_nodes: + rule = JSRule(shape, obj) else: raise RuleLoadError( "when using sh:rule, the Rule must be defined as either a TripleRule or SPARQLRule.", diff --git a/test/resources/js/multiply.js b/test/resources/js/multiply.js new file mode 100644 index 0000000..cf583cd --- /dev/null +++ b/test/resources/js/multiply.js @@ -0,0 +1,3 @@ +function multiply($op1, $op2) { + return $op1.lex * $op2.lex; +} diff --git a/test/resources/js/rectangle.js b/test/resources/js/rectangle.js new file mode 100644 index 0000000..dc69965 --- /dev/null +++ b/test/resources/js/rectangle.js @@ -0,0 +1,18 @@ +var NS = "http://datashapes.org/js/tests/rules/rectangle.test#"; + +function computeArea($this) { + var width = getProperty($this, "width"); + var height = getProperty($this, "height"); + var area = TermFactory.literal(width.lex * height.lex, width.datatype); + var areaProperty = TermFactory.namedNode(NS + "area"); + return [ + [$this, areaProperty, area] + ]; +} + +function getProperty($this, name) { + var it = $data.find($this, TermFactory.namedNode(NS + name), null); + var result = it.next().object; + it.close(); + return result; +} diff --git a/test/test_js/__init__.py b/test/test_js/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/js/test_js_constraint.py b/test/test_js/test_js_constraint.py similarity index 100% rename from test/js/test_js_constraint.py rename to test/test_js/test_js_constraint.py diff --git a/test/js/test_js_constraint_component.py b/test/test_js/test_js_constraint_component.py similarity index 100% rename from test/js/test_js_constraint_component.py rename to test/test_js/test_js_constraint_component.py diff --git a/test/js/test_js_constraint_path_component.py b/test/test_js/test_js_constraint_path_component.py similarity index 100% rename from test/js/test_js_constraint_path_component.py rename to test/test_js/test_js_constraint_path_component.py diff --git a/test/test_js/test_js_function.py b/test/test_js/test_js_function.py new file mode 100644 index 0000000..65a014e --- /dev/null +++ b/test/test_js/test_js_function.py @@ -0,0 +1,106 @@ +from rdflib import Graph +from pyshacl import validate +shapes_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix xsd: . +@prefix ex: . + +ex:Rectangle + rdf:type rdfs:Class ; + rdf:type sh:NodeShape ; + rdfs:label "Rectangle" ; + rdfs:subClassOf rdfs:Resource ; + sh:property [ + sh:path ex:height ; + sh:datatype xsd:integer ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:name "height" ; + ] ; + sh:property [ + sh:path ex:width ; + sh:datatype xsd:integer ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:name "width" ; + ] ; + sh:property [ + sh:path ex:area ; + sh:datatype xsd:integer ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:name "area" ; + ] ; +. + +ex:CheckArea + rdf:type sh:PropertyShape ; + sh:path ex:area ; + sh:sparql ex:CheckArea-sparql ; + sh:targetClass ex:Rectangle ; +. +ex:CheckArea-sparql + rdf:type sh:SPARQLConstraintObject ; + sh:message "Height * Width = Area." ; + sh:prefixes ; + sh:select """ + SELECT $this ?value + WHERE { + $this ex:width ?width . + $this ex:height ?height . + $this $PATH ?value . + FILTER (ex:multiply(?width, ?height) != ?value) + } + """ ; +. +ex:multiply + a sh:JSFunction ; + rdfs:comment "Multiplies its two arguments $op1 and $op2." ; + sh:parameter [ + sh:path ex:op1 ; + sh:datatype xsd:integer ; + sh:description "The first operand" ; + ] ; + sh:parameter [ + sh:path ex:op2 ; + sh:datatype xsd:integer ; + sh:description "The second operand" ; + ] ; + sh:returnType xsd:integer ; + sh:jsLibrary [ sh:jsLibraryURL "file://resources/js/multiply.js" ] ; + sh:jsFunctionName "multiply" ; +. +''' + +data_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix ex: . +@prefix exdata: . + +exdata:NonSquareRectangle + rdf:type ex:Rectangle ; + ex:height 3 ; + ex:width 4 ; + ex:area 12 ; +. +exdata:SquareRectangle + rdf:type ex:Rectangle ; + ex:height 6 ; + ex:width 6 ; + ex:area 12 ; +. + +''' + +def test_js_function(): + s1 = Graph().parse(data=shapes_graph, format="turtle") + g1 = Graph().parse(data=data_graph, format="turtle") + conforms, result_graph, result_text = validate(g1, shacl_graph=s1, advanced=True, debug=True, js=True) + assert not conforms + +if __name__ == "__main__": + test_js_function() diff --git a/test/test_js/test_js_rules.py b/test/test_js/test_js_rules.py new file mode 100644 index 0000000..602f59c --- /dev/null +++ b/test/test_js/test_js_rules.py @@ -0,0 +1,40 @@ +from rdflib import Graph +from pyshacl import validate +shapes_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix xsd: . +@prefix ex: . + +ex:RectangleShape + a sh:NodeShape ; + sh:targetClass ex:Rectangle ; + sh:rule [ + a sh:JSRule ; # This triple is optional + sh:jsFunctionName "computeArea" ; + sh:jsLibrary [ sh:jsLibraryURL "resources/js/rectangle.js"^^xsd:anyURI ] ; + ] . +''' + +data_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix ex: . +@prefix exdata: . + +exdata:ExampleRectangle + a ex:Rectangle ; + ex:width 7 ; + ex:height 8 . +''' + +def test_js_rules(): + s1 = Graph().parse(data=shapes_graph, format="turtle") + g1 = Graph().parse(data=data_graph, format="turtle") + conforms, result_graph, result_text = validate(g1, shacl_graph=s1, advanced=True, debug=True, js=True) + assert not conforms + +if __name__ == "__main__": + test_js_rules() From 524b3261eca055bd8d6c2a53e87594ba15f941c3 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Wed, 14 Oct 2020 09:15:33 +1000 Subject: [PATCH 06/10] Fix test for JSRule --- test/test_js/test_js_rules.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_js/test_js_rules.py b/test/test_js/test_js_rules.py index 602f59c..e278089 100644 --- a/test/test_js/test_js_rules.py +++ b/test/test_js/test_js_rules.py @@ -14,6 +14,12 @@ a sh:JSRule ; # This triple is optional sh:jsFunctionName "computeArea" ; sh:jsLibrary [ sh:jsLibraryURL "resources/js/rectangle.js"^^xsd:anyURI ] ; + ] ; + sh:property [ + sh:path ex:area ; + sh:datatype xsd:double ; + sh:minCount 1 ; + sh:maxCount 1 ; ] . ''' From be5675949c314c6790857b32ac164110203efed3 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Wed, 14 Oct 2020 11:46:21 +1000 Subject: [PATCH 07/10] Implement JSTarget and JSTargetType Enhance the way custom targets detected and used by the Shapes Graph Update features matrix --- FEATURES.md | 40 ++++++ pyshacl/constraints/core/value_constraints.py | 4 +- pyshacl/consts.py | 3 + pyshacl/extras/js/context.py | 33 ++++- pyshacl/extras/js/js_executable.py | 2 + pyshacl/extras/js/target.py | 86 +++++++++++++ pyshacl/parameter.py | 20 ++- pyshacl/shape.py | 44 +++++-- pyshacl/target.py | 18 ++- test/resources/js/findBornIn.js | 11 ++ test/resources/js/findThings.js | 12 ++ test/test_js/test_js_target.py | 80 ++++++++++++ test/test_js/test_js_target_type.py | 121 ++++++++++++++++++ 13 files changed, 457 insertions(+), 17 deletions(-) create mode 100644 pyshacl/extras/js/target.py create mode 100644 test/resources/js/findBornIn.js create mode 100644 test/resources/js/findThings.js create mode 100644 test/test_js/test_js_target.py create mode 100644 test/test_js/test_js_target_type.py diff --git a/FEATURES.md b/FEATURES.md index 9b408d3..6e61abd 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -160,6 +160,39 @@ | `sh:TripleRule` | [â–¶][AFTripleRule] | ![status-complete] | | | `sh:SPARQLRule` | [â–¶][AFSPARQLRule] | ![status-complete] | | +# SHACL-JS [spec](https://www.w3.org/TR/shacl-js/) + +The SHACL-JS features are implemented behind a Python "extras" feature. +To enable it, you must install PySHACL using PIP with the extras included like `pyshacl[js]` + +### [Javascript-based Constraints](https://www.w3.org/TR/shacl-js/#js-constraints) +| Parameter | Link | Status | Comments | +|:---------- |:------: |:-----------------: |:------ | +| `sh:js` | [â–¶][JSConstraintValidation] | ![status-complete] | | + +### [Javascript-based Constraints-Components](https://www.w3.org/TR/shacl-js/#js-components) +| Parameter | Link | Status | Comments | +|:---------- |:------: |:-----------------: |:------ | +| `sh:validator` | [â–¶][JSConstraintComponentValidation] | ![status-complete] | | +| `sh:JSValidator` | [â–¶][JSConstraintComponentValidation] | ![status-complete] | | + +### [Javascript-based SHACL Functions](https://www.w3.org/TR/shacl-js/#js-functions) +| Parameter | Link | Status | Comments | +|:---------- |:------: |:-----------------: |:------ | +| `sh:JSFunction` | [â–¶][JSFunctionsSyntax] | ![status-complete] | | +| `sh:parameter` | [â–¶][JSFunctionsSyntax] | ![status-complete] | | + +### [Javascript-based SHACL Rules](https://www.w3.org/TR/shacl-js/#rules) +| Parameter | Link | Status | Comments | +|:---------- |:------: |:-----------------: |:------ | +| `sh:JSRule` | [â–¶][JSRulesExecution] | ![status-complete] | | + +### [Javascript-based Custom Targets](https://www.w3.org/TR/shacl-js/#targets) +| Parameter | Link | Status | Comments | +|:---------- |:------: |:-----------------: |:------ | +| `sh:JSTarget` | [â–¶][JSTarget] | ![status-complete] | | +| `sh:JSTargetType` | [â–¶][JSTargetType] | ![status-complete] | | + # Implementation Notes @@ -240,3 +273,10 @@ [AFEntailment]: https://www.w3.org/TR/shacl-af/#Rules [AFTripleRule]: https://www.w3.org/TR/shacl-af/#TripleRule [AFSPARQLRule]: https://www.w3.org/TR/shacl-af/#SPARQLRule + +[JSConstraintValidation]: https://www.w3.org/TR/shacl-js/#js-constraints-validation +[JSConstraintComponentValidation]: https://www.w3.org/TR/shacl-js/#validation-of-javascript-based-constraint-components +[JSFunctionsSyntax]: https://www.w3.org/TR/shacl-js/#syntax-and-semantics-of-javascript-based-functions +[JSRulesExecution]: https://www.w3.org/TR/shacl-js/#rules-execution +[JSTarget]: https://www.w3.org/TR/shacl-js/#JSTarget +[JSTargetType]: https://www.w3.org/TR/shacl-js/#JSTargetType diff --git a/pyshacl/constraints/core/value_constraints.py b/pyshacl/constraints/core/value_constraints.py index 4da8dc4..f883d4c 100644 --- a/pyshacl/constraints/core/value_constraints.py +++ b/pyshacl/constraints/core/value_constraints.py @@ -21,6 +21,8 @@ SH_BlankNodeORLiteral, SH_IRIOrLiteral, SH_Literal, + SH_nodeKind, + SH_datatype ) from pyshacl.errors import ConstraintLoadError from pyshacl.pytypes import GraphLike @@ -37,8 +39,6 @@ XSD_dateTime = XSD.term('dateTime') SH_class = SH.term('class') -SH_datatype = SH.term('datatype') -SH_nodeKind = SH.term('nodeKind') SH_ClassConstraintComponent = SH.term('ClassConstraintComponent') SH_DatatypeConstraintComponent = SH.term('DatatypeConstraintComponent') SH_NodeKindConstraintComponent = SH.term('NodeKindConstraintComponent') diff --git a/pyshacl/consts.py b/pyshacl/consts.py index af1ad33..010ba08 100644 --- a/pyshacl/consts.py +++ b/pyshacl/consts.py @@ -33,6 +33,8 @@ SH_TripleRule = SH.term('TripleRule') SH_SPARQLTarget = SH.term('SPARQLTarget') SH_SPARQLTargetType = SH.term('SPARQLTargetType') +SH_JSTarget = SH.term('JSTarget') +SH_JSTargetType = SH.term('JSTargetType') # predicates RDF_type = RDF.term('type') @@ -90,6 +92,7 @@ SH_union = SH.term('union') SH_intersection = SH.term('intersection') SH_datatype = SH.term('datatype') +SH_nodeKind = SH.term('nodeKind') SH_optional = SH.term('optional') SH_js = SH.term('js') SH_jsFunctionName = SH.term('jsFunctionName') diff --git a/pyshacl/extras/js/context.py b/pyshacl/extras/js/context.py index 1783c52..b68075a 100644 --- a/pyshacl/extras/js/context.py +++ b/pyshacl/extras/js/context.py @@ -505,6 +505,37 @@ def build_results_as_construct(cls, res): return [] return res + @classmethod + def build_results_as_target(cls, res): + if isinstance(res, JSProxy): + try: + return res.toPython() + except AttributeError: + pass + # this means its a JS Array or Object + keys = list(iter(res)) + if len(keys) < 1: + res = [] + else: + first_key = keys[0] + if isinstance(first_key, JSProxy): + # res is an array of objects or array of arrays + new_res = [] + for k in keys: + try: + k = k.toPython() + except AttributeError: + pass + if k is not None and hasattr(k, 'inner'): + k = k.inner + new_res.append(k) + return new_res + else: + # a JS Target function must return an array of Nodes + # otherwise, it does nothing! + return [] + return res + @classmethod def build_results_as_shacl_function(cls, res, return_type=None): if isinstance(res, JSProxy): @@ -532,7 +563,7 @@ def get_fn_args(self, fn_name, args_map): except BaseException as e: print(e) raise - if not fn: + if fn is None: raise ReportableRuntimeError("JS Function {} cannot be found in the loaded files.".format(fn_name)) if fn_name not in self.fns: raise ReportableRuntimeError("JS Function {} args cannot be determined. Bad JS structure?".format(fn_name)) diff --git a/pyshacl/extras/js/js_executable.py b/pyshacl/extras/js/js_executable.py index da22c2c..477a956 100644 --- a/pyshacl/extras/js/js_executable.py +++ b/pyshacl/extras/js/js_executable.py @@ -113,6 +113,8 @@ def execute(self, data_graph, args_map, *args, mode=None, return_type=None, **kw rvals['_result'] = ctx.build_results_as_shacl_function(res, return_type) elif mode == "construct": rvals['_result'] = ctx.build_results_as_construct(res) + elif mode == 'target': + rvals['_result'] = ctx.build_results_as_target(res) else: rvals['_result'] = ctx.build_results_as_constraint(res) return rvals diff --git a/pyshacl/extras/js/target.py b/pyshacl/extras/js/target.py new file mode 100644 index 0000000..cb51854 --- /dev/null +++ b/pyshacl/extras/js/target.py @@ -0,0 +1,86 @@ +# +# +import typing +from typing import List, Dict +from warnings import warn + +from rdflib import URIRef +from pyshacl.target import SHACLTargetType, BoundSHACLTargetType +from pyshacl.consts import SH, SH_JSTarget, SH_JSTargetType +from .js_executable import JSExecutable +from ...errors import ShapeLoadError + +if typing.TYPE_CHECKING: + from pyshacl.pytypes import GraphLike + from pyshacl.shapes_graph import ShapesGraph + from pyshacl.shape import Shape + + +class JSTarget(JSExecutable): + + def __init__(self, shapes_graph: 'ShapesGraph', exe_node): + super(JSTarget, self).__init__(shapes_graph, exe_node) + + def find_targets(self, data_graph): + results = self.execute(data_graph, {}, mode='target') + return [u for u in results['_result'] if isinstance(u, URIRef)] + + +class BoundJSTargetType(BoundSHACLTargetType): + __slots__ = ('params_kv',) + + def __init__(self, target_type: 'JSTargetType', target_declaration, shape: 'Shape', params_kv): + super(BoundJSTargetType, self).__init__(target_type, target_declaration, shape) + self.params_kv = params_kv # type: dict + + @classmethod + def constraint_parameters(cls): + return [] + + @classmethod + def constraint_name(cls): + return "JSTargetType" + + @classmethod + def shacl_constraint_class(cls): + return SH_JSTargetType + + def evaluate(self, target_graph: 'GraphLike', focus_value_nodes: Dict, _evaluation_path: List): + raise NotImplementedError() + + def find_targets(self, data_graph): + results = self.target_type.js_exe.execute(data_graph, self.params_kv, mode='target') + return [u for u in results['_result'] if isinstance(u, URIRef)] + + +class JSTargetType(SHACLTargetType): + __slots__ = ('js_exe',) + + def __init__(self, tt_node, sg: 'ShapesGraph'): + super(JSTargetType, self).__init__(tt_node, sg) + self.js_exe = JSExecutable(sg, tt_node) + + def check_params(self, target_declaration): + param_kv = {} + for p in self.parameters: + path = p.path() + name = p.localname + vals = set(self.sg.objects(target_declaration, path)) + if len(vals) < 1: + if p.optional: + continue + raise ShapeLoadError( + "sh:target does not have a value for {}".format(name), + "https://www.w3.org/TR/shacl-js/#JSTargetType", + ) + if len(vals) > 1: + warn(Warning("Found more than one value for {} on sh:target. Using just first one.".format(n))) + param_kv[name] = next(iter(vals)) + return param_kv + + def bind(self, shape, target_declaration): + param_vals = self.check_params(target_declaration) + return BoundJSTargetType(self, target_declaration, shape, param_vals) + + + diff --git a/pyshacl/parameter.py b/pyshacl/parameter.py index dcd9cec..5ce37ec 100644 --- a/pyshacl/parameter.py +++ b/pyshacl/parameter.py @@ -3,13 +3,13 @@ from rdflib import Literal, URIRef -from .consts import SH_datatype, SH_optional, SH_order, SH_path +from .consts import SH_datatype, SH_optional, SH_order, SH_path, SH_nodeKind from .errors import ConstraintLoadError, ReportableRuntimeError from .shape import Shape class SHACLParameter(Shape): - __slots__ = ("datatype", "param_order", "optional") + __slots__ = ("nodekind", "datatype", "param_order", "optional") def __init__(self, sg, param_node, path=None, logger: Union[Logger, None] = None): """ @@ -29,6 +29,17 @@ def __init__(self, sg, param_node, path=None, logger: Union[Logger, None] = None path = paths[0] super(SHACLParameter, self).__init__(sg, param_node, p=True, path=path, logger=logger) + nodekinds = list(sg.objects(self.node, SH_nodeKind)) + if len(nodekinds) < 1: + self.nodekind = None + elif len(nodekinds) > 1: + raise ConstraintLoadError( + "sh:parameter cannot have more than one value for sh:nodeKind.", + "https://www.w3.org/TR/shacl-af/#functions-example", + ) + else: + self.nodekind = nodekinds[0] + datatypes = list(sg.objects(self.node, SH_datatype)) if len(datatypes) < 1: self.datatype = None @@ -39,6 +50,7 @@ def __init__(self, sg, param_node, path=None, logger: Union[Logger, None] = None ) else: self.datatype = datatypes[0] + orders = list(sg.objects(self.node, SH_order)) if len(orders) < 1: self.param_order = None @@ -68,6 +80,10 @@ def __init__(self, sg, param_node, path=None, logger: Union[Logger, None] = None ) self.optional = bool(optionals[0]) + def __str__(self): + name = str(self.node) + return "".format(name) + @property def localname(self): path = self.path() diff --git a/pyshacl/shape.py b/pyshacl/shape.py index 54a7730..2f6e93d 100644 --- a/pyshacl/shape.py +++ b/pyshacl/shape.py @@ -30,7 +30,7 @@ SH_targetSubjectsOf, SH_Violation, SH_zeroOrMorePath, - SH_zeroOrOnePath, + SH_zeroOrOnePath, SH_jsFunctionName, SH_JSTarget, SH_JSTargetType, ) from .errors import ConstraintLoadError, ConstraintLoadWarning, ReportableRuntimeError, ShapeLoadError from .helper import get_query_helper_cls @@ -230,10 +230,18 @@ def target(self): def advanced_target(self): custom_targets = set(self.sg.objects(self.node, SH_target)) result_set = dict() + if self.sg.js_enabled: + use_js = True + from pyshacl.extras.js.target import JSTarget + else: + use_js = False + JSTarget = object # for linter for c in custom_targets: ct = dict() selects = list(self.sg.objects(c, SH_select)) has_select = len(selects) > 0 + fn_names = list(self.sg.objects(c, SH_jsFunctionName)) + has_fnname = len(fn_names) > 0 is_types = set(self.sg.objects(c, RDF_type)) if has_select or (SH_SPARQLTarget in is_types): ct['type'] = SH_SPARQLTarget @@ -241,6 +249,13 @@ def advanced_target(self): qh = SPARQLQueryHelper(self, c, selects[0], deactivated=self._deactivated) qh.collect_prefixes() ct['qh'] = qh + elif has_fnname or (SH_JSTarget in is_types): + if use_js: + ct['type'] = SH_JSTarget + ct['targeter'] = JSTarget(self.sg, c) + else: + # Found JSTarget, but JS is not enabled in PySHACL. Ignore this target. + pass else: found_tt = None for t in is_types: @@ -252,9 +267,12 @@ def advanced_target(self): if not found_tt: msg = "None of these types match a TargetType: {}".format(" ".join(is_types)) raise ShapeLoadError(msg, "https://www.w3.org/TR/shacl-af/#SPARQLTargetType") - ct['type'] = SH_SPARQLTargetType bound_tt = found_tt.bind(self, c) - ct['qt'] = bound_tt + ct['type'] = bound_tt.shacl_constraint_class() + if ct['type'] == SH_SPARQLTargetType: + ct['qt'] = bound_tt + elif ct['type'] == SH_JSTargetType: + ct['targeter'] = bound_tt result_set[c] = ct return result_set @@ -305,14 +323,22 @@ def focus_nodes(self, data_graph): qh = at['qh'] select = qh.apply_prefixes(qh.select_text) results = data_graph.query(select, initBindings=None) + if not results or len(results.bindings) < 1: + continue + for r in results: + t = r['this'] + found_node_targets.add(t) + elif at['type'] in (SH_JSTarget, SH_JSTargetType): + results = at['targeter'].find_targets(data_graph) + for r in results: + found_node_targets.add(r) else: results = at['qt'].find_targets(data_graph) - if not results or len(results.bindings) < 1: - continue - for r in results: - t = r['this'] - found_node_targets.add(t) - + if not results or len(results.bindings) < 1: + continue + for r in results: + t = r['this'] + found_node_targets.add(t) return found_node_targets @classmethod diff --git a/pyshacl/target.py b/pyshacl/target.py index a964346..704dfe5 100644 --- a/pyshacl/target.py +++ b/pyshacl/target.py @@ -53,6 +53,7 @@ def apply(self): self.sg.add_shacl_target_type(self.node, self) def check_params(self, target_declaration): + assert False # is this even used? param_kv = {} for p in self.parameters: n = p.node @@ -70,6 +71,7 @@ def check_params(self, target_declaration): return param_kv def bind(self, shape, target_declaration): + assert False # is this even used? param_vals = self.check_params(target_declaration) return BoundSHACLTargetType(self, target_declaration, shape, param_vals) @@ -110,11 +112,11 @@ def constraint_parameters(cls): @classmethod def constraint_name(cls): - return "TargetType" + return "SPARQLTargetType" @classmethod def shacl_constraint_class(cls): - return SH_TargetType + return SH_SPARQLTargetType def evaluate(self, target_graph: GraphLike, focus_value_nodes: typing.Dict, _evaluation_path: List): raise NotImplementedError() @@ -184,6 +186,13 @@ def gather_target_types(shacl_graph: 'ShapesGraph') -> Sequence[Union['SHACLTarg # remove these two which are the known native types in shacl.ttl sub_targets = sub_targets.difference({SH_JSTarget, SH_SPARQLTarget}) + if shacl_graph.js_enabled: + use_js = True + from pyshacl.extras.js.target import JSTargetType + else: + use_js = False + JSTargetType = object # for linter + for s in sub_targets: types = set(shacl_graph.objects(s, RDF_type)) found = False @@ -191,8 +200,11 @@ def gather_target_types(shacl_graph: 'ShapesGraph') -> Sequence[Union['SHACLTarg all_target_types.append(SPARQLTargetType(s, shacl_graph)) found = True if SH_JSTargetType in types: - warn(Warning("sh:JSTargetType is not implemented in PySHACL.\n<{}> a <{}>".format(s, SH_JSTargetType))) found = True + if use_js: + all_target_types.append(JSTargetType(s, shacl_graph)) + else: + pass # JS Mode is not enabled. Silently ignore JSTargetTypes if not found: warn(Warning("The only SHACLTargetType currently implemented is SPARQLTargetType.")) diff --git a/test/resources/js/findBornIn.js b/test/resources/js/findBornIn.js new file mode 100644 index 0000000..7dcc203 --- /dev/null +++ b/test/resources/js/findBornIn.js @@ -0,0 +1,11 @@ +var EXbornIn = TermFactory.namedNode("http://datashapes.org/sh/tests/js/target/jsTargetType-001.test#bornIn"); + +function findBornIn($country) { + var spo = $data.find(null, EXbornIn, $country); + var accum = []; + for(var t = spo.next(); t; t = spo.next()) { + var subject = t.subject; + accum.push(subject); + } + return accum; +} diff --git a/test/resources/js/findThings.js b/test/resources/js/findThings.js new file mode 100644 index 0000000..6aee7a4 --- /dev/null +++ b/test/resources/js/findThings.js @@ -0,0 +1,12 @@ +var RDFtype = TermFactory.namedNode("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"); +var OWLThing = TermFactory.namedNode("http://www.w3.org/2002/07/owl#Thing"); + +function findThings() { + var spo = $data.find(null, RDFtype, OWLThing); + var accum = []; + for(var t = spo.next(); t; t = spo.next()) { + var subject = t.subject; + accum.push(subject); + } + return accum; +} diff --git a/test/test_js/test_js_target.py b/test/test_js/test_js_target.py new file mode 100644 index 0000000..f41a4e3 --- /dev/null +++ b/test/test_js/test_js_target.py @@ -0,0 +1,80 @@ +from rdflib import Graph +from pyshacl import validate +shapes_graph = '''\ +@prefix dash: . +@prefix ex: . +@prefix owl: . +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix xsd: . + + + rdf:type owl:Ontology ; + rdfs:label "Test of sh:SPARQLTarget 001" ; + owl:imports ; +. +ex:GraphValidationTestCase + rdf:type dash:GraphValidationTestCase ; + dash:expectedResult [ + rdf:type sh:ValidationReport ; + sh:conforms "false"^^xsd:boolean ; + sh:result [ + rdf:type sh:ValidationResult ; + sh:focusNode ex:InvalidInstance1 ; + sh:resultPath rdfs:label ; + sh:resultSeverity sh:Violation ; + sh:sourceConstraintComponent sh:MaxCountConstraintComponent ; + sh:sourceShape ex:TestShape-label ; + ] ; + ] ; +. + +ex:TestShape + rdf:type sh:NodeShape ; + rdfs:label "Test shape" ; + sh:property ex:TestShape-label ; + sh:target [ + rdf:type sh:JSTarget ; + sh:jsFunctionName "findThings" ; + sh:jsLibrary [ sh:jsLibraryURL "resources/js/findThings.js"^^xsd:anyURI ] ; + ] ; +. + +ex:TestShape-label + sh:path rdfs:label ; + rdfs:comment "Must not have any rdfs:label" ; + rdfs:label "label" ; + sh:datatype xsd:string ; + sh:maxCount 0 ; +. + + +''' + +data_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix owl: . +@prefix ex: . +@prefix exdata: . + +exdata:InvalidInstance1 + rdf:type owl:Thing ; + rdfs:label "Invalid instance1" ; +. +exdata:ValidInstance1 + rdf:type owl:Thing ; +. + +''' + +def test_js_target(): + s1 = Graph().parse(data=shapes_graph, format="turtle") + g1 = Graph().parse(data=data_graph, format="turtle") + conforms, result_graph, result_text = validate(g1, shacl_graph=s1, advanced=True, debug=True, js=True) + assert not conforms + +if __name__ == "__main__": + test_js_target() diff --git a/test/test_js/test_js_target_type.py b/test/test_js/test_js_target_type.py new file mode 100644 index 0000000..132babd --- /dev/null +++ b/test/test_js/test_js_target_type.py @@ -0,0 +1,121 @@ +from rdflib import Graph +from pyshacl import validate +shapes_graph = '''\ +@prefix dash: . +@prefix ex: . +@prefix owl: . +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix xsd: . + + + rdf:type owl:Ontology ; + rdfs:label "Test of sh:JSTargetType 001" ; + owl:imports ; +. +ex:GraphValidationTestCase + rdf:type dash:GraphValidationTestCase ; + dash:expectedResult [ + rdf:type sh:ValidationReport ; + sh:conforms "false"^^xsd:boolean ; + sh:result [ + rdf:type sh:ValidationResult ; + sh:focusNode ex:Barry ; + sh:resultSeverity sh:Violation ; + sh:sourceConstraintComponent sh:ClassConstraintComponent ; + sh:sourceShape ex:USCitizenShape ; + sh:value ex:Barry ; + ] ; + ] ; +. + +ex:Person + rdf:type owl:Class ; + rdfs:label "A person" ; +. + +ex:Country + rdf:type owl:Class ; + rdfs:label "A country" ; +. + +ex:USA + rdf:type ex:Country ; +. + +ex:Germany + rdf:type ex:Country ; +. + +ex:bornIn + rdf:type owl:ObjectProperty ; +. + +ex:GermanCitizen + rdf:type owl:Class ; +. +ex:USCitizen + rdf:type owl:Class ; +. + +ex:PeopleBornInCountryTarget + a sh:JSTargetType ; + rdfs:subClassOf sh:Target ; + sh:labelTemplate "All persons born in {$country}" ; + sh:parameter [ + sh:path ex:country ; + sh:description "The country that the focus nodes are 'born' in." ; + sh:class ex:Country ; + sh:nodeKind sh:IRI ; + ] ; + sh:jsFunctionName "findBornIn" ; + sh:jsLibrary [ sh:jsLibraryURL "resources/js/findBornIn.js"^^xsd:anyURI ] ; +. + +ex:GermanCitizenShape + a sh:NodeShape ; + sh:target [ + a ex:PeopleBornInCountryTarget ; + ex:country ex:Germany ; + ] ; + sh:class ex:GermanCitizen ; +. + +ex:USCitizenShape + a sh:NodeShape ; + sh:target [ + a ex:PeopleBornInCountryTarget ; + ex:country ex:USA ; + ] ; + sh:class ex:USCitizen ; +. +''' + +data_graph = '''\ +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix owl: . +@prefix ex: . +@prefix exdata: . + +exdata:Ludwig + rdf:type ex:Person ; + rdf:type ex:GermanCitizen ; + ex:bornIn ex:Germany . + +exdata:Barry + rdf:type ex:Person ; + ex:bornIn ex:USA . + +''' + +def test_js_target_type(): + s1 = Graph().parse(data=shapes_graph, format="turtle") + g1 = Graph().parse(data=data_graph, format="turtle") + conforms, result_graph, result_text = validate(g1, shacl_graph=s1, advanced=True, debug=True, js=True) + assert not conforms + +if __name__ == "__main__": + test_js_target_type() From fe2c27f51fa012f182cd8a1653104424298a5bed Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Wed, 14 Oct 2020 12:34:39 +1000 Subject: [PATCH 08/10] Finalize the SHACL-JS additions add type hints, fix MyPy run, isort, black, fix Flake8 --- poetry.lock | 178 ++++++++++++------ pyshacl/constraints/constraint_component.py | 14 +- pyshacl/constraints/core/value_constraints.py | 2 +- .../sparql_based_constraint_components.py | 6 +- pyshacl/extras/__init__.py | 14 +- pyshacl/extras/js/__init__.py | 1 + pyshacl/extras/js/constraint.py | 19 +- pyshacl/extras/js/constraint_component.py | 53 +++--- pyshacl/extras/js/context.py | 48 +++-- pyshacl/extras/js/function.py | 12 +- pyshacl/extras/js/js_executable.py | 10 +- pyshacl/extras/js/loader.py | 18 +- pyshacl/extras/js/rules.py | 17 +- pyshacl/extras/js/target.py | 20 +- pyshacl/functions/__init__.py | 32 ++-- pyshacl/functions/shacl_function.py | 1 - pyshacl/helper/__init__.py | 3 + pyshacl/parameter.py | 2 +- pyshacl/rules/__init__.py | 19 +- pyshacl/rules/sparql/__init__.py | 6 +- pyshacl/shape.py | 22 ++- pyshacl/shapes_graph.py | 4 +- pyshacl/target.py | 9 +- pyshacl/validate.py | 10 +- 24 files changed, 328 insertions(+), 192 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2a08ab8..d3f2646 100644 --- a/poetry.lock +++ b/poetry.lock @@ -22,13 +22,13 @@ description = "Classes Without Boilerplate" name = "attrs" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "19.3.0" +version = "20.2.0" [package.extras] -azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] -dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"] -docs = ["sphinx", "zope.interface"] -tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] +dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] +tests_no_zope = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] [[package]] category = "dev" @@ -67,7 +67,7 @@ marker = "sys_platform == \"win32\"" name = "colorama" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "0.4.3" +version = "0.4.4" [[package]] category = "dev" @@ -77,13 +77,21 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, <4" version = "4.5.4" +[[package]] +category = "main" +description = "The Cython compiler for writing C extensions for the Python language." +name = "cython" +optional = true +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "0.29.21" + [[package]] category = "dev" description = "the modular source code checker: pep8 pyflakes and co" name = "flake8" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" -version = "3.8.3" +version = "3.8.4" [package.dependencies] mccabe = ">=0.6.0,<0.7.0" @@ -101,7 +109,7 @@ marker = "python_version < \"3.8\"" name = "importlib-metadata" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -version = "1.7.0" +version = "2.0.0" [package.dependencies] zipp = ">=0.5" @@ -128,10 +136,11 @@ marker = "python_version >= \"3.6\"" name = "isort" optional = false python-versions = ">=3.6,<4.0" -version = "5.0.6" +version = "5.6.4" [package.extras] -pipfile_deprecated_finder = ["pipreqs", "requirementslib", "tomlkit (>=0.5.3)"] +colors = ["colorama (>=0.4.3,<0.5.0)"] +pipfile_deprecated_finder = ["pipreqs", "requirementslib"] requirements_deprecated_finder = ["pipreqs", "pip-api"] [[package]] @@ -148,7 +157,7 @@ description = "More routines for operating on iterables, beyond itertools" name = "more-itertools" optional = false python-versions = ">=3.5" -version = "8.4.0" +version = "8.5.0" [[package]] category = "dev" @@ -241,6 +250,17 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "2.6.0" +[[package]] +category = "main" +description = "Python integration for the Duktape Javascript interpreter" +name = "pyduktape2" +optional = true +python-versions = "*" +version = "0.4.1" + +[package.dependencies] +Cython = "*" + [[package]] category = "dev" description = "passive checker of Python programs" @@ -289,7 +309,7 @@ description = "Pytest plugin for measuring coverage." name = "pytest-cov" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.10.0" +version = "2.10.1" [package.dependencies] coverage = ">=4.4" @@ -335,7 +355,7 @@ marker = "python_version >= \"3.6\"" name = "regex" optional = false python-versions = "*" -version = "2020.6.8" +version = "2020.10.11" [[package]] category = "main" @@ -370,7 +390,7 @@ marker = "python_version >= \"3.6\"" name = "typing-extensions" optional = false python-versions = "*" -version = "3.7.4.2" +version = "3.7.4.3" [[package]] category = "dev" @@ -387,18 +407,19 @@ marker = "python_version < \"3.8\"" name = "zipp" optional = false python-versions = ">=3.6" -version = "3.1.0" +version = "3.3.0" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["jaraco.itertools", "func-timeout"] +testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] [extras] dev-lint = [] dev-type-checking = [] +js = ["pyduktape2"] [metadata] -content-hash = "174f7f8dbaff0aeaa98e1abd9c87a48142f1d94256e219e16fbd4b34e8c3fb61" +content-hash = "aa9587d8256b5b2fe98784e5f9ebec5a83ec213201374cd0f024280ba0ae7ac1" python-versions = "^3.6" # Compatible python versions must be declared here [metadata.files] @@ -411,8 +432,8 @@ atomicwrites = [ {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, - {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, + {file = "attrs-20.2.0-py2.py3-none-any.whl", hash = "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"}, + {file = "attrs-20.2.0.tar.gz", hash = "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594"}, ] black = [ {file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"}, @@ -423,8 +444,7 @@ click = [ {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] colorama = [ - {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, - {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, ] coverage = [ {file = "coverage-4.5.4-cp26-cp26m-macosx_10_12_x86_64.whl", hash = "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28"}, @@ -460,29 +480,64 @@ coverage = [ {file = "coverage-4.5.4-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5"}, {file = "coverage-4.5.4.tar.gz", hash = "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c"}, ] +cython = [ + {file = "Cython-0.29.21-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c541b2b49c6638f2b5beb9316726db84a8d1c132bf31b942dae1f9c7f6ad3b92"}, + {file = "Cython-0.29.21-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b8d8497091c1dc8705d1575c71e908a93b1f127a174b2d472020f3d84263ac28"}, + {file = "Cython-0.29.21-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:695a6bcaf9e12b1e471dfce96bbecf22a1487adc2ac6106b15960a2b51b97f5d"}, + {file = "Cython-0.29.21-cp27-cp27m-win32.whl", hash = "sha256:171b9f70ceafcec5852089d0f9c1e75b0d554f46c882cd4e2e4acaba9bd7d148"}, + {file = "Cython-0.29.21-cp27-cp27m-win_amd64.whl", hash = "sha256:539e59949aab4955c143a468810123bf22d3e8556421e1ce2531ed4893914ca0"}, + {file = "Cython-0.29.21-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:e93acd1f603a0c1786e0841f066ae7cef014cf4750e3cd06fd03cfdf46361419"}, + {file = "Cython-0.29.21-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:2922e3031ba9ebbe7cb9200b585cc33b71d66023d78450dcb883f824f4969371"}, + {file = "Cython-0.29.21-cp34-cp34m-manylinux1_i686.whl", hash = "sha256:497841897942f734b0abc2dead2d4009795ee992267a70a23485fd0e937edc0b"}, + {file = "Cython-0.29.21-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:0ac10bf476476a9f7ef61ec6e44c280ef434473124ad31d3132b720f7b0e8d2a"}, + {file = "Cython-0.29.21-cp34-cp34m-win32.whl", hash = "sha256:31c71a615f38401b0dc1f2a5a9a6c421ffd8908c4cd5bbedc4014c1b876488e8"}, + {file = "Cython-0.29.21-cp34-cp34m-win_amd64.whl", hash = "sha256:c4b78356074fcaac04ecb4de289f11d506e438859877670992ece11f9c90f37b"}, + {file = "Cython-0.29.21-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:b2f9172e4d6358f33ecce6a4339b5960f9f83eab67ea244baa812737793826b7"}, + {file = "Cython-0.29.21-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:856c7fb31d247ce713d60116375e1f8153d0291ab5e92cca7d8833a524ba9991"}, + {file = "Cython-0.29.21-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:715294cd2246b39a8edca464a8366eb635f17213e4a6b9e74e52d8b877a8cb63"}, + {file = "Cython-0.29.21-cp35-cp35m-win32.whl", hash = "sha256:23f3a00b843a19de8bb4468b087db5b413a903213f67188729782488d67040e0"}, + {file = "Cython-0.29.21-cp35-cp35m-win_amd64.whl", hash = "sha256:ccb77faeaad99e99c6c444d04862c6cf604204fe0a07d4c8f9cbf2c9012d7d5a"}, + {file = "Cython-0.29.21-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e272ed97d20b026f4f25a012b25d7d7672a60e4f72b9ca385239d693cd91b2d5"}, + {file = "Cython-0.29.21-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:8c6e25e9cc4961bb2abb1777c6fa9d0fa2d9b014beb3276cebe69996ff162b78"}, + {file = "Cython-0.29.21-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:57ead89128dee9609119c93d3926c7a2add451453063147900408a50144598c6"}, + {file = "Cython-0.29.21-cp36-cp36m-win32.whl", hash = "sha256:0e25c209c75df8785480dcef85db3d36c165dbc0f4c503168e8763eb735704f2"}, + {file = "Cython-0.29.21-cp36-cp36m-win_amd64.whl", hash = "sha256:a0674f246ad5e1571ef29d4c5ec1d6ecabe9e6c424ad0d6fee46b914d5d24d69"}, + {file = "Cython-0.29.21-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5da187bebe38030325e1c0b5b8a804d489410be2d384c0ef3ba39493c67eb51e"}, + {file = "Cython-0.29.21-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:9ce5e5209f8406ffc2b058b1293cce7a954911bb7991e623564d489197c9ba30"}, + {file = "Cython-0.29.21-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5e545a48f919e40079b0efe7b0e081c74b96f9ef25b9c1ff4cdbd95764426b58"}, + {file = "Cython-0.29.21-cp37-cp37m-win32.whl", hash = "sha256:c8435959321cf8aec867bbad54b83b7fb8343204b530d85d9ea7a1f5329d5ac2"}, + {file = "Cython-0.29.21-cp37-cp37m-win_amd64.whl", hash = "sha256:540b3bee0711aac2e99bda4fa0a46dbcd8c74941666bfc1ef9236b1a64eeffd9"}, + {file = "Cython-0.29.21-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:93f5fed1c9445fb7afe20450cdaf94b0e0356d47cc75008105be89c6a2e417b1"}, + {file = "Cython-0.29.21-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9207fdedc7e789a3dcaca628176b80c82fbed9ae0997210738cbb12536a56699"}, + {file = "Cython-0.29.21-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:603b9f1b8e93e8b494d3e89320c410679e21018e48b6cbc77280f5db71f17dc0"}, + {file = "Cython-0.29.21-cp38-cp38-win32.whl", hash = "sha256:473df5d5e400444a36ed81c6596f56a5b52a3481312d0a48d68b777790f730ae"}, + {file = "Cython-0.29.21-cp38-cp38-win_amd64.whl", hash = "sha256:b8a8a31b9e8860634adbca30fea1d0c7f08e208b3d7611f3e580e5f20992e5d7"}, + {file = "Cython-0.29.21-py2.py3-none-any.whl", hash = "sha256:5c4276fdcbccdf1e3c1756c7aeb8395e9a36874fa4d30860e7694f43d325ae13"}, + {file = "Cython-0.29.21.tar.gz", hash = "sha256:e57acb89bd55943c8d8bf813763d20b9099cc7165c0f16b707631a7654be9cad"}, +] flake8 = [ - {file = "flake8-3.8.3-py2.py3-none-any.whl", hash = "sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c"}, - {file = "flake8-3.8.3.tar.gz", hash = "sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208"}, + {file = "flake8-3.8.4-py2.py3-none-any.whl", hash = "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839"}, + {file = "flake8-3.8.4.tar.gz", hash = "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"}, ] importlib-metadata = [ - {file = "importlib_metadata-1.7.0-py2.py3-none-any.whl", hash = "sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070"}, - {file = "importlib_metadata-1.7.0.tar.gz", hash = "sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83"}, + {file = "importlib_metadata-2.0.0-py2.py3-none-any.whl", hash = "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3"}, + {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"}, ] isodate = [ {file = "isodate-0.6.0-py2.py3-none-any.whl", hash = "sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81"}, {file = "isodate-0.6.0.tar.gz", hash = "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8"}, ] isort = [ - {file = "isort-5.0.6-py3-none-any.whl", hash = "sha256:76a97d9acdb3a376c48fafddbe9177192d4ac35b0b2c21a0c965b9604a412af1"}, - {file = "isort-5.0.6.tar.gz", hash = "sha256:af3ac4524d1256e3f32d155da8fb43b2a94201644fa6b45813c8783fc51c3841"}, + {file = "isort-5.6.4-py3-none-any.whl", hash = "sha256:dcab1d98b469a12a1a624ead220584391648790275560e1a43e54c5dceae65e7"}, + {file = "isort-5.6.4.tar.gz", hash = "sha256:dcaeec1b5f0eca77faea2a35ab790b4f3680ff75590bfcb7145986905aab2f58"}, ] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] more-itertools = [ - {file = "more-itertools-8.4.0.tar.gz", hash = "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5"}, - {file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"}, + {file = "more-itertools-8.5.0.tar.gz", hash = "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20"}, + {file = "more_itertools-8.5.0-py3-none-any.whl", hash = "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c"}, ] mypy = [ {file = "mypy-0.770-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:a34b577cdf6313bf24755f7a0e3f3c326d5c1f4fe7422d1d06498eb25ad0c600"}, @@ -528,6 +583,9 @@ pycodestyle = [ {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, ] +pyduktape2 = [ + {file = "pyduktape2-0.4.1.tar.gz", hash = "sha256:7f07bdf2f1f198f4f5f240f6053898610125b3d654ad3070014c1f7348657614"}, +] pyflakes = [ {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"}, {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, @@ -541,8 +599,8 @@ pytest = [ {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"}, ] pytest-cov = [ - {file = "pytest-cov-2.10.0.tar.gz", hash = "sha256:1a629dc9f48e53512fcbfda6b07de490c374b0c83c55ff7a1720b3fccff0ac87"}, - {file = "pytest_cov-2.10.0-py2.py3-none-any.whl", hash = "sha256:6e6d18092dce6fad667cd7020deed816f858ad3b49d5b5e2b1cc1c97a4dba65c"}, + {file = "pytest-cov-2.10.1.tar.gz", hash = "sha256:47bd0ce14056fdd79f93e1713f88fad7bdcc583dcd7783da86ef2f085a0bb88e"}, + {file = "pytest_cov-2.10.1-py2.py3-none-any.whl", hash = "sha256:45ec2d5182f89a81fc3eb29e3d1ed3113b9e9a873bcddb2a71faaab066110191"}, ] rdflib = [ {file = "rdflib-5.0.0-py3-none-any.whl", hash = "sha256:88208ea971a87886d60ae2b1a4b2cdc263527af0454c422118d43fe64b357877"}, @@ -552,27 +610,33 @@ rdflib-jsonld = [ {file = "rdflib-jsonld-0.5.0.tar.gz", hash = "sha256:4f7d55326405071c7bce9acf5484643bcb984eadb84a6503053367da207105ed"}, ] regex = [ - {file = "regex-2020.6.8-cp27-cp27m-win32.whl", hash = "sha256:fbff901c54c22425a5b809b914a3bfaf4b9570eee0e5ce8186ac71eb2025191c"}, - {file = "regex-2020.6.8-cp27-cp27m-win_amd64.whl", hash = "sha256:112e34adf95e45158c597feea65d06a8124898bdeac975c9087fe71b572bd938"}, - {file = "regex-2020.6.8-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:92d8a043a4241a710c1cf7593f5577fbb832cf6c3a00ff3fc1ff2052aff5dd89"}, - {file = "regex-2020.6.8-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:bae83f2a56ab30d5353b47f9b2a33e4aac4de9401fb582b55c42b132a8ac3868"}, - {file = "regex-2020.6.8-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:b2ba0f78b3ef375114856cbdaa30559914d081c416b431f2437f83ce4f8b7f2f"}, - {file = "regex-2020.6.8-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:95fa7726d073c87141f7bbfb04c284901f8328e2d430eeb71b8ffdd5742a5ded"}, - {file = "regex-2020.6.8-cp36-cp36m-win32.whl", hash = "sha256:e3cdc9423808f7e1bb9c2e0bdb1c9dc37b0607b30d646ff6faf0d4e41ee8fee3"}, - {file = "regex-2020.6.8-cp36-cp36m-win_amd64.whl", hash = "sha256:c78e66a922de1c95a208e4ec02e2e5cf0bb83a36ceececc10a72841e53fbf2bd"}, - {file = "regex-2020.6.8-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:08997a37b221a3e27d68ffb601e45abfb0093d39ee770e4257bd2f5115e8cb0a"}, - {file = "regex-2020.6.8-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:2f6f211633ee8d3f7706953e9d3edc7ce63a1d6aad0be5dcee1ece127eea13ae"}, - {file = "regex-2020.6.8-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:55b4c25cbb3b29f8d5e63aeed27b49fa0f8476b0d4e1b3171d85db891938cc3a"}, - {file = "regex-2020.6.8-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:89cda1a5d3e33ec9e231ece7307afc101b5217523d55ef4dc7fb2abd6de71ba3"}, - {file = "regex-2020.6.8-cp37-cp37m-win32.whl", hash = "sha256:690f858d9a94d903cf5cada62ce069b5d93b313d7d05456dbcd99420856562d9"}, - {file = "regex-2020.6.8-cp37-cp37m-win_amd64.whl", hash = "sha256:1700419d8a18c26ff396b3b06ace315b5f2a6e780dad387e4c48717a12a22c29"}, - {file = "regex-2020.6.8-cp38-cp38-manylinux1_i686.whl", hash = "sha256:654cb773b2792e50151f0e22be0f2b6e1c3a04c5328ff1d9d59c0398d37ef610"}, - {file = "regex-2020.6.8-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:52e1b4bef02f4040b2fd547357a170fc1146e60ab310cdbdd098db86e929b387"}, - {file = "regex-2020.6.8-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:cf59bbf282b627130f5ba68b7fa3abdb96372b24b66bdf72a4920e8153fc7910"}, - {file = "regex-2020.6.8-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:5aaa5928b039ae440d775acea11d01e42ff26e1561c0ffcd3d805750973c6baf"}, - {file = "regex-2020.6.8-cp38-cp38-win32.whl", hash = "sha256:97712e0d0af05febd8ab63d2ef0ab2d0cd9deddf4476f7aa153f76feef4b2754"}, - {file = "regex-2020.6.8-cp38-cp38-win_amd64.whl", hash = "sha256:6ad8663c17db4c5ef438141f99e291c4d4edfeaacc0ce28b5bba2b0bf273d9b5"}, - {file = "regex-2020.6.8.tar.gz", hash = "sha256:e9b64e609d37438f7d6e68c2546d2cb8062f3adb27e6336bc129b51be20773ac"}, + {file = "regex-2020.10.11-cp27-cp27m-win32.whl", hash = "sha256:4f5c0fe46fb79a7adf766b365cae56cafbf352c27358fda811e4a1dc8216d0db"}, + {file = "regex-2020.10.11-cp27-cp27m-win_amd64.whl", hash = "sha256:39a5ef30bca911f5a8a3d4476f5713ed4d66e313d9fb6755b32bec8a2e519635"}, + {file = "regex-2020.10.11-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:7c4fc5a8ec91a2254bb459db27dbd9e16bba1dabff638f425d736888d34aaefa"}, + {file = "regex-2020.10.11-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d537e270b3e6bfaea4f49eaf267984bfb3628c86670e9ad2a257358d3b8f0955"}, + {file = "regex-2020.10.11-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:a8240df4957a5b0e641998a5d78b3c4ea762c845d8cb8997bf820626826fde9a"}, + {file = "regex-2020.10.11-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:4302153abb96859beb2c778cc4662607a34175065fc2f33a21f49eb3fbd1ccd3"}, + {file = "regex-2020.10.11-cp36-cp36m-win32.whl", hash = "sha256:c077c9d04a040dba001cf62b3aff08fd85be86bccf2c51a770c77377662a2d55"}, + {file = "regex-2020.10.11-cp36-cp36m-win_amd64.whl", hash = "sha256:46ab6070b0d2cb85700b8863b3f5504c7f75d8af44289e9562195fe02a8dd72d"}, + {file = "regex-2020.10.11-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:d629d750ebe75a88184db98f759633b0a7772c2e6f4da529f0027b4a402c0e2f"}, + {file = "regex-2020.10.11-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8e7ef296b84d44425760fe813cabd7afbb48c8dd62023018b338bbd9d7d6f2f0"}, + {file = "regex-2020.10.11-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:e490f08897cb44e54bddf5c6e27deca9b58c4076849f32aaa7a0b9f1730f2c20"}, + {file = "regex-2020.10.11-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:850339226aa4fec04916386577674bb9d69abe0048f5d1a99f91b0004bfdcc01"}, + {file = "regex-2020.10.11-cp37-cp37m-win32.whl", hash = "sha256:60c4f64d9a326fe48e8738c3dbc068e1edc41ff7895a9e3723840deec4bc1c28"}, + {file = "regex-2020.10.11-cp37-cp37m-win_amd64.whl", hash = "sha256:8ba3efdd60bfee1aa784dbcea175eb442d059b576934c9d099e381e5a9f48930"}, + {file = "regex-2020.10.11-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2308491b3e6c530a3bb38a8a4bb1dc5fd32cbf1e11ca623f2172ba17a81acef1"}, + {file = "regex-2020.10.11-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:b8806649983a1c78874ec7e04393ef076805740f6319e87a56f91f1767960212"}, + {file = "regex-2020.10.11-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:a2a31ee8a354fa3036d12804730e1e20d58bc4e250365ead34b9c30bbe9908c3"}, + {file = "regex-2020.10.11-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d9d53518eeed12190744d366ec4a3f39b99d7daa705abca95f87dd8b442df4ad"}, + {file = "regex-2020.10.11-cp38-cp38-win32.whl", hash = "sha256:3d5a8d007116021cf65355ada47bf405656c4b3b9a988493d26688275fde1f1c"}, + {file = "regex-2020.10.11-cp38-cp38-win_amd64.whl", hash = "sha256:f579caecbbca291b0fcc7d473664c8c08635da2f9b1567c22ea32311c86ef68c"}, + {file = "regex-2020.10.11-cp39-cp39-manylinux1_i686.whl", hash = "sha256:8c8c42aa5d3ac9a49829c4b28a81bebfa0378996f9e0ca5b5ab8a36870c3e5ee"}, + {file = "regex-2020.10.11-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:c529ba90c1775697a65b46c83d47a2d3de70f24d96da5d41d05a761c73b063af"}, + {file = "regex-2020.10.11-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:6cf527ec2f3565248408b61dd36e380d799c2a1047eab04e13a2b0c15dd9c767"}, + {file = "regex-2020.10.11-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:671c51d352cfb146e48baee82b1ee8d6ffe357c292f5e13300cdc5c00867ebfc"}, + {file = "regex-2020.10.11-cp39-cp39-win32.whl", hash = "sha256:a63907332531a499b8cdfd18953febb5a4c525e9e7ca4ac147423b917244b260"}, + {file = "regex-2020.10.11-cp39-cp39-win_amd64.whl", hash = "sha256:1a16afbfadaadc1397353f9b32e19a65dc1d1804c80ad73a14f435348ca017ad"}, + {file = "regex-2020.10.11.tar.gz", hash = "sha256:463e770c48da76a8da82b8d4a48a541f314e0df91cbb6d873a341dbe578efafd"}, ] six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, @@ -606,15 +670,15 @@ typed-ast = [ {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, ] typing-extensions = [ - {file = "typing_extensions-3.7.4.2-py2-none-any.whl", hash = "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"}, - {file = "typing_extensions-3.7.4.2-py3-none-any.whl", hash = "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5"}, - {file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"}, + {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, + {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, + {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, ] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] zipp = [ - {file = "zipp-3.1.0-py3-none-any.whl", hash = "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b"}, - {file = "zipp-3.1.0.tar.gz", hash = "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"}, + {file = "zipp-3.3.0-py3-none-any.whl", hash = "sha256:eed8ec0b8d1416b2ca33516a37a08892442f3954dee131e92cfd92d8fe3e7066"}, + {file = "zipp-3.3.0.tar.gz", hash = "sha256:64ad89efee774d1897a58607895d80789c59778ea02185dd846ac38394a8642b"}, ] diff --git a/pyshacl/constraints/constraint_component.py b/pyshacl/constraints/constraint_component.py index ac5fadd..6a3488e 100644 --- a/pyshacl/constraints/constraint_component.py +++ b/pyshacl/constraints/constraint_component.py @@ -5,24 +5,31 @@ """ import abc import typing -from typing import TYPE_CHECKING, Dict, Iterable, Tuple, List, Set, Optional, Any + +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Set, Tuple import rdflib from rdflib import BNode, Literal, URIRef from pyshacl.consts import ( + SH, RDF_type, + SH_ask, SH_focusNode, + SH_jsFunctionName, + SH_parameter, + SH_path, SH_resultMessage, SH_resultPath, SH_resultSeverity, + SH_select, SH_sourceConstraint, SH_sourceConstraintComponent, SH_sourceShape, SH_ValidationResult, SH_value, - SH_Violation, SH_parameter, SH_path, SH, SH_ask, SH_select, SH_jsFunctionName, + SH_Violation, ) from pyshacl.errors import ConstraintLoadError from pyshacl.parameter import SHACLParameter @@ -249,6 +256,7 @@ def make_v_result( SH_SPARQLAskValidator = SH.term('SPARQLAskValidator') SH_JSValidator = SH.term('JSValidator') + class CustomConstraintComponentFactory(object): __slots__: Tuple = tuple() @@ -331,9 +339,11 @@ def __new__(cls, shacl_graph: 'ShapesGraph', node): ) if is_sparql_constraint_component: from pyshacl.constraints.sparql.sparql_based_constraint_components import SPARQLConstraintComponent + return SPARQLConstraintComponent(*self) elif is_js_constraint_component and shacl_graph.js_enabled: from pyshacl.extras.js.constraint_component import JSConstraintComponent + return JSConstraintComponent(*self) else: return CustomConstraintComponent(*self) diff --git a/pyshacl/constraints/core/value_constraints.py b/pyshacl/constraints/core/value_constraints.py index f883d4c..07b34a5 100644 --- a/pyshacl/constraints/core/value_constraints.py +++ b/pyshacl/constraints/core/value_constraints.py @@ -19,10 +19,10 @@ SH_BlankNode, SH_BlankNodeOrIRI, SH_BlankNodeORLiteral, + SH_datatype, SH_IRIOrLiteral, SH_Literal, SH_nodeKind, - SH_datatype ) from pyshacl.errors import ConstraintLoadError from pyshacl.pytypes import GraphLike diff --git a/pyshacl/constraints/sparql/sparql_based_constraint_components.py b/pyshacl/constraints/sparql/sparql_based_constraint_components.py index 0b9d98d..7cb2722 100644 --- a/pyshacl/constraints/sparql/sparql_based_constraint_components.py +++ b/pyshacl/constraints/sparql/sparql_based_constraint_components.py @@ -4,12 +4,12 @@ """ import typing -from typing import Any, Dict, List, Set, Tuple, Type, Union +from typing import Dict, List, Tuple, Type, Union import rdflib from pyshacl.constraints.constraint_component import ConstraintComponent, CustomConstraintComponent -from pyshacl.consts import SH, RDF_type, SH_ask, SH_message, SH_select, SH_ConstraintComponent +from pyshacl.consts import SH, RDF_type, SH_ask, SH_ConstraintComponent, SH_message, SH_select from pyshacl.errors import ConstraintLoadError, ValidationFailure from pyshacl.helper import get_query_helper_cls from pyshacl.pytypes import GraphLike @@ -363,6 +363,7 @@ def validate(self, focus, value_nodes, target_graph, query_helper=None, new_bind pass return violations + class SPARQLConstraintComponent(CustomConstraintComponent): """ SPARQL-based constraints provide a lot of flexibility but may be hard to understand for some people or lead to repetition. This section introduces SPARQL-based constraint components as a way to abstract the complexity of SPARQL and to declare high-level reusable components similar to the Core constraint components. Such constraint components can be declared using the SHACL RDF vocabulary and thus shared and reused. @@ -413,4 +414,3 @@ def make_validator_for_shape(self, shape: 'Shape'): self, shape, must_be_ask_val=must_be_ask_val, must_be_select_val=must_be_select_val ) return applied_validator - diff --git a/pyshacl/extras/__init__.py b/pyshacl/extras/__init__.py index 992bb29..7585e9f 100644 --- a/pyshacl/extras/__init__.py +++ b/pyshacl/extras/__init__.py @@ -1,25 +1,29 @@ # -*- coding: utf-8 -*- # -import typing from functools import lru_cache from warnings import warn + import pkg_resources -from pkg_resources import UnknownExtra, DistributionNotFound + +from pkg_resources import DistributionNotFound, UnknownExtra + dev_mode = True + + @lru_cache() def check_extra_installed(extra_name: str): if dev_mode: return True - check_name = "pyshacl["+extra_name+"]" + check_name = "pyshacl[" + extra_name + "]" # first check if pyshacl is installed using the normal means try: - res1 = pkg_resources.require("pyshacl") + _ = pkg_resources.require("pyshacl") except DistributionNotFound: # Hmm, it thinks pyshacl isn't installed. Can't even check for extras return None try: - res2 = pkg_resources.require(check_name) + _ = pkg_resources.require(check_name) return True except UnknownExtra: # That extra doesn't exist in this version of pyshacl diff --git a/pyshacl/extras/js/__init__.py b/pyshacl/extras/js/__init__.py index 64d9000..955f5a1 100644 --- a/pyshacl/extras/js/__init__.py +++ b/pyshacl/extras/js/__init__.py @@ -4,3 +4,4 @@ from .loader import load_into_context +__all__ = ['load_into_context'] diff --git a/pyshacl/extras/js/constraint.py b/pyshacl/extras/js/constraint.py index 7f9ab56..a4d3510 100644 --- a/pyshacl/extras/js/constraint.py +++ b/pyshacl/extras/js/constraint.py @@ -1,14 +1,19 @@ # # import typing -from typing import List, Dict + +from typing import Dict, List + from rdflib import Literal + from pyshacl.constraints import ConstraintComponent from pyshacl.consts import SH, SH_js, SH_message from pyshacl.errors import ConstraintLoadError from pyshacl.pytypes import GraphLike + from .js_executable import JSExecutable + if typing.TYPE_CHECKING: from pyshacl.shape import Shape from pyshacl.shapes_graph import ShapesGraph @@ -28,8 +33,7 @@ def __init__(self, shapes_graph: 'ShapesGraph', node): for m in msgs_iter: if not isinstance(m, Literal): raise ConstraintLoadError( - "JSConstraint sh:message must be a RDF Literal.", - "https://www.w3.org/TR/shacl-js/#js-constraints", + "JSConstraint sh:message must be a RDF Literal.", "https://www.w3.org/TR/shacl-js/#js-constraints", ) if not isinstance(m.value, str): raise ConstraintLoadError( @@ -139,11 +143,14 @@ def _evaluate_js_exe(self, data_graph, f_v_dict, js_impl: JSConstraintImpl): message = res.get('message', None) if message is not None: msgs.append(Literal(message)) - reports.append(self.make_v_result(data_graph, f, value_node=val, result_path=path, extra_messages=msgs)) + reports.append( + self.make_v_result( + data_graph, f, value_node=val, result_path=path, extra_messages=msgs + ) + ) except Exception as e: + print(e) raise if failed: non_conformant = True return non_conformant, reports - - diff --git a/pyshacl/extras/js/constraint_component.py b/pyshacl/extras/js/constraint_component.py index d1d0375..799224a 100644 --- a/pyshacl/extras/js/constraint_component.py +++ b/pyshacl/extras/js/constraint_component.py @@ -1,25 +1,32 @@ # # import typing -from typing import List, Dict, Tuple + +from typing import Any, Dict, List, Tuple, Union + from rdflib import Literal + from pyshacl.constraints import ConstraintComponent from pyshacl.constraints.constraint_component import CustomConstraintComponent -from pyshacl.consts import SH, SH_message, SH_js, SH_jsLibrary, SH_jsFunctionName, SH_ConstraintComponent -from pyshacl.errors import ConstraintLoadError, ValidationFailure, ReportableRuntimeError +from pyshacl.consts import SH, SH_ConstraintComponent, SH_message +from pyshacl.errors import ConstraintLoadError, ReportableRuntimeError, ValidationFailure from pyshacl.pytypes import GraphLike + from .js_executable import JSExecutable + if typing.TYPE_CHECKING: - from pyshacl.shapes_graph import ShapesGraph from pyshacl.shape import Shape + from pyshacl.shapes_graph import ShapesGraph SH_JSConstraint = SH.term('JSConstraint') SH_JSConstraintComponent = SH.term('JSConstraintComponent') + class BoundShapeJSValidatorComponent(ConstraintComponent): invalid_parameter_names = {'this', 'shapesGraph', 'currentShape', 'path', 'PATH', 'value'} + def __init__(self, constraint, shape: 'Shape', validator): """ Create a new custom constraint, by applying a ConstraintComponent and a Validator to a Shape @@ -33,8 +40,8 @@ def __init__(self, constraint, shape: 'Shape', validator): super(BoundShapeJSValidatorComponent, self).__init__(shape) self.constraint = constraint self.validator = validator - self.param_bind_map = {} - self.messages = [] + self.param_bind_map: Dict[str, Any] = {} + self.messages: List[Any] = [] self.bind_params() def bind_params(self): @@ -49,8 +56,7 @@ def bind_params(self): if len(shape_params) < 1: if not p.optional: # TODO:coverage: No test for this case - raise ReportableRuntimeError( - "Shape does not have mandatory parameter {}.".format(str(p.path()))) + raise ReportableRuntimeError("Shape does not have mandatory parameter {}.".format(str(p.path()))) continue # TODO: Can shapes have more than one value for the predicate? # Just use one for now. @@ -58,7 +64,6 @@ def bind_params(self): bind_map[name] = next(iter(shape_params)) self.param_bind_map = bind_map - @classmethod def constraint_parameters(cls): # TODO:coverage: this is never used for this constraint? @@ -142,11 +147,13 @@ def evaluate(self, data_graph: GraphLike, focus_value_nodes: Dict, _evaluation_p new_kwargs = rept_kwargs.copy() new_kwargs['extra_messages'].extend(msgs) reports.append( - self.make_v_result(data_graph, f, value_node=val, result_path=path, **new_kwargs)) + self.make_v_result(data_graph, f, value_node=val, result_path=path, **new_kwargs) + ) if failed: non_conformant = True return (not non_conformant), reports + class JSConstraintComponent(CustomConstraintComponent): """ SPARQL-based constraints provide a lot of flexibility but may be hard to understand for some people or lead to repetition. This section introduces SPARQL-based constraint components as a way to abstract the complexity of SPARQL and to declare high-level reusable components similar to the Core constraint components. Such constraint components can be declared using the SHACL RDF vocabulary and thus shared and reused. @@ -184,7 +191,9 @@ def make_validator_for_shape(self, shape: 'Shape'): "https://www.w3.org/TR/shacl/#constraint-components-validators", ) if is_property_val: - validator = JSConstraintComponentPathValidator(self.sg, validator_node) + validator: Union[ + JSConstraintComponentValidator, JSConstraintComponentPathValidator + ] = JSConstraintComponentPathValidator(self.sg, validator_node) else: validator = JSConstraintComponentValidator(self.sg, validator_node) applied_validator = validator.apply_to_shape_via_constraint(self, shape) @@ -253,19 +262,16 @@ def validate(self, f, value_nodes, path, data_graph, param_bind_vals, new_bind_v results = [] for v in value_nodes: args_map = bind_vals.copy() - args_map.update({ - 'this': f, - 'value': v - }) + args_map.update({'this': f, 'value': v}) try: result_dict = self.execute(data_graph, args_map) results.append((v, result_dict['_result'])) except Exception as e: + print(e) raise return results - def apply_to_shape_via_constraint(self, constraint, shape, **kwargs)\ - -> BoundShapeJSValidatorComponent: + def apply_to_shape_via_constraint(self, constraint, shape, **kwargs) -> BoundShapeJSValidatorComponent: """ Create a new Custom Constraint (BoundShapeValidatorComponent) :param constraint: @@ -280,15 +286,15 @@ def apply_to_shape_via_constraint(self, constraint, shape, **kwargs)\ class JSConstraintComponentPathValidator(JSConstraintComponentValidator): - validator_cache: Dict[Tuple[int, str], 'JSConstraintComponentPathValidator'] = {} + path_validator_cache: Dict[Tuple[int, str], 'JSConstraintComponentPathValidator'] = {} def __new__(cls, shacl_graph: 'ShapesGraph', node, *args, **kwargs): cache_key = (id(shacl_graph.graph), str(node)) - found_in_cache = cls.validator_cache.get(cache_key, False) + found_in_cache = cls.path_validator_cache.get(cache_key, False) if found_in_cache: return found_in_cache self = super(JSConstraintComponentPathValidator, cls).__new__(cls, shacl_graph, node) - cls.validator_cache[cache_key] = self + cls.path_validator_cache[cache_key] = self return self def validate(self, f, value_nodes, path, data_graph, param_bind_vals, new_bind_vals=None): @@ -305,15 +311,12 @@ def validate(self, f, value_nodes, path, data_graph, param_bind_vals, new_bind_v new_bind_vals = new_bind_vals or {} args_map = param_bind_vals.copy() args_map.update(new_bind_vals) - args_map.update({ - 'this': f, - 'path': path - }) + args_map.update({'this': f, 'path': path}) results = [] try: result_dict = self.execute(data_graph, args_map) results.append((f, result_dict['_result'])) except Exception as e: + print(e) raise return results - diff --git a/pyshacl/extras/js/context.py b/pyshacl/extras/js/context.py index b68075a..da90d21 100644 --- a/pyshacl/extras/js/context.py +++ b/pyshacl/extras/js/context.py @@ -1,12 +1,17 @@ import pprint -from rdflib import URIRef, BNode, Literal -from rdflib.namespace import XSD + from decimal import Decimal +from typing import Union + import pyduktape2 + from pyduktape2 import JSProxy +from rdflib import BNode, Literal, URIRef +from rdflib.namespace import XSD + +from pyshacl.errors import ReportableRuntimeError from . import load_into_context -from ...errors import ReportableRuntimeError class URIRefNativeWrapper(object): @@ -86,6 +91,7 @@ class GraphNativeWrapper(object): def __init__(self, g): self.inner = g + class IteratorNativeWrapper(object): def __init__(self, it): self.it = it @@ -93,14 +99,17 @@ def __init__(self, it): def it_next(self): return next(self.it) + def _make_uriref(args): uri = getattr(args, '0') return URIRefNativeWrapper(uri) + def _make_bnode(args): id_ = getattr(args, '0') return BNodeNativeWrapper(id_) + def _make_literal(args): lexical, dtype, lang = getattr(args, '0'), getattr(args, '1'), getattr(args, '2') if isinstance(dtype, JSProxy): @@ -109,6 +118,7 @@ def _make_literal(args): dtype = as_native return LiteralNativeWrapper(lexical, dtype, lang) + def _native_node_equals(args): this, other = getattr(args, '0'), getattr(args, '1') if isinstance(this, (URIRefNativeWrapper, BNodeNativeWrapper, LiteralNativeWrapper)): @@ -117,8 +127,9 @@ def _native_node_equals(args): other = other.inner return this == other + def _native_graph_find(args): - #args are: g, s, p, o + # args are: g, s, p, o triple = [getattr(args, '0'), getattr(args, '1'), getattr(args, '2'), getattr(args, '3')] wrapped_triples = [] for t in triple: @@ -130,6 +141,7 @@ def _native_graph_find(args): it = iter(g.triples((s, p, o))) return IteratorNativeWrapper(it) + def _native_iterator_next(args): arg0 = getattr(args, "0") if isinstance(arg0, IteratorNativeWrapper): @@ -150,6 +162,7 @@ def _native_iterator_next(args): raise RuntimeError("Bad item returned from iterator!") return wrapped_list + def _pprint(args): arg0 = getattr(args, '0') pprint.pprint(arg0) @@ -333,15 +346,18 @@ def _pprint(args): ''' - class SHACLJSContext(object): __slots__ = ("context", "fns") def __init__(self, data_graph, *args, shapes_graph=None, **kwargs): context = pyduktape2.DuktapeContext() context.set_globals( - _pprint=_pprint, _make_uriref=_make_uriref, _make_bnode=_make_bnode, _make_literal=_make_literal, - _native_node_equals=_native_node_equals, _native_iterator_next=_native_iterator_next, + _pprint=_pprint, + _make_uriref=_make_uriref, + _make_bnode=_make_bnode, + _make_literal=_make_literal, + _native_node_equals=_native_node_equals, + _native_iterator_next=_native_iterator_next, _native_graph_find=_native_graph_find, ) context.eval_js(printJs) @@ -474,7 +490,9 @@ def build_results_as_construct(cls, res): this_p = subkeys[1] this_o = subkeys[2] else: - raise ReportableRuntimeError("JS Function returned incorrect number of items in the array.") + raise ReportableRuntimeError( + "JS Function returned incorrect number of items in the array." + ) else: this_s = getattr(k, 'subject', None) this_p = getattr(k, 'predicate', None) @@ -570,7 +588,7 @@ def get_fn_args(self, fn_name, args_map): known_fn_args = self.fns[fn_name] # type: tuple needed_args = [] for k, v in args_map.items(): - look_for = "$"+str(k) + look_for = "$" + str(k) positions = [] start = 0 while True: @@ -579,7 +597,7 @@ def get_fn_args(self, fn_name, args_map): positions.append(pos) except ValueError: break - start = pos+1 + start = pos + 1 if not positions: continue for p in positions: @@ -589,7 +607,7 @@ def get_fn_args(self, fn_name, args_map): a = a[1:] if a not in args_map: needed_args.append((i, a, None)) - needed_args = [v for p,k,v in sorted(needed_args)] + needed_args = [v for p, k, v in sorted(needed_args)] return needed_args def run_js_function(self, fn_name, args, returns: list = None): @@ -600,9 +618,11 @@ def run_js_function(self, fn_name, args, returns: list = None): bind_dict = {} preamble = "" for i, a in enumerate(args): - arg_name = "fn_arg_"+str(i+1) + arg_name = "fn_arg_" + str(i + 1) if isinstance(a, URIRef): - wrapped_a = URIRefNativeWrapper(a) + wrapped_a: Union[URIRefNativeWrapper, BNodeNativeWrapper, LiteralNativeWrapper] = URIRefNativeWrapper( + a + ) native_name = "_{}_native".format(arg_name) preamble += "var {} = NamedNode.from_native({});\n".format(arg_name, native_name) bind_dict[native_name] = wrapped_a @@ -621,7 +641,7 @@ def run_js_function(self, fn_name, args, returns: list = None): else: bind_dict[arg_name] = a - args_string = args_string+arg_name+"," + args_string = args_string + arg_name + "," c.set_globals(**bind_dict) args_string = args_string.rstrip(',') c.eval_js(preamble) diff --git a/pyshacl/extras/js/function.py b/pyshacl/extras/js/function.py index 48db7c3..6ba6152 100644 --- a/pyshacl/extras/js/function.py +++ b/pyshacl/extras/js/function.py @@ -1,20 +1,23 @@ # # import typing + from rdflib.plugins.sparql.operators import register_custom_function, unregister_custom_function from rdflib.plugins.sparql.sparql import SPARQLError from pyshacl.consts import SH -from pyshacl.functions import SHACLFunction from pyshacl.errors import ReportableRuntimeError +from pyshacl.functions import SHACLFunction + from .js_executable import JSExecutable + if typing.TYPE_CHECKING: - from pyshacl.pytypes import GraphLike from pyshacl.shapes_graph import ShapesGraph SH_JSFunction = SH.term('JSFunction') + class JSFunction(SHACLFunction): __slots__ = ('js_exe',) @@ -64,8 +67,3 @@ def apply(self, g): def unapply(self, g): super(JSFunction, self).unapply(g) unregister_custom_function(self.node, self.execute_from_sparql) - - - - - diff --git a/pyshacl/extras/js/js_executable.py b/pyshacl/extras/js/js_executable.py index 477a956..321321c 100644 --- a/pyshacl/extras/js/js_executable.py +++ b/pyshacl/extras/js/js_executable.py @@ -1,11 +1,17 @@ # # import typing + +from typing import Dict + from rdflib import Literal -from pyshacl.consts import SH, SH_jsLibrary, SH_jsFunctionName + +from pyshacl.consts import SH, SH_jsFunctionName, SH_jsLibrary from pyshacl.errors import ConstraintLoadError + from .context import SHACLJSContext + if typing.TYPE_CHECKING: from pyshacl.shapes_graph import ShapesGraph @@ -43,7 +49,7 @@ def __init__(self, shapes_graph: 'ShapesGraph', node): self.fn_name = fn_name library_defs = shapes_graph.objects(node, SH_jsLibrary) seen_library_defs = [] - libraries = {} + libraries: Dict = {} for libn in library_defs: # Library defs can only do two levels deep for now. # TODO: Make this recursive somehow to some further depth diff --git a/pyshacl/extras/js/loader.py b/pyshacl/extras/js/loader.py index e69fa08..8fe185c 100644 --- a/pyshacl/extras/js/loader.py +++ b/pyshacl/extras/js/loader.py @@ -1,13 +1,20 @@ # # import typing -import regex + from urllib import request + +import regex + + if typing.TYPE_CHECKING: from pyduktape2 import DuktapeContext JS_FN_RE1 = regex.compile(rb'function\s+([^ \n]+)\s*\((.*)\)\s*\{', regex.MULTILINE, regex.IGNORECASE) -JS_FN_RE2 = regex.compile(rb'(?:let|const|var)\s+([^ \n]+)\s*=\s*function\s*\((.*)\)\s*\{', regex.MULTILINE, regex.IGNORECASE) +JS_FN_RE2 = regex.compile( + rb'(?:let|const|var)\s+([^ \n]+)\s*=\s*function\s*\((.*)\)\s*\{', regex.MULTILINE, regex.IGNORECASE +) + def get_js_from_web(url: str): """ @@ -16,8 +23,9 @@ def get_js_from_web(url: str): :type url: str :return: """ - headers = {'Accept': 'application/javascript, text/javascript, application/ecmascript, text/ecmascript,' - 'text/plain'} + headers = { + 'Accept': 'application/javascript, text/javascript, application/ecmascript, text/ecmascript,' 'text/plain' + } r = request.Request(url, headers=headers) resp = request.urlopen(r) code = resp.getcode() @@ -25,12 +33,14 @@ def get_js_from_web(url: str): raise RuntimeError("Cannot pull JS Library URL from the web: {}, code: {}".format(url, str(code))) return resp + def get_js_from_file(filepath: str): if filepath.startswith("file://"): filepath = filepath[7:] f = open(filepath, "rb") return f + def extract_functions(content): fns = {} matches1 = regex.findall(JS_FN_RE1, content) diff --git a/pyshacl/extras/js/rules.py b/pyshacl/extras/js/rules.py index d8f46fa..e119080 100644 --- a/pyshacl/extras/js/rules.py +++ b/pyshacl/extras/js/rules.py @@ -1,20 +1,20 @@ # # import typing -from rdflib.plugins.sparql.operators import register_custom_function, unregister_custom_function -from rdflib.plugins.sparql.sparql import SPARQLError from pyshacl.consts import SH from pyshacl.rules.shacl_rule import SHACLRule -from pyshacl.errors import ReportableRuntimeError + from .js_executable import JSExecutable + if typing.TYPE_CHECKING: - from pyshacl.shapes_graph import ShapesGraph from pyshacl.shape import Shape + from pyshacl.shapes_graph import ShapesGraph SH_JSRule = SH.term('JSRule') + class JSRule(SHACLRule): __slots__ = ('js_exe',) @@ -34,15 +34,10 @@ def apply(self, data_graph): if triples is not None and isinstance(triples, (list, tuple)): set_to_add = set() for t in triples: - s,p,o = t[:3] - set_to_add.add((s,p,o)) + s, p, o = t[:3] + set_to_add.add((s, p, o)) sets_to_add.append(set_to_add) for s in sets_to_add: for t in s: data_graph.add(t) return - - - - - diff --git a/pyshacl/extras/js/target.py b/pyshacl/extras/js/target.py index cb51854..6726722 100644 --- a/pyshacl/extras/js/target.py +++ b/pyshacl/extras/js/target.py @@ -1,23 +1,26 @@ # # import typing -from typing import List, Dict + +from typing import Dict, List from warnings import warn from rdflib import URIRef -from pyshacl.target import SHACLTargetType, BoundSHACLTargetType -from pyshacl.consts import SH, SH_JSTarget, SH_JSTargetType + +from pyshacl.consts import SH_JSTargetType +from pyshacl.errors import ShapeLoadError +from pyshacl.target import BoundSHACLTargetType, SHACLTargetType + from .js_executable import JSExecutable -from ...errors import ShapeLoadError + if typing.TYPE_CHECKING: from pyshacl.pytypes import GraphLike - from pyshacl.shapes_graph import ShapesGraph from pyshacl.shape import Shape + from pyshacl.shapes_graph import ShapesGraph class JSTarget(JSExecutable): - def __init__(self, shapes_graph: 'ShapesGraph', exe_node): super(JSTarget, self).__init__(shapes_graph, exe_node) @@ -74,13 +77,10 @@ def check_params(self, target_declaration): "https://www.w3.org/TR/shacl-js/#JSTargetType", ) if len(vals) > 1: - warn(Warning("Found more than one value for {} on sh:target. Using just first one.".format(n))) + warn(Warning("Found more than one value for {} on sh:target. Using just first one.".format(name))) param_kv[name] = next(iter(vals)) return param_kv def bind(self, shape, target_declaration): param_vals = self.check_params(target_declaration) return BoundJSTargetType(self, target_declaration, shape, param_vals) - - - diff --git a/pyshacl/functions/__init__.py b/pyshacl/functions/__init__.py index a2b8d94..7b6dfbe 100644 --- a/pyshacl/functions/__init__.py +++ b/pyshacl/functions/__init__.py @@ -1,15 +1,22 @@ # -*- coding: utf-8 -*- # -import typing +from typing import TYPE_CHECKING, List, Sequence, Type, Union -from typing import List, Sequence, Union - -from pyshacl.consts import RDF_type, SH_ask, SH_select, SH_SHACLFunction, SH_SPARQLFunction, SH_jsLibrary, \ - SH_jsFunctionName +from pyshacl.consts import ( + RDF_type, + SH_ask, + SH_jsFunctionName, + SH_jsLibrary, + SH_select, + SH_SHACLFunction, + SH_SPARQLFunction, +) from pyshacl.pytypes import GraphLike + from .shacl_function import SHACLFunction, SPARQLFunction -if typing.TYPE_CHECKING: + +if TYPE_CHECKING: from pyshacl.shapes_graph import ShapesGraph @@ -22,15 +29,14 @@ def gather_functions(shacl_graph: 'ShapesGraph') -> Sequence[Union['SHACLFunctio :rtype: [SHACLRule] """ - spq_nodes = set(shacl_graph.subjects(RDF_type, SH_SPARQLFunction)) if shacl_graph.js_enabled: - use_js = True from pyshacl.extras.js.function import JSFunction, SH_JSFunction + js_nodes = set(shacl_graph.subjects(RDF_type, SH_JSFunction)) + use_JSFunction: Union[bool, Type] = JSFunction else: - use_js = False - JSFunction = object # for error checking purposes, needs to be defined + use_JSFunction = False js_nodes = set() scl_nodes = set(shacl_graph.subjects(RDF_type, SH_SHACLFunction)).difference(spq_nodes).difference(js_nodes) to_swap_spq = set() @@ -41,7 +47,7 @@ def gather_functions(shacl_graph: 'ShapesGraph') -> Sequence[Union['SHACLFunctio if has_ask or has_select: to_swap_spq.add(n) continue - if use_js: + if use_JSFunction: has_jslibrary = len(shacl_graph.objects(n, SH_jsLibrary)) > 0 has_jsfuncitonnname = len(shacl_graph.objects(n, SH_jsFunctionName)) > 0 if has_jslibrary or has_jsfuncitonnname: @@ -58,9 +64,9 @@ def gather_functions(shacl_graph: 'ShapesGraph') -> Sequence[Union['SHACLFunctio all_fns.append(SPARQLFunction(n, shacl_graph)) for n in scl_nodes: all_fns.append(SHACLFunction(n, shacl_graph)) - if use_js: + if use_JSFunction and callable(use_JSFunction): for n in js_nodes: - all_fns.append(JSFunction(n, shacl_graph)) + all_fns.append(use_JSFunction(n, shacl_graph)) return all_fns diff --git a/pyshacl/functions/shacl_function.py b/pyshacl/functions/shacl_function.py index c568a02..b5f0eef 100644 --- a/pyshacl/functions/shacl_function.py +++ b/pyshacl/functions/shacl_function.py @@ -195,4 +195,3 @@ def apply(self, g): def unapply(self, g): super(SPARQLFunction, self).unapply(g) unregister_custom_function(self.node, self.execute_from_sparql) - diff --git a/pyshacl/helper/__init__.py b/pyshacl/helper/__init__.py index d1ed7da..d4898ce 100644 --- a/pyshacl/helper/__init__.py +++ b/pyshacl/helper/__init__.py @@ -1,8 +1,10 @@ import sys + mod = sys.modules[__name__] setattr(mod, 'SPARQLQueryHelperCls', None) + def get_query_helper_cls(): # The SPARQLQueryHelper file is expensive to load due to regex compilation steps # so we do it this way so its only loaded when something actually needs to use @@ -10,6 +12,7 @@ def get_query_helper_cls(): SPARQLQueryHelperCls = getattr(mod, 'SPARQLQueryHelperCls', None) if SPARQLQueryHelperCls is None: from .sparql_query_helper import SPARQLQueryHelper + SPARQLQueryHelperCls = SPARQLQueryHelper setattr(mod, 'SPARQLQueryHelperCls', SPARQLQueryHelperCls) return SPARQLQueryHelperCls diff --git a/pyshacl/parameter.py b/pyshacl/parameter.py index 5ce37ec..46ca6a8 100644 --- a/pyshacl/parameter.py +++ b/pyshacl/parameter.py @@ -3,7 +3,7 @@ from rdflib import Literal, URIRef -from .consts import SH_datatype, SH_optional, SH_order, SH_path, SH_nodeKind +from .consts import SH_datatype, SH_nodeKind, SH_optional, SH_order, SH_path from .errors import ConstraintLoadError, ReportableRuntimeError from .shape import Shape diff --git a/pyshacl/rules/__init__.py b/pyshacl/rules/__init__.py index 00b021b..e989407 100644 --- a/pyshacl/rules/__init__.py +++ b/pyshacl/rules/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from collections import defaultdict -from typing import TYPE_CHECKING, Any, Dict, List, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Type, Union from pyshacl.consts import RDF_type, SH_rule, SH_SPARQLRule, SH_TripleRule from pyshacl.errors import RuleLoadError @@ -15,6 +15,7 @@ from .shacl_rule import SHACLRule + def gather_rules(shacl_graph: 'ShapesGraph') -> Dict['Shape', List['SHACLRule']]: """ @@ -26,13 +27,13 @@ def gather_rules(shacl_graph: 'ShapesGraph') -> Dict['Shape', List['SHACLRule']] triple_rule_nodes = set(shacl_graph.subjects(RDF_type, SH_TripleRule)) sparql_rule_nodes = set(shacl_graph.subjects(RDF_type, SH_SPARQLRule)) if shacl_graph.js_enabled: - use_js = True from pyshacl.extras.js.rules import JSRule, SH_JSRule + js_rule_nodes = set(shacl_graph.subjects(RDF_type, SH_JSRule)) + use_JSRule: Union[bool, Type] = JSRule else: - use_js = False + use_JSRule = False js_rule_nodes = set() - JSRule = object # to keep the linter happy overlaps = triple_rule_nodes.intersection(sparql_rule_nodes) if len(overlaps) > 0: raise RuleLoadError( @@ -42,14 +43,12 @@ def gather_rules(shacl_graph: 'ShapesGraph') -> Dict['Shape', List['SHACLRule']] overlaps = triple_rule_nodes.intersection(js_rule_nodes) if len(overlaps) > 0: raise RuleLoadError( - "A SHACL Rule cannot be both a TripleRule and a JSRule.", - "https://www.w3.org/TR/shacl-af/#rules-syntax", + "A SHACL Rule cannot be both a TripleRule and a JSRule.", "https://www.w3.org/TR/shacl-af/#rules-syntax", ) overlaps = sparql_rule_nodes.intersection(js_rule_nodes) if len(overlaps) > 0: raise RuleLoadError( - "A SHACL Rule cannot be both a SPARQLRule and a JSRule.", - "https://www.w3.org/TR/shacl-af/#rules-syntax", + "A SHACL Rule cannot be both a SPARQLRule and a JSRule.", "https://www.w3.org/TR/shacl-af/#rules-syntax", ) used_rules = shacl_graph.subject_objects(SH_rule) ret_rules = defaultdict(list) @@ -65,8 +64,8 @@ def gather_rules(shacl_graph: 'ShapesGraph') -> Dict['Shape', List['SHACLRule']] rule: SHACLRule = TripleRule(shape, obj) elif obj in sparql_rule_nodes: rule = SPARQLRule(shape, obj) - elif use_js and obj in js_rule_nodes: - rule = JSRule(shape, obj) + elif use_JSRule and callable(use_JSRule) and obj in js_rule_nodes: + rule = use_JSRule(shape, obj) else: raise RuleLoadError( "when using sh:rule, the Rule must be defined as either a TripleRule or SPARQLRule.", diff --git a/pyshacl/rules/sparql/__init__.py b/pyshacl/rules/sparql/__init__.py index 916aa3f..85b7a83 100644 --- a/pyshacl/rules/sparql/__init__.py +++ b/pyshacl/rules/sparql/__init__.py @@ -8,10 +8,12 @@ from pyshacl.consts import SH_construct from pyshacl.errors import ReportableRuntimeError, RuleLoadError -from pyshacl.rdfutil import clone_graph from pyshacl.helper import get_query_helper_cls +from pyshacl.rdfutil import clone_graph + from ..shacl_rule import SHACLRule + if TYPE_CHECKING: from pyshacl.shape import Shape @@ -65,5 +67,3 @@ def apply(self, data_graph): construct_graphs.add(results.graph) for g in construct_graphs: data_graph = clone_graph(g, target_graph=data_graph) - - diff --git a/pyshacl/shape.py b/pyshacl/shape.py index 2f6e93d..f871ff9 100644 --- a/pyshacl/shape.py +++ b/pyshacl/shape.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- # import logging + from decimal import Decimal -from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Type, Union from rdflib import RDF, BNode, Literal, URIRef @@ -14,6 +15,9 @@ SH_deactivated, SH_description, SH_inversePath, + SH_jsFunctionName, + SH_JSTarget, + SH_JSTargetType, SH_message, SH_name, SH_oneOrMorePath, @@ -30,15 +34,17 @@ SH_targetSubjectsOf, SH_Violation, SH_zeroOrMorePath, - SH_zeroOrOnePath, SH_jsFunctionName, SH_JSTarget, SH_JSTargetType, + SH_zeroOrOnePath, ) from .errors import ConstraintLoadError, ConstraintLoadWarning, ReportableRuntimeError, ShapeLoadError from .helper import get_query_helper_cls from .pytypes import GraphLike + if TYPE_CHECKING: from pyshacl.shapes_graph import ShapesGraph + class Shape(object): __slots__ = ( @@ -231,11 +237,12 @@ def advanced_target(self): custom_targets = set(self.sg.objects(self.node, SH_target)) result_set = dict() if self.sg.js_enabled: - use_js = True from pyshacl.extras.js.target import JSTarget + + use_JSTarget: Union[bool, Type] = JSTarget else: - use_js = False - JSTarget = object # for linter + use_JSTarget = False + for c in custom_targets: ct = dict() selects = list(self.sg.objects(c, SH_select)) @@ -250,9 +257,9 @@ def advanced_target(self): qh.collect_prefixes() ct['qh'] = qh elif has_fnname or (SH_JSTarget in is_types): - if use_js: + if use_JSTarget: ct['type'] = SH_JSTarget - ct['targeter'] = JSTarget(self.sg, c) + ct['targeter'] = use_JSTarget(self.sg, c) else: # Found JSTarget, but JS is not enabled in PySHACL. Ignore this target. pass @@ -515,6 +522,7 @@ def validate( search_parameters = ALL_CONSTRAINT_PARAMETERS.copy() constraint_map = CONSTRAINT_PARAMETERS_MAP.copy() from pyshacl.extras.js.constraint import JSConstraint, SH_js + search_parameters.append(SH_js) constraint_map[SH_js] = JSConstraint else: diff --git a/pyshacl/shapes_graph.py b/pyshacl/shapes_graph.py index c8af745..c5d433f 100644 --- a/pyshacl/shapes_graph.py +++ b/pyshacl/shapes_graph.py @@ -4,9 +4,9 @@ import rdflib +from .constraints.constraint_component import CustomConstraintComponentFactory from .constraints.core.logical_constraints import SH_and, SH_not, SH_or, SH_xone from .constraints.core.shape_based_constraints import SH_qualifiedValueShape -from .constraints.constraint_component import CustomConstraintComponentFactory from .consts import ( SH, OWL_Class, @@ -15,6 +15,7 @@ RDF_type, RDFS_Class, RDFS_subClassOf, + SH_ConstraintComponent, SH_node, SH_NodeShape, SH_path, @@ -24,7 +25,6 @@ SH_targetNode, SH_targetObjectsOf, SH_targetSubjectsOf, - SH_ConstraintComponent ) from .errors import ShapeLoadError from .shape import Shape diff --git a/pyshacl/target.py b/pyshacl/target.py index 704dfe5..10e34ea 100644 --- a/pyshacl/target.py +++ b/pyshacl/target.py @@ -1,13 +1,13 @@ import typing -from typing import List, Sequence, Union +from typing import List, Sequence, Type, Union from warnings import warn from .constraints import ConstraintComponent from .consts import SH, RDF_type, RDFS_subClassOf, SH_parameter, SH_select, SH_SPARQLTargetType from .errors import ConstraintLoadError, ShapeLoadError -from .parameter import SHACLParameter from .helper import get_query_helper_cls +from .parameter import SHACLParameter from .pytypes import GraphLike @@ -187,11 +187,11 @@ def gather_target_types(shacl_graph: 'ShapesGraph') -> Sequence[Union['SHACLTarg sub_targets = sub_targets.difference({SH_JSTarget, SH_SPARQLTarget}) if shacl_graph.js_enabled: - use_js = True from pyshacl.extras.js.target import JSTargetType + + use_js: Union[bool, Type] = JSTargetType else: use_js = False - JSTargetType = object # for linter for s in sub_targets: types = set(shacl_graph.objects(s, RDF_type)) @@ -214,4 +214,3 @@ def gather_target_types(shacl_graph: 'ShapesGraph') -> Sequence[Union['SHACLTarg def apply_target_types(tts: Sequence): for t in tts: t.apply() - diff --git a/pyshacl/validate.py b/pyshacl/validate.py index f41550d..1dc64f9 100644 --- a/pyshacl/validate.py +++ b/pyshacl/validate.py @@ -184,6 +184,7 @@ def __init__( is_js_installed = check_extra_installed('js') if is_js_installed: self.shacl_graph.enable_js() + @property def target_graph(self): return self._target_graph @@ -364,8 +365,11 @@ def validate( shacl_graph=shacl_graph, ont_graph=ont_graph, options={ - 'inference': inference, 'abort_on_error': abort_on_error, 'advanced': advanced, - 'use_js': use_js, 'logger': log + 'inference': inference, + 'abort_on_error': abort_on_error, + 'advanced': advanced, + 'use_js': use_js, + 'logger': log, }, ) conforms, report_graph, report_text = validator.run() @@ -373,7 +377,7 @@ def validate( conforms = False report_graph = e report_text = "Validation Failure - {}".format(e.message) - if do_check_dash_result: + if do_check_dash_result and validator is not None: passes = check_dash_result(validator, report_graph, shacl_graph or data_graph) return passes, report_graph, report_text if do_check_sht_result: From a28b4b03442e15721f9e617b3e6d0eeab50b5806 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Wed, 14 Oct 2020 12:59:08 +1000 Subject: [PATCH 09/10] Update README and Cli Tool to reflect SHACL-JS changes --- README.md | 9 ++++--- pyshacl/cli.py | 10 ++++++++ pyshacl/extras/__init__.py | 3 +-- test/test_schema_org.py | 48 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 test/test_schema_org.py diff --git a/README.md b/README.md index b688dea..153c73f 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ $ deactivate For command line use: _(these example commandline instructions are for a Linux/Unix based OS)_ ```bash -pyshacl -s /path/to/shapesGraph.ttl -m -i rdfs -a -f human /path/to/dataGraph.ttl +pyshacl -s /path/to/shapesGraph.ttl -m -i rdfs -a -j -f human /path/to/dataGraph.ttl ``` Where - `-s` is an (optional) path to the shapes graph to use @@ -43,6 +43,7 @@ Where - `-f` is the ValidationReport output format (`human` = human-readable validation report) - `-m` enable the meta-shacl feature - `-a` enable SHACL Advanced Features + - `-j` enable SHACL-JS Features (if `pyhsacl[js]` is installed) System exit codes are: `0` = DataGraph is Conformant @@ -53,7 +54,7 @@ System exit codes are: Full CLI Usage options: ```bash usage: pyshacl [-h] [-s [SHACL]] [-e [ONT]] [-i {none,rdfs,owlrl,both}] [-m] - [--imports] [--abort] [-a] [-d] [-f {human,turtle,xml,json-ld,nt,n3}] + [--imports] [--abort] [-a] [-j] [-d] [-f {human,turtle,xml,json-ld,nt,n3}] [-df {auto,turtle,xml,json-ld,nt,n3}] [-sf {auto,turtle,xml,json-ld,nt,n3}] [-ef {auto,turtle,xml,json-ld,nt,n3}] [-V] [-o [OUTPUT]] @@ -80,6 +81,7 @@ optional arguments: --imports Allow import of sub-graphs defined in statements with owl:imports. -a, --advanced Enable support for SHACL Advanced Features. + -j, --js Enable support for SHACL-JS Features. --abort Abort on first error. -d, --debug Output additional runtime messages, including violations that didn\'t lead to non-conformance. @@ -104,7 +106,7 @@ For basic use of this module, you can just call the `validate` function of the ` ``` from pyshacl import validate -r = validate(data_graph, shacl_graph=sg, ont_graph=og, inference='rdfs', abort_on_error=False, meta_shacl=False, advanced=False, debug=False) +r = validate(data_graph, shacl_graph=sg, ont_graph=og, inference='rdfs', abort_on_error=False, meta_shacl=False, advanced=False, js=False, debug=False) conforms, results_graph, results_text = r ``` @@ -117,6 +119,7 @@ Options are 'rdfs', 'owlrl', 'both', or 'none'. The default is 'none'. * `abort_on_error` (optional) a Python `bool` value to indicate whether or not the program should abort after encountering a validation error or to continue. Default is to continue. * `meta_shacl` (optional) a Python `bool` value to indicate whether or not the program should enable the Meta-SHACL feature. Default is False. * `advanced`: (optional) a Python `bool` value to enable SHACL Advanced Features +* `js`: (optional) a Python `bool` value to enable SHACL-JS Features (if `pyshacl[js]` is installed) * `debug` (optional) a Python `bool` value to indicate whether or not the program should emit debugging output text, including violations that didn't lead to non-conformance overall. So when debug is True don't judge conformance by absense of violation messages. Default is False. Some other optional keyword variables available available on the `validate` function: diff --git a/pyshacl/cli.py b/pyshacl/cli.py index 7da7c59..c28d62e 100644 --- a/pyshacl/cli.py +++ b/pyshacl/cli.py @@ -68,6 +68,14 @@ def __call__(self, parser, namespace, values, option_string=None): default=False, help='Enable features from the SHACL Advanced Features specification.', ) +parser.add_argument( + '-j', + '--js', + dest='js', + action='store_true', + default=False, + help='Enable features from the SHACL-JS Specification.', +) parser.add_argument('--abort', dest='abort', action='store_true', default=False, help='Abort on first error.') parser.add_argument( '-d', '--debug', dest='debug', action='store_true', default=False, help='Output additional runtime messages.' @@ -141,6 +149,8 @@ def main(): validator_kwargs['meta_shacl'] = True if args.advanced: validator_kwargs['advanced'] = True + if args.js: + validator_kwargs['js'] = True if args.abort: validator_kwargs['abort_on_error'] = True if args.shacl_file_format: diff --git a/pyshacl/extras/__init__.py b/pyshacl/extras/__init__.py index 7585e9f..7b01657 100644 --- a/pyshacl/extras/__init__.py +++ b/pyshacl/extras/__init__.py @@ -8,8 +8,7 @@ from pkg_resources import DistributionNotFound, UnknownExtra -dev_mode = True - +dev_mode = False @lru_cache() def check_extra_installed(extra_name: str): diff --git a/test/test_schema_org.py b/test/test_schema_org.py new file mode 100644 index 0000000..b4fc09d --- /dev/null +++ b/test/test_schema_org.py @@ -0,0 +1,48 @@ +from pyshacl import validate +import rdflib + +SCHEMA_PATH = "http://datashapes.org/schema.ttl" + +data = """\ +@prefix ex: . +@prefix sch: . +@prefix xsd: . + +ex:asdgjkj a sch:CommunicateAction ; + sch:about [ a sch:GameServer ; + sch:playersOnline "42"^^xsd:integer ] . +""" + +shacl = """\ +# baseURI: http://example.org/myschema +# imports: http://datashapes.org/schema + +@prefix owl: . +@prefix rdf: . +@prefix rdfs: . +@prefix schema: . +@prefix sh: . +@prefix xsd: . +@prefix : . + + + a owl:Ontology ; + rdfs:comment "Dummy Schema importing from Schema.org shape"@en ; + rdfs:label "Schema.org importer" ; + owl:imports . +""" + + +def schema_org(): + dataGraph = rdflib.Graph().parse(data=data, format='ttl') + #print(dataGraph.serialize(format='ttl').decode('utf8')) + + shaclGraph = rdflib.Dataset() + shaclGraph.parse(data=shacl, format='ttl') + + report = validate(dataGraph, shacl_graph=shaclGraph, abort_on_error=False, inference='both', meta_shacl=False, debug=False, advanced=True, do_owl_imports=True) + + print(report[2]) + +if __name__ == "__main__": + schema_org() From 4190f04f345422ac8db8f43da893a53440c113b8 Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Wed, 14 Oct 2020 13:21:38 +1000 Subject: [PATCH 10/10] Fix JS tests in TOX runners --- MANIFEST.in | 3 +-- poetry.lock | 8 +++++--- pyproject.toml | 6 ++++-- pyshacl/extras/__init__.py | 1 + test/test_js/test_js_constraint.py | 2 +- test/test_js/test_js_constraint_component.py | 2 +- test/test_js/test_js_constraint_path_component.py | 2 +- test/test_js/test_js_function.py | 2 +- test/test_js/test_js_rules.py | 2 +- test/test_js/test_js_target.py | 2 +- test/test_js/test_js_target_type.py | 2 +- 11 files changed, 18 insertions(+), 14 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 8b977a9..d0246f0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,6 +6,5 @@ include pyshacl/assets/*.pickle include pyshacl/*.spec include test/*.py include test/issues/*.py -recursive-include test/resources *.ttl +recursive-include test/resources *.ttl *.js recursive-include docs *.txt *.md - diff --git a/poetry.lock b/poetry.lock index d3f2646..09ffc6a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -80,8 +80,9 @@ version = "4.5.4" [[package]] category = "main" description = "The Cython compiler for writing C extensions for the Python language." +marker = "python_version >= \"3.6\"" name = "cython" -optional = true +optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" version = "0.29.21" @@ -253,8 +254,9 @@ version = "2.6.0" [[package]] category = "main" description = "Python integration for the Duktape Javascript interpreter" +marker = "python_version >= \"3.6\"" name = "pyduktape2" -optional = true +optional = false python-versions = "*" version = "0.4.1" @@ -419,7 +421,7 @@ dev-type-checking = [] js = ["pyduktape2"] [metadata] -content-hash = "aa9587d8256b5b2fe98784e5f9ebec5a83ec213201374cd0f024280ba0ae7ac1" +content-hash = "f18f78fa2063744efb8014dd0f5554f12f8946a8e663d384ee4e794366996c15" python-versions = "^3.6" # Compatible python versions must be declared here [metadata.files] diff --git a/pyproject.toml b/pyproject.toml index d762804..afd82f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,17 +47,19 @@ python = "^3.6" # Compatible python versions must be declared here rdflib = "^5.0.0" rdflib-jsonld = "^0.5.0" owlrl = "^5.2.1" -pyduktape2 = {version="^0.4.1", optional=true} +pyduktape2 = {version="^0.4.1", python=">=3.6", optional=true} [tool.poetry.dev-dependencies] coverage = "^4.5" pytest = "^5.0" pytest-cov = "^2.8.1" flake8 = "^3.7" +pyduktape2 = {version="^0.4.1", python=">=3.6"} isort = {version="^5.0.0", python=">=3.6"} black = {version=">=18.9b0,<=19.10b0", python=">=3.6"} mypy = {version="^0.770.0", python=">=3.6"} + [tool.poetry.extras] js = ["pyduktape2"] dev-lint = ["isort", "black", "flake8"] @@ -125,7 +127,7 @@ deps = py36: coveralls passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH skip_install = true -commands_pre = poetry install -vvv +commands_pre = poetry install -vvv --extras "js" commands = - poetry show poetry run pytest --cov=pyshacl test/ diff --git a/pyshacl/extras/__init__.py b/pyshacl/extras/__init__.py index 7b01657..6b51cf9 100644 --- a/pyshacl/extras/__init__.py +++ b/pyshacl/extras/__init__.py @@ -10,6 +10,7 @@ dev_mode = False + @lru_cache() def check_extra_installed(extra_name: str): if dev_mode: diff --git a/test/test_js/test_js_constraint.py b/test/test_js/test_js_constraint.py index 96e2d1b..42458b6 100644 --- a/test/test_js/test_js_constraint.py +++ b/test/test_js/test_js_constraint.py @@ -13,7 +13,7 @@ sh:js [ a sh:JSConstraint ; sh:message "Values are literals with German language tag." ; - sh:jsLibrary [ sh:jsLibraryURL "file://resources/js/germanLabel.js" ] ; + sh:jsLibrary [ sh:jsLibraryURL "file://test/resources/js/germanLabel.js" ] ; sh:jsFunctionName "validateGermanLabel" ; ] . ''' diff --git a/test/test_js/test_js_constraint_component.py b/test/test_js/test_js_constraint_component.py index 2ae2b16..b1a8877 100644 --- a/test/test_js/test_js_constraint_component.py +++ b/test/test_js/test_js_constraint_component.py @@ -23,7 +23,7 @@ Their string value is accessed via the .lex and .uri attributes. The function returns true if no violation has been found. """ ; - sh:jsLibrary [ sh:jsLibraryURL "file://resources/js/hasMaxLength.js"^^xsd:anyURI ] ; + sh:jsLibrary [ sh:jsLibraryURL "file://test/resources/js/hasMaxLength.js"^^xsd:anyURI ] ; sh:jsFunctionName "hasMaxLength" . ex:TestShape diff --git a/test/test_js/test_js_constraint_path_component.py b/test/test_js/test_js_constraint_path_component.py index ada61bd..20006a6 100644 --- a/test/test_js/test_js_constraint_path_component.py +++ b/test/test_js/test_js_constraint_path_component.py @@ -18,7 +18,7 @@ ex:hasMaxCount a sh:JSValidator ; sh:message "Path has more than {$maxCount} values." ; - sh:jsLibrary [ sh:jsLibraryURL "file://resources/js/hasMaxCount.js"^^xsd:anyURI ] ; + sh:jsLibrary [ sh:jsLibraryURL "file://test/resources/js/hasMaxCount.js"^^xsd:anyURI ] ; sh:jsFunctionName "hasMaxCount" . ex:TestShape diff --git a/test/test_js/test_js_function.py b/test/test_js/test_js_function.py index 65a014e..0f88d38 100644 --- a/test/test_js/test_js_function.py +++ b/test/test_js/test_js_function.py @@ -69,7 +69,7 @@ sh:description "The second operand" ; ] ; sh:returnType xsd:integer ; - sh:jsLibrary [ sh:jsLibraryURL "file://resources/js/multiply.js" ] ; + sh:jsLibrary [ sh:jsLibraryURL "file://test/resources/js/multiply.js" ] ; sh:jsFunctionName "multiply" ; . ''' diff --git a/test/test_js/test_js_rules.py b/test/test_js/test_js_rules.py index e278089..4f73f5c 100644 --- a/test/test_js/test_js_rules.py +++ b/test/test_js/test_js_rules.py @@ -13,7 +13,7 @@ sh:rule [ a sh:JSRule ; # This triple is optional sh:jsFunctionName "computeArea" ; - sh:jsLibrary [ sh:jsLibraryURL "resources/js/rectangle.js"^^xsd:anyURI ] ; + sh:jsLibrary [ sh:jsLibraryURL "file://test/resources/js/rectangle.js"^^xsd:anyURI ] ; ] ; sh:property [ sh:path ex:area ; diff --git a/test/test_js/test_js_target.py b/test/test_js/test_js_target.py index f41a4e3..5e79fb7 100644 --- a/test/test_js/test_js_target.py +++ b/test/test_js/test_js_target.py @@ -37,7 +37,7 @@ sh:target [ rdf:type sh:JSTarget ; sh:jsFunctionName "findThings" ; - sh:jsLibrary [ sh:jsLibraryURL "resources/js/findThings.js"^^xsd:anyURI ] ; + sh:jsLibrary [ sh:jsLibraryURL "file://test/resources/js/findThings.js"^^xsd:anyURI ] ; ] ; . diff --git a/test/test_js/test_js_target_type.py b/test/test_js/test_js_target_type.py index 132babd..416f435 100644 --- a/test/test_js/test_js_target_type.py +++ b/test/test_js/test_js_target_type.py @@ -70,7 +70,7 @@ sh:nodeKind sh:IRI ; ] ; sh:jsFunctionName "findBornIn" ; - sh:jsLibrary [ sh:jsLibraryURL "resources/js/findBornIn.js"^^xsd:anyURI ] ; + sh:jsLibrary [ sh:jsLibraryURL "file://test/resources/js/findBornIn.js"^^xsd:anyURI ] ; . ex:GermanCitizenShape