feat(rdf-api): Add a general-purpose SHACL validation utility (DSP-930) (#1762)

Author: Benjamin Geer

docs/05-internals/design/principles/rdf-api.md

@@ -49,10 +49,12 @@ The API is in the package `org.knora.webapi.messages.util.rdf`. It includes:
 - `JsonLDUtil`, which provides specialised functionality for working
   with RDF in JSON-LD format, and for converting between RDF models
   and JSON-LD documents. `RdfFormatUtil` uses `JsonLDUtil` when appropriate.
+  
+- `ShaclValidator`, which validates RDF models using SHACL shapes.
 
 To work with RDF models, start with `RdfFeatureFactory`, which returns instances
-of `RdfNodeFactory`, `RdfModelFactory`, and `RdfFormatUtil`, using feature toggle
-configuration. `JsonLDUtil` does not need a feature factory.
+of `RdfNodeFactory`, `RdfModelFactory`, `RdfFormatUtil`, and `ShaclValidator`,
+using feature toggle configuration. `JsonLDUtil` does not need a feature factory.
 
 To iterate efficiently over the statements in an `RdfModel`, use its `iterator` method.
 An `RdfModel` cannot be modified while you are iterating over it.
@@ -85,6 +87,15 @@ an `RdfModel`. To do this, use the `RdfModel.asRepository` method, which
 returns an `RdfRepository` that can run `SELECT` queries.
 
 
+## SHACL validation
+
+On startup, graphs of SHACL shapes are loaded from Turtle files in a directory specified
+by `app.shacl.shapes-dir` in `application.conf`, and in subdirectories of
+that directory. To validate the default graph of an `RdfModel` using a graph of
+SHACL shapes, call `ShaclValidator.validate`, specifying the relative path of the
+Turtle file containing the graph of shapes.
+
+
 ## Implementations
 
 - The Jena-based implementation, in package `org.knora.webapi.messages.util.rdf.jenaimpl`.

test_data/shacl/test/person.ttl

@@ -0,0 +1,13 @@
+@prefix ex: <http://example.com/ns#> .
+@prefix sh: <http://www.w3.org/ns/shacl#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix foaf: <http://xmlns.com/foaf/0.1/> .
+
+ex:PersonShape a sh:NodeShape ;
+  sh:targetClass foaf:Person ;
+  sh:property ex:PersonShapeProperty .
+
+ex:PersonShapeProperty sh:path foaf:age ;
+  sh:datatype xsd:int ;
+  sh:maxCount 1 ;
+  sh:minCount 1 .

third_party/dependencies.bzl

@@ -84,7 +84,9 @@ def dependencies():
             "org.xmlunit:xmlunit-core:2.1.1",
 
             # other
-            "org.eclipse.rdf4j:rdf4j-runtime:3.0.0",
+            "org.eclipse.rdf4j:rdf4j-runtime:3.4.4",
+            "org.eclipse.rdf4j:rdf4j-client:3.4.4",
+            "org.eclipse.rdf4j:rdf4j-shacl:3.4.4",
             "org.rogach:scallop_2.12:3.2.0",
             "com.google.gwt:gwt-servlet:2.8.0",
             "net.sf.saxon:Saxon-HE:9.9.0-2",

webapi/BUILD.bazel

@@ -212,6 +212,7 @@ scala_library(
         "//webapi/src/main/scala/org/knora/webapi/app",
         "//webapi/src/main/scala/org/knora/webapi/core",
         "//webapi/src/main/scala/org/knora/webapi/exceptions",
+        "//webapi/src/main/scala/org/knora/webapi/feature",
         "//webapi/src/main/scala/org/knora/webapi/instrumentation",
         "//webapi/src/main/scala/org/knora/webapi/messages",
         "//webapi/src/main/scala/org/knora/webapi/routing",

webapi/src/it/scala/org/knora/webapi/ITKnoraFakeSpec.scala

@@ -20,6 +20,7 @@
 package org.knora.webapi
 
 import akka.actor.ActorSystem
+import akka.event.LoggingAdapter
 import akka.http.scaladsl.Http
 import akka.http.scaladsl.client.RequestBuilding
 import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes}
@@ -33,6 +34,7 @@ import org.scalatest.wordspec.AnyWordSpecLike
 import org.scalatest.{BeforeAndAfterAll, Suite}
 import spray.json.{JsObject, _}
 import org.knora.webapi.messages.StringFormatter
+import org.knora.webapi.messages.util.rdf.RdfFeatureFactory
 
 import scala.concurrent.duration.{Duration, _}
 import scala.concurrent.{Await, ExecutionContext}
@@ -63,8 +65,9 @@ class ITKnoraFakeSpec(_system: ActorSystem) extends Core with KnoraFakeCore with
 
     /* Needs to be initialized before any responders */
     StringFormatter.initForTest()
+    RdfFeatureFactory.init(settings)
 
-    val log = akka.event.Logging(system, this.getClass)
+    val log: LoggingAdapter = akka.event.Logging(system, this.getClass)
 
     protected val baseApiUrl: String = settings.internalKnoraApiBaseUrl
     protected val baseInternalSipiUrl: String = settings.internalSipiBaseUrl

webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala

@@ -36,7 +36,7 @@ import org.knora.webapi.exceptions.AssertionException
 import org.knora.webapi.messages.StringFormatter
 import org.knora.webapi.messages.app.appmessages.{AppStart, AppStop, SetAllowReloadOverHTTPState}
 import org.knora.webapi.messages.store.triplestoremessages.{RdfDataObject, TriplestoreJsonProtocol}
-import org.knora.webapi.messages.util.rdf.{JsonLDDocument, JsonLDUtil}
+import org.knora.webapi.messages.util.rdf.{JsonLDDocument, JsonLDUtil, RdfFeatureFactory}
 import org.knora.webapi.settings._
 import org.knora.webapi.util.StartupUtils
 import org.scalatest.matchers.should.Matchers
@@ -75,6 +75,7 @@ class ITKnoraLiveSpec(_system: ActorSystem) extends Core with StartupUtils with
 
     /* Needs to be initialized before any responders */
     StringFormatter.initForTest()
+    RdfFeatureFactory.init(settings)
 
     val log: LoggingAdapter = akka.event.Logging(system, this.getClass)
 

webapi/src/main/resources/application.conf

@@ -264,35 +264,41 @@ akka-http-cors {
 }
 
 app {
-     feature-toggles {
-         new-list-admin-routes {
-             description = "Replace the old list admin routes with new ones."
-
-             available-versions = [ 1 ]
-             default-version = 1
-             enabled-by-default = no
-             override-allowed = yes
-             expiration-date = "2021-12-01T00:00:00Z"
-
-             developer-emails = [
-                 "Sepideh Alassi <sepideh.alassi@dasch.swiss>"
-                 "Benjamin Geer <benjamin.geer@dasch.swiss>"
-             ]
-         }
-
-         jena-rdf-library {
-             description = "Use the Jena API for RDF processing. If turned off, use the RDF4J API."
-
-             available-versions = [ 1 ]
-             default-version = 1
-             enabled-by-default = no
-             override-allowed = yes
-
-             developer-emails = [
-                 "Benjamin Geer <benjamin.geer@dasch.swiss>"
-             ]
-         }
-     }
+    feature-toggles {
+        new-list-admin-routes {
+            description = "Replace the old list admin routes with new ones."
+
+            available-versions = [ 1 ]
+            default-version = 1
+            enabled-by-default = no
+            override-allowed = yes
+            expiration-date = "2021-12-01T00:00:00Z"
+
+            developer-emails = [
+                "Sepideh Alassi <sepideh.alassi@dasch.swiss>"
+                "Benjamin Geer <benjamin.geer@dasch.swiss>"
+            ]
+        }
+
+        jena-rdf-library {
+            description = "Use the Jena API for RDF processing. If turned off, use the RDF4J API."
+
+            available-versions = [ 1 ]
+            default-version = 1
+            enabled-by-default = no
+            override-allowed = yes
+
+            developer-emails = [
+                "Benjamin Geer <benjamin.geer@dasch.swiss>"
+            ]
+        }
+    }
+
+    shacl {
+        # The directory that SHACL shapes are loaded from.
+        shapes-dir = "shacl"
+        shapes-dir = ${?KNORA_WEBAPI_SHACLE_SHAPES_DIR}
+    }
 
     print-extended-config = false // If true, an extended list of configuration parameters will be printed out at startup.
     print-extended-config = ${?KNORA_WEBAPI_PRINT_EXTENDED_CONFIG}

webapi/src/main/scala/org/knora/webapi/app/LiveCore.scala

@@ -27,8 +27,8 @@ import org.knora.webapi.settings.{KnoraDispatchers, KnoraSettings, KnoraSettings
 import scala.concurrent.ExecutionContext
 import scala.language.postfixOps
 import scala.languageFeature.postfixOps
-
 import org.knora.webapi.core.Core
+import org.knora.webapi.messages.util.rdf.RdfFeatureFactory
 
 
 /**
@@ -57,8 +57,10 @@ trait LiveCore extends Core {
     implicit val executionContext: ExecutionContext = system.dispatchers.lookup(KnoraDispatchers.KnoraActorDispatcher)
 
 
-    // Initialise StringFormatter with the system settings. This must happen before any responders are constructed.
+    // Initialise StringFormatter and RdfFeatureFactory with the system settings.
+    // This must happen before any responders are constructed.
     StringFormatter.init(settings)
+    RdfFeatureFactory.init(settings)
 
     /**
      * The main application supervisor actor which is at the top of the actor

webapi/src/main/scala/org/knora/webapi/messages/BUILD.bazel

@@ -37,6 +37,7 @@ scala_library(
         "@maven//:org_eclipse_rdf4j_rdf4j_repository_sail",
         "@maven//:org_eclipse_rdf4j_rdf4j_sail_api",
         "@maven//:org_eclipse_rdf4j_rdf4j_sail_memory",
+        "@maven//:org_eclipse_rdf4j_rdf4j_shacl",
         "@maven//:org_jodd_jodd",
         "@maven//:org_scala_lang_modules_scala_xml_2_12",
         "@maven//:org_scala_lang_scala_library",

webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala

@@ -125,6 +125,16 @@ object OntologyConstants {
         val DateTimeStamp: IRI = XsdPrefixExpansion + "dateTimeStamp"
     }
 
+    object Shacl {
+        val ShaclPrefixExpansion: IRI = "http://www.w3.org/ns/shacl#"
+
+        val Conforms: IRI = ShaclPrefixExpansion + "conforms"
+        val Result: IRI = ShaclPrefixExpansion + "result"
+        val SourceConstraintComponent: IRI = ShaclPrefixExpansion + "sourceConstraintComponent"
+        val DatatypeConstraintComponent: IRI = ShaclPrefixExpansion + "DatatypeConstraintComponent"
+        val MaxCountConstraintComponent: IRI = ShaclPrefixExpansion + "MaxCountConstraintComponent"
+    }
+
     /**
      * http://schema.org
      */