diff --git a/build.sbt b/build.sbt index 393758b9..974340ee 100644 --- a/build.sbt +++ b/build.sbt @@ -31,6 +31,7 @@ lazy val scalatagsVersion = "0.6.7" lazy val scallopVersion = "3.3.2" lazy val seleniumVersion = "2.35.0" lazy val sextVersion = "0.2.6" +lazy val shaclTQVersion = "1.3.2" lazy val typesafeConfigVersion = "1.3.4" lazy val xercesVersion = "2.12.0" @@ -72,7 +73,8 @@ lazy val jenaFuseki = "org.apache.jena" % "jena-fuseki-main" lazy val jline = "org.jline" % "jline" % jlineVersion lazy val jna = "net.java.dev.jna" % "jna" % jnaVersion lazy val pprint = "com.lihaoyi" %% "pprint" % pprintVersion -lazy val rdf4j_runtime = "org.eclipse.rdf4j" % "rdf4j-runtime" % rdf4jVersion +lazy val rdf4j_runtime = "org.eclipse.rdf4j" % "rdf4j-runtime" % rdf4jVersion +lazy val shaclTQ = "org.topbraid" % "shacl" % shaclTQVersion lazy val scalaLogging = "com.typesafe.scala-logging" %% "scala-logging" % loggingVersion lazy val scallop = "org.rogach" %% "scallop" % scallopVersion lazy val scalactic = "org.scalactic" %% "scalactic" % scalacticVersion @@ -149,7 +151,8 @@ lazy val schema = project shex, shacl, shapeMaps, - jenaShacl + jenaShacl, + shaclTQ ) ) .dependsOn( diff --git a/modules/converter/src/main/scala/es/weso/shacl/converter/Shacl2ShEx.scala b/modules/converter/src/main/scala/es/weso/shacl/converter/Shacl2ShEx.scala index 9ca3a069..c99bb322 100644 --- a/modules/converter/src/main/scala/es/weso/shacl/converter/Shacl2ShEx.scala +++ b/modules/converter/src/main/scala/es/weso/shacl/converter/Shacl2ShEx.scala @@ -5,41 +5,61 @@ import cats.data._ import cats.implicits._ import es.weso.rdf.PrefixMap import es.weso.rdf.nodes._ -import es.weso.rdf.path.{InversePath, PredicatePath, SHACLPath} +import es.weso.rdf.PREFIXES._ +import es.weso.rdf.path._ import es.weso.shex.implicits.showShEx._ import es.weso.shex.linter.ShExLinter import es.weso.{shacl, _} -import es.weso.shacl.TargetClass -import es.weso.shacl.TargetNode -import es.weso.shacl.TargetObjectsOf -import es.weso.shacl.TargetSubjectsOf +import es.weso.{shapeMaps,_} +import es.weso.shapeMaps.BNodeLabel +import es.weso.shapeMaps.IRILabel object Shacl2ShEx { - def shacl2ShEx(schema: shacl.Schema): Either[String, (shex.Schema, shapeMaps.QueryShapeMap)] = { + def shacl2ShEx(schema: shacl.Schema, nodesPrefixMap: Option[PrefixMap] = None): Either[String, (shex.Schema, shapeMaps.QueryShapeMap)] = { val (state, eitherSchema) = cnvSchema(schema).value.run(initialState) val e = for { shexSchema <- eitherSchema schema1 = shexSchema.addTripleExprMap(state.tripleExprMap) - queryMap <- cnvShapeMap(schema) + queryMap <- cnvShapeMap(schema, nodesPrefixMap) lintedSchema <- ShExLinter.inlineInclusions(schema1) } yield (lintedSchema,queryMap) - // println(s"Result of conversion: \n$e") + // priprontln(s"Result of conversion: \n$e") e } - def cnvShapeMap(schema: shacl.Schema): Either[String,shapeMaps.QueryShapeMap] = { - val associations: List[Association] = schema.shapesMap.values.map(shape2Associations) - Right(shapeMaps.QueryShapeMap(List(), PrefixMap.empty, PrefixMap.empty)) + def cnvShapeMap(schema: shacl.Schema, nodesPrefixMap: Option[PrefixMap] = None): Either[String,shapeMaps.QueryShapeMap] = for { + associations <- schema.shapesMap.values.toList.map(shape2Associations).sequence + } yield { + val as: List[shapeMaps.Association] = associations.flatten + shapeMaps.QueryShapeMap(as, nodesPrefixMap.getOrElse(schema.pm), schema.pm) } - private def shape2Associations(shape: shacl.Shape): List[Association] = shape.targets.map(target2Association) + private def shape2Associations(shape: shacl.Shape): Either[String, List[shapeMaps.Association]] = + shape.targets.toList.map(target2Association(shape)).sequence - private def target2Association(target: shacl.Target): Association = target match { - case TargetClass(node) => ??? - case TargetNode(node) => ??? - case TargetObjectsOf(pred) => ??? - case TargetSubjectsOf(pred) => ??? + private def rdfTypeShacl: SHACLPath = + SequencePath(List(PredicatePath(`rdf:type`), + ZeroOrMorePath(PredicatePath(`rdfs:subClassOf`))) + ) + + private def shape2ShapeMapLabel(shape: shacl.Shape): Either[String, shapeMaps.ShapeMapLabel] = shape.id match { + case bnode: BNode => Right(shapeMaps.BNodeLabel(bnode)) + case iri: IRI => Right(shapeMaps.IRILabel(iri)) + case _ => Left(s"Cannot convert shape identifier ${shape.id} to shape map label") + } + + private def target2Association(shape: shacl.Shape)(target: shacl.Target): Either[String,shapeMaps.Association] = for { + lbl <- shape2ShapeMapLabel(shape) + nodeSelector <- target2NodeSelector(target) + } yield shapeMaps.Association(nodeSelector,lbl,shapeMaps.Info.undefined("Generated by Shacl2ShEx converter")) + + private def target2NodeSelector(target: shacl.Target): Either[String,shapeMaps.NodeSelector] = target match { + case shacl.TargetClass(node) => Right(shapeMaps.TriplePattern(shapeMaps.Focus,rdfTypeShacl,shapeMaps.NodePattern(node))) + case shacl.TargetNode(node) => Right(shapeMaps.RDFNodeSelector(node)) + case shacl.TargetObjectsOf(pred) => Right(shapeMaps.TriplePattern(shapeMaps.WildCard, PredicatePath(pred), shapeMaps.Focus)) + case shacl.TargetSubjectsOf(pred) => Right(shapeMaps.TriplePattern(shapeMaps.Focus, PredicatePath(pred), shapeMaps.WildCard)) + case _ => Left(s"target2NodeSelector: Unsupported conversion of ${target}") } case class State(tripleExprMap: TEMap) { diff --git a/modules/converter/src/test/scala/es/weso/shacl/converter/shacl2ShapeMapTest.scala b/modules/converter/src/test/scala/es/weso/shacl/converter/shacl2ShapeMapTest.scala new file mode 100644 index 00000000..98fee64b --- /dev/null +++ b/modules/converter/src/test/scala/es/weso/shacl/converter/shacl2ShapeMapTest.scala @@ -0,0 +1,84 @@ +package es.weso.shacl.converter + +import cats.implicits._ +import es.weso._ +import es.weso.rdf.jena.RDFAsJenaModel +import es.weso.utils.IOUtils +import org.scalatest.matchers.should._ +import org.scalatest.funspec._ +import es.weso.shapeMaps.Association +import es.weso.shapeMaps.ShapeMapLabel +import es.weso.shapeMaps.NodeSelector +import es.weso.shapeMaps.ShapeMap + +class shacl2ShapeMapTest extends AnyFunSpec with Matchers { + + describe("shacl2ShapeMaps converter") { + { + shouldConvertSHACLShapeMap( + """|prefix : + |prefix sh: + |:S a sh:NodeShape ; + | sh:targetNode :x ; + | sh:nodeKind sh:IRI . + """.stripMargin, + """:x@:S""".stripMargin) + + shouldConvertSHACLShapeMap( + """|prefix : + |prefix sh: + |prefix rdf: + |prefix rdfs: + |:S a sh:NodeShape ; + | sh:targetClass :C ; + | sh:nodeKind sh:IRI . + """.stripMargin, + """{FOCUS rdf:type/rdfs:subClassOf* :C}@:S""".stripMargin) + + shouldConvertSHACLShapeMap( + """|prefix : + |prefix sh: + |prefix rdf: + |prefix rdfs: + |:S a sh:NodeShape ; + | sh:targetSubjectsOf :p ; + | sh:nodeKind sh:IRI . + """.stripMargin, + """{FOCUS :p _}@:S""".stripMargin) + + shouldConvertSHACLShapeMap( + """|prefix : + |prefix sh: + |prefix rdf: + |prefix rdfs: + |:S a sh:NodeShape ; + | sh:targetObjectsOf :p ; + | sh:nodeKind sh:IRI . + """.stripMargin, + """{_ :p FOCUS}@:S""".stripMargin) + } + } + + private def getAssociationPair(a: Association): (NodeSelector,ShapeMapLabel) = (a.node,a.shape) + + private def getPairs(shapeMap: ShapeMap): List[(NodeSelector,ShapeMapLabel)] = shapeMap.associations.map(getAssociationPair) + + def shouldConvertSHACLShapeMap(strSHACL: String, expected: String): Unit = { + it(s"Should convert: $strSHACL to ShapeMap and obtain: $expected") { + val cmp = RDFAsJenaModel.fromString(strSHACL, "TURTLE", None).flatMap(_.use(shaclRDF => for { + shacl <- RDF2Shacl.getShacl(shaclRDF) + shapeMapConverted <- IOUtils.fromES(Shacl2ShEx.shacl2ShEx(shacl).leftMap(e => s"Error in Shacl2ShEx conversion: $e")) + expectedShapeMap <- IOUtils.fromES(shapeMaps.ShapeMap.fromString(expected, "Compact", None,shacl.pm,shacl.pm).leftMap(e => s"Error in Shape maps parsing: $e")) + } yield (shapeMapConverted, expectedShapeMap, shacl))) + cmp.attempt.unsafeRunSync().fold( + e => fail(s"Error: $e"), + values => { + val (converted, expected, shacl) = values + val (schema,shapeMap) = converted + getPairs(shapeMap) should contain theSameElementsAs getPairs(expected) + } + ) + } + } + +} diff --git a/modules/schema/src/main/scala/es/weso/schema/Schema.scala b/modules/schema/src/main/scala/es/weso/schema/Schema.scala index baf5629a..003c6d17 100644 --- a/modules/schema/src/main/scala/es/weso/schema/Schema.scala +++ b/modules/schema/src/main/scala/es/weso/schema/Schema.scala @@ -63,6 +63,9 @@ abstract class Schema { */ def pm: PrefixMap + /** + * Convert this schema into another schema + */ def convert(targetFormat: Option[String], targetEngine: Option[String], base: Option[IRI] diff --git a/modules/schema/src/main/scala/es/weso/schema/Schemas.scala b/modules/schema/src/main/scala/es/weso/schema/Schemas.scala index de929a08..2e4205d2 100644 --- a/modules/schema/src/main/scala/es/weso/schema/Schemas.scala +++ b/modules/schema/src/main/scala/es/weso/schema/Schemas.scala @@ -14,9 +14,9 @@ object Schemas { lazy val shEx: Schema = ShExSchema.empty lazy val shaclex : Schema = ShaclexSchema.empty lazy val jenaShacl : Schema = JenaShacl.empty - // lazy val shacl_tq = Shacl_TQ.empty + lazy val shaclTQ = ShaclTQ.empty - val availableSchemas: List[Schema] = List(shEx, shaclex, jenaShacl) // shEx,shaclex) //,shacl_tq) + val availableSchemas: List[Schema] = List(shEx, shaclex, jenaShacl, shaclTQ) val defaultSchema: Schema = shEx val defaultSchemaName: String = defaultSchema.name val defaultSchemaFormat: String = defaultSchema.defaultFormat diff --git a/modules/schema/src/main/scala/es/weso/schema/ShaclTQ.scala b/modules/schema/src/main/scala/es/weso/schema/ShaclTQ.scala new file mode 100644 index 00000000..241e87f1 --- /dev/null +++ b/modules/schema/src/main/scala/es/weso/schema/ShaclTQ.scala @@ -0,0 +1,182 @@ +package es.weso.schema + +import cats.implicits._ +import es.weso.rdf._ +import es.weso.rdf.nodes._ +import es.weso.rdf.jena.RDFAsJenaModel +import cats.effect._ +import cats.effect.concurrent._ +import scala.util.control.NoStackTrace +import org.apache.jena.rdf.model.Model +import org.apache.jena.rdf.model.ModelFactory +import java.io.StringReader +import org.apache.jena.riot._ +import org.apache.jena.riot.Lang +import org.apache.jena.rdf.model.ModelFactory +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.graph.Graph +import org.apache.jena.riot.system.{PrefixMap => _, _} +import org.apache.jena.riot.RDFLanguages +import es.weso.shapeMaps.ResultShapeMap +import collection.JavaConverters._ +import es.weso.shapeMaps.ShapeMap +import java.io._ +import es.weso.utils.JenaUtils +import org.topbraid.shacl.validation.ValidationUtil +import org.topbraid.jenax.util.JenaDatatypes +import org.topbraid.shacl.vocabulary.SH +import org.apache.jena.vocabulary.RDF + +case class ShaclTQException(msg: String) extends Exception(msg) with NoStackTrace + + +case class ShaclTQ(shapesGraph: Model) extends Schema { + override def name = "SHACL_TQ" + + override def formats: Seq[String] = DataFormats.formatNames ++ Seq(Lang.SHACLC.getName().toUpperCase()) + + override def defaultTriggerMode: ValidationTrigger = TargetDeclarations + + override def validate(rdf: RDFReader, trigger: ValidationTrigger, builder: RDFBuilder): IO[Result] = trigger match { + case TargetDeclarations => validateTargetDecls(rdf).map(_.addTrigger(trigger)) + case _ => IO(Result.errStr(s"Not implemented trigger ${trigger.name} for ${name} yet")) + } + + private def validateTargetDecls(rdf: RDFReader): IO[Result] = rdf match { + case rdfJena: RDFAsJenaModel => for { + rdfModel <- rdfJena.getModel + pm <- rdfJena.getPrefixMap + shapesPm = shapesGraph.getNsPrefixMap() + report <- IO { + val report: Resource = ValidationUtil.validateModel(rdfModel, shapesGraph, true); + + // val report: ValidationReport = ShaclValidator.get().validate(shapesGraph.getGraph(), rdfModel.getGraph()) + report + } + result <- report2Result(report, pm, prefixMapFromModel(shapesGraph)) + } yield result + case _ => IO.raiseError(ShaclTQException(s"Not Implemented Jena SHACL validation for ${rdf.rdfReaderName} yet")) + } + + private def prefixMapFromModel(model: Model): PrefixMap = PrefixMap(model.getNsPrefixMap().asScala.toMap.map { + case (alias, iri) => (Prefix(alias), IRI(iri)) + }) + + private def report2Result( + report: Resource, + nodesPrefixMap: PrefixMap, + shapesPrefixMap: PrefixMap + ): IO[Result] = for { + eitherRdf <- report2reader(report.getModel()).attempt + isValid <- conforms(report) + numViolations <- countViolations(report) + } yield { + val message = if (isValid) s"Validated" + else s"Number of violations: ${numViolations}" + val shapesMap = report2ShapesMap() + val errors: Seq[ErrorInfo] = report2errors() + // val esRdf = eitherRdf.leftMap(_.getMessage()) + Result(isValid = isValid, + message = message, + shapeMaps = Seq(shapesMap), + validationReport = JenaShaclReport(report.getModel), + errors = errors, + trigger = Some(TargetDeclarations), + nodesPrefixMap = nodesPrefixMap, + shapesPrefixMap = shapesPrefixMap) + } + + private def conforms(report: Resource): IO[Boolean] = + IO { report.hasProperty(SH.conforms,JenaDatatypes.TRUE) } + + private def countViolations(report: Resource): IO[Int] = + IO { report.getModel.listResourcesWithProperty(RDF.`type`,SH.Violation).toList.size } + + + private def report2reader(model: Model): IO[RDFReader] = for { + refModel <- Ref.of[IO, Model](model) + } yield RDFAsJenaModel(refModel,None,None) + + + private def report2errors(): Seq[ErrorInfo] = Seq() + + private def report2ShapesMap(): ResultShapeMap = { + ResultShapeMap.empty + } + + override def fromString(cs: CharSequence, + format: String, + base: Option[String] + ): IO[Schema] = for { + model <- IO { + val m : Model = ModelFactory.createDefaultModel() + val str_reader = new StringReader(cs.toString) + val g: Graph = m.getGraph + val dest: StreamRDF = StreamRDFLib.graph(g) + RDFParser.create.source(str_reader).lang(RDFLanguages.shortnameToLang(format)).parse(dest) + m + } + } yield ShaclTQ(model) + + // private def err[A](msg:String): EitherT[IO,String, A] = EitherT.leftT[IO,A](msg) + + override def fromRDF(rdf: RDFReader): IO[es.weso.schema.Schema] = rdf match { + case rdfJena: RDFAsJenaModel => for { + _ <- IO { println(s"SHACL_TQ: Parsing Shapes graph from RDF data")} + model <- rdfJena.getModel + str <- rdfJena.serialize("TURTLE") + _ <- IO { println(s"RDF to parse:\n${str}")} + } yield ShaclTQ(model) + case _ => IO.raiseError(ShaclTQException(s"Cannot obtain ${name} from RDFReader ${rdf.rdfReaderName} yet")) + } + + override def serialize(format: String, base: Option[IRI]): IO[String] = + if (formats.contains(format.toUpperCase)) IO { + val out = new ByteArrayOutputStream() + val relativizedModel = JenaUtils.relativizeModel(shapesGraph, base.map(_.uri)) + relativizedModel.write(out, format) + out.toString + } + else + IO.raiseError(ShaclTQException(s"Format $format not supported to serialize $name. Supported formats=$formats")) + + override def empty: Schema = ShaclTQ.empty + + override def shapes: List[String] = { + List() + } + + override def pm: PrefixMap = prefixMapFromModel(shapesGraph) + + override def convert(targetFormat: Option[String], + targetEngine: Option[String], + base: Option[IRI] + ): IO[String] = { + targetEngine.map(_.toUpperCase) match { + case None => serialize(targetFormat.getOrElse(DataFormats.defaultFormatName)) + case Some("SHACL") | Some("SHACLEX") => + serialize(targetFormat.getOrElse(DataFormats.defaultFormatName)) + case Some("SHEX") => + IO.raiseError(ShaclTQException(s"Not implemented conversion between ${name} to ShEx yet")) + case Some(other) => + IO.raiseError(ShaclTQException(s"Conversion $name -> $other not implemented yet")) + } + } + + override def info: SchemaInfo = { + // TODO: Check if shacl schemas are well formed + SchemaInfo(name,"SHACLex", isWellFormed = true, List()) + } + + override def toClingo(rdf: RDFReader, shapeMap: ShapeMap): IO[String] = + IO.raiseError(ShaclTQException(s"Not implemented yet toClingo for $name")) + +} + +object ShaclTQ { + def empty: ShaclTQ = { + val m = ModelFactory.createDefaultModel() + ShaclTQ(m) + } + +} diff --git a/modules/schema/src/main/scala/es/weso/schema/ShaclexSchema.scala b/modules/schema/src/main/scala/es/weso/schema/ShaclexSchema.scala index 0bcaecff..ae04dbab 100644 --- a/modules/schema/src/main/scala/es/weso/schema/ShaclexSchema.scala +++ b/modules/schema/src/main/scala/es/weso/schema/ShaclexSchema.scala @@ -164,30 +164,30 @@ case class ShaclexSchema(schema: ShaclSchema) extends Schema { override def pm: PrefixMap = schema.pm - override def convert(targetFormat: Option[String], - targetEngine: Option[String], + override def convert(maybeTargetFormat: Option[String], + maybeTargetEngine: Option[String], base: Option[IRI] ): IO[String] = { - targetEngine.map(_.toUpperCase) match { - case None => serialize(targetFormat.getOrElse(DataFormats.defaultFormatName)) + val targetFormat = maybeTargetFormat.getOrElse(DataFormats.defaultFormatName) + for { + str <- maybeTargetEngine.map(_.toUpperCase) match { + case None => + serialize(targetFormat) case Some("SHACL") | Some("SHACLEX") => - serialize(targetFormat.getOrElse(DataFormats.defaultFormatName)) + serialize(targetFormat) case Some("SHEX") => RDFAsJenaModel.empty.flatMap(_.use(builder => for { pair <- Shacl2ShEx.shacl2ShEx(schema).fold( s => IO.raiseError(new RuntimeException(s"SHACL2ShEx: Error converting: $s")), IO.pure ) (newSchema,_) = pair - str <- es.weso.shex.Schema.serialize( - newSchema, - targetFormat.getOrElse(DataFormats.defaultFormatName), - base, - builder) - } yield str)) + str <- es.weso.shex.Schema.serialize(newSchema,targetFormat,base,builder) + } yield (str))) case Some(other) => IO.raiseError(new RuntimeException(s"Conversion $name -> $other not implemented yet")) - } - } + } + } yield str + } override def info: SchemaInfo = { // TODO: Check if shacl schemas are well formed diff --git a/modules/schema/src/test/scala/es/weso/schema/SchemaTest.scala b/modules/schema/src/test/scala/es/weso/schema/SchemaTest.scala index ba57e9b1..2c20a0a0 100644 --- a/modules/schema/src/test/scala/es/weso/schema/SchemaTest.scala +++ b/modules/schema/src/test/scala/es/weso/schema/SchemaTest.scala @@ -48,7 +48,6 @@ class SchemaTest extends AnyFunSpec with Matchers with EitherValues { tryResult.attempt.unsafeRunSync match { case Right(result) => - info(s"Result: ${result.serialize(Result.TEXT)}") info(s"Result solution: ${result.solution}") result.isValid should be(true) result.hasShapes(node) should contain only shape diff --git a/version.sbt b/version.sbt index 05d2f358..950680e6 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.1.69" \ No newline at end of file +version in ThisBuild := "0.1.70" \ No newline at end of file