Merge c56d3c1 into 27bc662

sansa-rdf-common/pom.xml

@@ -48,6 +48,11 @@
 			<artifactId>scala-logging_${scala.binary.version}</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>de.javakaffee</groupId>
+			<artifactId>kryo-serializers</artifactId>
+		</dependency>
+
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>
@@ -59,6 +64,11 @@
 			<artifactId>specs</artifactId>
 			<scope>test</scope>
 		</dependency>
+
+		<dependency>
+			<groupId>com.typesafe</groupId>
+			<artifactId>config</artifactId>
+		</dependency>
 	</dependencies>
 
 	<build>

sansa-rdf-common/src/main/resources/metrics.conf

@@ -0,0 +1,10 @@
+# This configuration file contains the settings for the assessment.
+rdf.qualityassessment.dataset.prefixes=["http://dbpedia.org/","http://ex.org/foo/"]
+
+rdf.qualityassessment.dataset.subject="http://commons.dbpedia.org/resource/Category:Events"
+rdf.qualityassessment.dataset.property="http://commons.dbpedia.org/property/author"
+
+rdf.qualityassessment.dataset.lowerBound=0
+rdf.qualityassessment.dataset.upperBound=1
+
+rdf.qualityassessment.dataset.shortUri.threshold = 95

sansa-rdf-spark/src/main/scala/net/sansa_stack/rdf/spark/utils/kryo/jena/JenaKryoSerializers.scala → sansa-rdf-common/src/main/scala/net/sansa_stack/rdf/common/kryo/jena/JenaKryoSerializers.scala

@@ -1,4 +1,4 @@
-package net.sansa_stack.rdf.spark.utils.kryo.jena
+package net.sansa_stack.rdf.common.kryo.jena
 
 import com.esotericsoftware.kryo.{Kryo, Serializer}
 import com.esotericsoftware.kryo.io.{Input, Output}

sansa-rdf-spark/src/main/scala/net/sansa_stack/rdf/spark/qualityassessment/utils/DatasetUtils.scala → sansa-rdf-common/src/main/scala/net/sansa_stack/rdf/common/qualityassessment/utils/DatasetUtils.scala

@@ -1,4 +1,4 @@
-package net.sansa_stack.rdf.spark.qualityassessment.utils
+package net.sansa_stack.rdf.common.qualityassessment.utils
 
 import collection.JavaConverters._
 import com.typesafe.config.{ Config, ConfigFactory }

sansa-rdf-spark/src/main/scala/net/sansa_stack/rdf/spark/qualityassessment/utils/NodeUtils.scala → sansa-rdf-common/src/main/scala/net/sansa_stack/rdf/common/qualityassessment/utils/NodeUtils.scala

@@ -1,16 +1,18 @@
-package net.sansa_stack.rdf.spark.qualityassessment.utils
+package net.sansa_stack.rdf.common.qualityassessment.utils
 
 import java.io.IOException
 import java.net.{ HttpURLConnection, MalformedURLException, ProtocolException }
 import java.net.URL
 
-import net.sansa_stack.rdf.spark.qualityassessment.utils.DatasetUtils._
-import net.sansa_stack.rdf.spark.qualityassessment.vocabularies.DQV
-import net.sansa_stack.rdf.spark.utils.StatsPrefixes._
-import org.apache.jena.graph.{ Node, Triple }
-import org.apache.jena.vocabulary.RDFS
+import scala.collection.Seq
 import scala.util.matching.Regex
 
+import net.sansa_stack.rdf.common.qualityassessment.utils.DatasetUtils._
+import net.sansa_stack.rdf.common.qualityassessment.utils.vocabularies.DQV
+import org.apache.jena.graph.{ Node, Triple }
+import org.apache.jena.vocabulary.{OWL, RDFS }
+
+
 /**
  * Node Utils.
  * @author Gezim Sejdiu
@@ -144,14 +146,15 @@ object NodeUtils extends Serializable {
 
   def checkLiteral(node: Node): String =
     if (node.isLiteral) node.getLiteralLexicalForm else node.toString()
+
   def isLabeled(node: Node): Boolean =
-    (if (node.isLiteral) node.getLiteralLexicalForm else node.toString).contains(RDFS_LABEL)
+    (if (node.isLiteral) node.getLiteralLexicalForm else node.toString).contains(RDFS.label)
 
   def isRDFSClass(node: Node): Boolean =
-    (if (node.isLiteral) node.getLiteralLexicalForm else node.toString).contains(RDFS_CLASS)
+    (if (node.isLiteral) node.getLiteralLexicalForm else node.toString).contains(RDFS.Class)
 
   def isOWLClass(node: Node): Boolean =
-    (if (node.isLiteral) node.getLiteralLexicalForm else node.toString).contains(OWL_CLASS)
+    (if (node.isLiteral) node.getLiteralLexicalForm else node.toString).contains(OWL.Class)
 
   def resourceTooLong(node: Node): Boolean =
     (node.getURI().length() >= shortURIThreshold)

sansa-rdf-spark/src/main/scala/net/sansa_stack/rdf/spark/qualityassessment/vocabularies/DQV.scala → sansa-rdf-common/src/main/scala/net/sansa_stack/rdf/common/qualityassessment/utils/vocabularies/DQV.scala

@@ -1,4 +1,4 @@
-package net.sansa_stack.rdf.spark.qualityassessment.vocabularies
+package net.sansa_stack.rdf.common.qualityassessment.utils.vocabularies
 
 /**
  * Data Quality Vocabulary

sansa-rdf-common/src/test/scala/net/sansa_stack/rdf/common/partition/core/RdfPartitionDefaultTests.scala

@@ -1,51 +1,31 @@
 package net.sansa_stack.rdf.common.partition.core
 
+import net.sansa_stack.rdf.common.partition.schema.SchemaStringStringLang
 import org.apache.jena.graph.{ Node, NodeFactory, Triple }
 import org.scalatest.FunSuite
 
-import net.sansa_stack.rdf.common.partition.core.RdfPartitionerDefault._
-import net.sansa_stack.rdf.common.partition.schema.{ SchemaStringString, SchemaStringStringLang }
 
 /**
  * @author Gezim Sejdiu
  */
-class RdfPartitionerDefaultTests extends FunSuite {
+class RdfPartitionDefaultTests extends FunSuite {
 
   val triple = Triple.create(
     NodeFactory.createURI("http://dbpedia.org/resource/Guy_de_Maupassant"),
     NodeFactory.createURI("http://xmlns.com/foaf/0.1/givenName"),
     NodeFactory.createLiteral("Guy De"))
 
-  test("getting URI or BNode string should match") {
-    val node = NodeFactory.createURI("http://dbpedia.org/resource/Guy_de_Maupassant")
-    assert(getUriOrBNodeString(triple.getSubject).matches(node.getURI))
-  }
-
-  test("getting RDF Term type should match") {
-    assert(getRdfTermType(triple.getSubject) == 1)
-  }
-
-  test("chechking if data type is PlainLiteral should match") {
-    assert(isPlainLiteralDatatype(triple.getObject.getLiteralDatatypeURI))
-  }
+  val partition = new RdfPartitionDefault(1, "http://xmlns.com/foaf/0.1/givenName",
+    2, "http://www.w3.org/2001/XMLSchema#string", true)
 
-  test("chechking if [[Node]] is TypedLiteral should match") {
-    assert(!isTypedLiteral(triple.getObject))
-  }
-
-  test("getting partitioning layout from [[Triple]] should match") {
-    val expectedPartition = new RdfPartitionDefault(1, "http://xmlns.com/foaf/0.1/givenName",
-      2, "http://www.w3.org/2001/XMLSchema#string", true)
-    assert(fromTriple(triple).equals(expectedPartition))
-  }
-
-  test("determining Layout should match") {
+  test("getting layout should match") {
     val expectedLayout = new SchemaStringStringLang("http://dbpedia.org/resource/Guy_de_Maupassant", "Guy De", "")
-    assert(determineLayout(fromTriple(triple)).fromTriple(triple).equals(expectedLayout))
+    assert(partition.layout.fromTriple(triple).equals(expectedLayout))
   }
 
-  test("determining Layout Datatype should match") {
-    val expectedLayoutDatatype = new SchemaStringString("http://dbpedia.org/resource/Guy_de_Maupassant", "Guy De")
-    assert(determineLayoutDatatype(triple.getObject.getLiteralDatatypeURI).fromTriple(triple).equals(expectedLayoutDatatype))
+  test("getting partition layout should match") {
+    val expectedLayout = partition.matches(triple)
+    assert(partition.matches(triple).equals(expectedLayout))
   }
+
 }

sansa-rdf-common/src/test/scala/net/sansa_stack/rdf/common/partition/core/RdfPartitionerDefaultTests.scala

@@ -0,0 +1,51 @@
+package net.sansa_stack.rdf.common.partition.core
+
+import net.sansa_stack.rdf.common.partition.core.RdfPartitionerDefault._
+import net.sansa_stack.rdf.common.partition.schema.{ SchemaStringString, SchemaStringStringLang }
+import org.apache.jena.graph.{ Node, NodeFactory, Triple }
+import org.scalatest.FunSuite
+
+
+/**
+ * @author Gezim Sejdiu
+ */
+class RdfPartitionerDefaultTests extends FunSuite {
+
+  val triple = Triple.create(
+    NodeFactory.createURI("http://dbpedia.org/resource/Guy_de_Maupassant"),
+    NodeFactory.createURI("http://xmlns.com/foaf/0.1/givenName"),
+    NodeFactory.createLiteral("Guy De"))
+
+  test("getting URI or BNode string should match") {
+    val node = NodeFactory.createURI("http://dbpedia.org/resource/Guy_de_Maupassant")
+    assert(getUriOrBNodeString(triple.getSubject).matches(node.getURI))
+  }
+
+  test("getting RDF Term type should match") {
+    assert(getRdfTermType(triple.getSubject) == 1)
+  }
+
+  test("chechking if data type is PlainLiteral should match") {
+    assert(isPlainLiteralDatatype(triple.getObject.getLiteralDatatypeURI))
+  }
+
+  test("chechking if [[Node]] is TypedLiteral should match") {
+    assert(!isTypedLiteral(triple.getObject))
+  }
+
+  test("getting partitioning layout from [[Triple]] should match") {
+    val expectedPartition = new RdfPartitionDefault(1, "http://xmlns.com/foaf/0.1/givenName",
+      2, "http://www.w3.org/2001/XMLSchema#string", true)
+    assert(fromTriple(triple).equals(expectedPartition))
+  }
+
+  test("determining Layout should match") {
+    val expectedLayout = new SchemaStringStringLang("http://dbpedia.org/resource/Guy_de_Maupassant", "Guy De", "")
+    assert(determineLayout(fromTriple(triple)).fromTriple(triple).equals(expectedLayout))
+  }
+
+  test("determining Layout Datatype should match") {
+    val expectedLayoutDatatype = new SchemaStringString("http://dbpedia.org/resource/Guy_de_Maupassant", "Guy De")
+    assert(determineLayoutDatatype(triple.getObject.getLiteralDatatypeURI).fromTriple(triple).equals(expectedLayoutDatatype))
+  }
+}

sansa-rdf-common/src/test/scala/net/sansa_stack/rdf/common/partition/layout/TripleLayoutDoubleTests.scala

@@ -0,0 +1,25 @@
+package net.sansa_stack.rdf.common.partition.layout
+
+import net.sansa_stack.rdf.common.partition.layout.TripleLayoutDouble._
+import net.sansa_stack.rdf.common.partition.schema.SchemaStringDouble
+import org.apache.jena.datatypes.xsd.XSDDatatype
+import org.apache.jena.graph.{ Node, NodeFactory, Triple }
+import org.scalatest.FunSuite
+
+
+/**
+ * @author Gezim Sejdiu
+ */
+class TripleLayoutDoubleTests extends FunSuite {
+
+  val triple = Triple.create(
+    NodeFactory.createURI("http://dbpedia.org/resource/Germany"),
+    NodeFactory.createURI("http://dbpedia.org/ontology/populationTotal"),
+    NodeFactory.createLiteral("82175700", XSDDatatype.XSDdouble))
+
+  test("getting layout from triple should match") {
+    val expectedLayout = new SchemaStringDouble(triple.getSubject.getURI, triple.getObject.getLiteralValue.toString().toDouble)
+    assert(fromTriple(triple).equals(expectedLayout))
+  }
+
+}

sansa-rdf-common/src/test/scala/net/sansa_stack/rdf/common/partition/layout/TripleLayoutLongTests.scala

@@ -0,0 +1,24 @@
+package net.sansa_stack.rdf.common.partition.layout
+
+import net.sansa_stack.rdf.common.partition.layout.TripleLayoutLong._
+import net.sansa_stack.rdf.common.partition.schema.SchemaStringLong
+import org.apache.jena.datatypes.xsd.XSDDatatype
+import org.apache.jena.graph.{ Node, NodeFactory, Triple }
+import org.scalatest.FunSuite
+
+/**
+ * @author Gezim Sejdiu
+ */
+class TripleLayoutLongTests extends FunSuite {
+
+  val triple = Triple.create(
+    NodeFactory.createURI("http://dbpedia.org/resource/Germany"),
+    NodeFactory.createURI("http://dbpedia.org/ontology/populationTotal"),
+    NodeFactory.createLiteral("82175700", XSDDatatype.XSDlong))
+
+  test("getting layout from triple should match") {
+    val expectedLayout = new SchemaStringLong(triple.getSubject.getURI, triple.getObject.getLiteralValue.toString().toLong)
+    assert(fromTriple(triple).equals(expectedLayout))
+  }
+
+}

sansa-rdf-common/src/test/scala/net/sansa_stack/rdf/common/partition/layout/TripleLayoutStringDatatypeTests.scala

@@ -0,0 +1,24 @@
+package net.sansa_stack.rdf.common.partition.layout
+
+import net.sansa_stack.rdf.common.partition.layout.TripleLayoutStringDatatype._
+import net.sansa_stack.rdf.common.partition.schema.SchemaStringStringType
+import org.apache.jena.datatypes.xsd.XSDDatatype
+import org.apache.jena.graph.{ Node, NodeFactory, Triple }
+import org.scalatest.FunSuite
+
+/**
+ * @author Gezim Sejdiu
+ */
+class TripleLayoutStringDatatypeTests extends FunSuite {
+
+  val triple = Triple.create(
+    NodeFactory.createURI("http://dbpedia.org/resource/Germany"),
+    NodeFactory.createURI("http://dbpedia.org/ontology/populationTotal"),
+    NodeFactory.createLiteral("82175700", XSDDatatype.XSDlong))
+
+  test("getting layout from triple should match") {
+    val expectedLayout = new SchemaStringStringType(triple.getSubject.getURI, triple.getObject.getLiteralLexicalForm, triple.getObject.getLiteralDatatype.getURI)
+    assert(fromTriple(triple).equals(expectedLayout))
+  }
+
+}

sansa-rdf-common/src/test/scala/net/sansa_stack/rdf/common/partition/layout/TripleLayoutStringLangTests.scala

@@ -0,0 +1,25 @@
+package net.sansa_stack.rdf.common.partition.layout
+
+import net.sansa_stack.rdf.common.partition.layout.TripleLayoutStringLang._
+import net.sansa_stack.rdf.common.partition.schema.SchemaStringStringLang
+import org.apache.commons.codec.language.bm.Lang
+import org.apache.jena.datatypes.xsd.XSDDatatype
+import org.apache.jena.graph.{ Node, NodeFactory, Triple }
+import org.scalatest.FunSuite
+
+/**
+ * @author Gezim Sejdiu
+ */
+class TripleLayoutStringLangTests extends FunSuite {
+
+  val triple = Triple.create(
+    NodeFactory.createURI("http://dbpedia.org/resource/Germany"),
+    NodeFactory.createURI("http://dbpedia.org/ontology/populationTotal"),
+    NodeFactory.createLiteral("82175700", "en", XSDDatatype.XSDdouble))
+
+  test("getting layout from triple should match") {
+    val expectedLayout = new SchemaStringStringLang(triple.getSubject.getURI, triple.getObject.getLiteralLexicalForm, triple.getObject.getLiteralLanguage)
+    assert(fromTriple(triple).equals(expectedLayout))
+  }
+
+}

sansa-rdf-common/src/test/scala/net/sansa_stack/rdf/common/partition/layout/TripleLayoutStringTests.scala

@@ -0,0 +1,24 @@
+package net.sansa_stack.rdf.common.partition.layout
+
+import net.sansa_stack.rdf.common.partition.layout.TripleLayoutString._
+import net.sansa_stack.rdf.common.partition.schema.SchemaStringString
+import org.apache.jena.datatypes.xsd.XSDDatatype
+import org.apache.jena.graph.{ Node, NodeFactory, Triple }
+import org.scalatest.FunSuite
+
+/**
+ * @author Gezim Sejdiu
+ */
+class TripleLayoutStringTests extends FunSuite {
+
+  val triple = Triple.create(
+    NodeFactory.createURI("http://dbpedia.org/resource/Germany"),
+    NodeFactory.createURI("http://dbpedia.org/ontology/populationTotal"),
+    NodeFactory.createLiteral("82175700", XSDDatatype.XSDinteger))
+
+  test("getting layout from triple should match") {
+    val expectedLayout = new SchemaStringString(triple.getSubject.getURI, triple.getObject.getLiteralLexicalForm)
+    assert(fromTriple(triple).equals(expectedLayout))
+  }
+
+}

sansa-rdf-common/src/test/scala/net/sansa_stack/rdf/common/partition/model/sparqlify/SparqlifyUtils2Tests.scala

@@ -0,0 +1,26 @@
+package net.sansa_stack.rdf.common.partition.model.sparqlify
+
+import net.sansa_stack.rdf.common.partition.core.RdfPartitionerDefault
+import net.sansa_stack.rdf.common.partition.model.sparqlify.SparqlifyUtils2._
+import org.aksw.obda.jena.domain.impl.ViewDefinition
+import org.apache.jena.graph.{ Node, NodeFactory, Triple }
+import org.scalatest.FunSuite
+
+/**
+ * @author Gezim Sejdiu
+ */
+class SparqlifyUtils2Tests extends FunSuite {
+
+  val triple = Triple.create(
+    NodeFactory.createURI("http://dbpedia.org/resource/Germany"),
+    NodeFactory.createURI("http://dbpedia.org/ontology/populationTotal"),
+    NodeFactory.createLiteral("82175700"))
+
+  test("creating view definition should pass") {
+    val partitioner = RdfPartitionerDefault.fromTriple(triple)
+    val viewDefinition = createViewDefinition(partitioner)
+    val expectedViewDefinition = new ViewDefinition(viewDefinition.getName, viewDefinition.getConstructTemplate, viewDefinition.getVarDefinition, viewDefinition.getConstraints, viewDefinition.getLogicalTable)
+    assert(createViewDefinition(partitioner).equals(expectedViewDefinition))
+  }
+
+}

sansa-rdf-common/src/test/scala/net/sansa_stack/rdf/common/partition/utils/RdfTermTests.scala

@@ -0,0 +1,27 @@
+package net.sansa_stack.rdf.common.partition.utils
+
+import net.sansa_stack.rdf.common.partition.utils.RdfTerm._
+import org.apache.jena.graph.{ Node, NodeFactory, Triple }
+import org.scalatest.FunSuite
+
+/**
+ * @author Gezim Sejdiu
+ */
+class RdfTermTests extends FunSuite {
+
+  val triple = Triple.create(
+    NodeFactory.createURI("http://dbpedia.org/resource/Germany"),
+    NodeFactory.createURI("http://dbpedia.org/ontology/populationTotal"),
+    NodeFactory.createLiteral("82175700"))
+
+  test("converting rdf term to node should pass") {
+    val term = nodeToTerm(triple.getSubject)
+    assert(termToNode(term).equals(triple.getSubject))
+  }
+
+  test("converting node to rdf term should pass") {
+    val requiredRDFTerm = new RdfTerm(1, "http://dbpedia.org/resource/Germany", null, null)
+    assert(nodeToTerm(triple.getSubject).equals(requiredRDFTerm))
+  }
+
+}