Merge branch 'feature/datalake' of https://github.com/SANSA-Stack/SAN…

…SA-Query into feature/datalake
SANSA-Stack · Oct 19, 2018 · 8fed061 · 8fed061
2 parents a0bf6a5 + d918230
commit 8fed061
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 13 deletions.
diff --git a/sansa-query-spark/pom.xml b/sansa-query-spark/pom.xml
@@ -75,6 +75,13 @@
 			<scope>test</scope>
 		</dependency>
 
+		<dependency>
+			<groupId>io.netty</groupId>
+			<artifactId>netty-all</artifactId>
+			<version>4.1.17.Final</version>
+		</dependency>
+
+
 	</dependencies>
 
 	<build>

diff --git a/...rk/src/main/resources/config_csv-only.ttl → ...-spark/src/main/resources/config_csv-only b/...rk/src/main/resources/config_csv-only.ttl → ...-spark/src/main/resources/config_csv-only
@@ -7,7 +7,7 @@
 			"delimiter": ",",
 			"mode": "DROPMALFORMED"
 		},
-		"source": "/root/person.csv",
+		"source": "src/main/resources/Data/person.csv",
 		"entity": "Person"
 	}
 	,	{
@@ -17,7 +17,7 @@
 			"delimiter": ",",
 			"mode": "DROPMALFORMED"
 		},
-		"source": "/root/review.csv",
+		"source": "src/main/resources/Data/review.csv",
 		"entity": "Review"
 	}
 	,	{
@@ -27,7 +27,7 @@
 			"delimiter": ",",
 			"mode": "DROPMALFORMED"
 		},
-		"source": "/root/offer.csv",
+		"source": "src/main/resources/Data/offer.csv",
 		"entity": "Offer"
 	}
 	,	{
@@ -37,7 +37,7 @@
 			"delimiter": ",",
 			"mode": "DROPMALFORMED"
 		},
-		"source": "/root/product.csv",
+		"source": "src/main/resources/Data/product.csv",
 		"entity": "Product"
 	}
 	,	{
@@ -47,7 +47,7 @@
 			"delimiter": ",",
 			"mode": "DROPMALFORMED"
 		},
-		"source": "/root/producer.csv",
+		"source": "src/main/resources/Data/producer.csv",
 		"entity": "Producer"
 	}
 	],

diff --git a/sansa-query-spark/src/main/resources/mappings_csv-only.ttl b/sansa-query-spark/src/main/resources/mappings_csv-only.ttl
@@ -15,7 +15,7 @@
 
 <#ProducerMapping>
 	rml:logicalSource [
-		rml:source "/root/producer.csv";
+		rml:source "src/main/resources/Data/producer.csv";
 		nosql:store nosql:csv
 	];
 	rr:subjectMap [
@@ -55,7 +55,7 @@
 
 <#ReviewMapping>
 	rml:logicalSource [
-		rml:source "/root/review.csv";
+		rml:source "src/main/resources/Data/review.csv";
 		nosql:store nosql:csv
 	];
 	rr:subjectMap [
@@ -130,7 +130,7 @@
 
 <#PersonMapping>
 	rml:logicalSource [
-		rml:source "/root/person.csv";
+		rml:source "src/main/resources/Data/person.csv";
 		nosql:store nosql:csv
 	];
 	rr:subjectMap [
@@ -165,7 +165,7 @@
 
 <#OfferMapping>
 	rml:logicalSource [
-		rml:source "/root/offer.csv";
+		rml:source "src/main/resources/Data/offer.csv";
 		nosql:store nosql:csv
 	];
 	rr:subjectMap [
@@ -225,7 +225,7 @@
 
 <#ProductMapping>
 	rml:logicalSource [
-		rml:source "/root/product.csv";
+		rml:source "src/main/resources/Data/product.csv";
 		nosql:store nosql:csv
 	];
 	rr:subjectMap [

diff --git a/sansa-query-spark/src/main/scala/net/sansa_stack/query/spark/package.scala b/sansa-query-spark/src/main/scala/net/sansa_stack/query/spark/package.scala
@@ -102,13 +102,12 @@ package object query {
 
   }
 
-  implicit class DataLake(triples: DataFrame) extends Serializable {
+  implicit class DataLake(spark: SparkSession) extends Serializable {
 
-    val spark = SparkSession.builder().getOrCreate()
     /**
      * Querying a Data Lake.
      */
-    def sparql(sparqlQuery: String, mappingsFile: String, configFile: String): DataFrame = {
+    def sparqlDL(sparqlQuery: String, mappingsFile: String, configFile: String): DataFrame = {
       DataLakeEngine.run(sparqlQuery, mappingsFile, configFile, spark)
     }
   }

diff --git a/...-query-spark/src/test/scala/net/sansa_stack/query/spark/datalake/TestDataLakeEngine.scala b/...-query-spark/src/test/scala/net/sansa_stack/query/spark/datalake/TestDataLakeEngine.scala
@@ -0,0 +1,23 @@
+package net.sansa_stack.query.spark.datalake
+
+import com.holdenkarau.spark.testing.DataFrameSuiteBase
+import org.scalatest.FunSuite
+
+class TestDataLakeEngine extends FunSuite with DataFrameSuiteBase {
+
+  import net.sansa_stack.query.spark.query._
+
+  val configFile = getClass.getResource("/config_csv-only").getPath
+  val mappingsFile = getClass.getResource("/mappings_csv-only.ttl").getPath
+
+  test("running BSBM Q1 should result 23") {
+
+    val query = getClass.getResource("/queries/Q1.sparql").getPath
+    val result = spark.sparqlDL(query, mappingsFile, configFile)
+
+    val size = result.count()
+
+    assert(size == 23)
+  }
+
+}