split cityData_server_scala into 3 projects: core, Jena, BigData.

Now cityData_server_scala_jena/ is the SBT project to run.

README.md

@@ -12,6 +12,11 @@ The process retrieves corresponding triples from the following sources ( tentati
 - [data.INSEE.fr](http://data.INSEE.fr) (for France)
 - [BANO](http://openstreetmap.fr/bano) (for France)
 
+Currently only dbPedia.org is used.
+There is a server based on PlayFramework backed by an Apache Jena TDB SPARQL database in this project: 
+[cityData\_server\_scala\_jena](cityData_server_scala_jena) .
+
+
  ## Panoply of tools
 - for semantic data transformation: Euler / EYE ; [introduction to the N3 language and a how to for the Euler/eye engine](http://svn.code.sf.net/p/eulergui/code/trunk/eulergui/html/N3_introduction.xhtml)
 - for serving RDF / JSON-LD data with SPARQL : [BigData(TM)](http://BigData.com)

cityData_server_scala_bigdata/README.md

@@ -0,0 +1,3 @@
+This is a project with BigData(R) , now named BlazeGraph, SPARQL database, used as embedded API.
+
+Due to a missing implementation in Banana-RDF, it is not yet working .

cityData_server_scala_bigdata/app/controllers/ApplicationBigData.scala

@@ -0,0 +1,37 @@
+package controllers
+
+import play.api._
+import play.api.mvc._
+import play.api.mvc.Request
+import play.api.libs.iteratee.Enumerator
+import play.api.libs.iteratee.Iteratee
+import scala.concurrent.Future
+import scala.concurrent.ExecutionContext.Implicits.global
+import models.REST_SPARQL_bridge
+import org.w3.banana.io.JsonLd
+import models.REST_SPARQL_bridge_Jena
+import models.REST_SPARQL_bridge_BigData
+
+object ApplicationBigData extends Controller with REST_SPARQL_bridge_BigData {
+
+  /**
+   * a simple wrapper with simple URL's for a SPARQL database.
+   *   For example for dbPedia it would accept URI's like:
+   *   /baseURL/cities/fra/01600
+   *   and maybe later:
+   *   /baseURL/Settlement?country=France&postalCode=01600
+   *   In fact we are only interested in a small number of classes and properties,
+   *   but it would be nice to have something as generic as possible.
+   *   
+   *  The entry point starts with a country and a postal code.
+   *  The process retrieves corresponding triples from the following sources:
+   *  - local (embedded) dbPedia instance
+   *  - hosted dbPedia instance at endpoint http://dbpedia.org/sparql - TODO
+   */
+  def displayCity(path: String) = {
+    Action { implicit request =>
+    Ok(getJSONLD(request)).
+     as("application/ld+json; charset=utf-8")
+    }
+  }
+}

cityData_server_scala_bigdata/app/models/REST_SPARQL_bridge_BigData.scala

@@ -0,0 +1,11 @@
+package models
+
+import play.api.mvc.Request
+import play.api.mvc.AnyContent
+import org.w3.banana.jena.Jena
+import com.hp.hpl.jena.query.Dataset
+import org.w3.banana.sesame.Sesame
+import org.openrdf.repository.RepositoryConnection
+
+trait REST_SPARQL_bridge_BigData extends REST_SPARQL_bridge[Sesame, RepositoryConnection]
+  with SPARQLDatabaseBigData

cityData_server_scala_bigdata/app/models/SPARQLDatabaseBigData.scala

@@ -0,0 +1,31 @@
+package models
+
+import org.w3.banana.SparqlOpsModule
+import org.w3.banana.sesame.Sesame
+import org.w3.banana.sesame.SesameModule
+import play.api.Logger
+import org.w3.banana.io.RDFWriter
+import org.w3.banana.io.JsonLdExpanded
+import org.w3.banana.io.JsonLdFlattened
+import com.bigdata.rdf.sail.remote.BigdataSailRemoteRepository
+import com.bigdata.rdf.sail.{BigdataSailTupleQuery, BigdataSailRepositoryConnection, BigdataSailRepository, BigdataSail}
+import java.io.File
+import java.util.Properties
+import org.apache.commons.io.FileUtils
+import scala.util.Try
+import org.openrdf.query.{TupleQueryResult, QueryLanguage}
+import org.openrdf.model.impl.URIImpl
+import org.openrdf.repository.RepositoryConnection
+import org.openrdf.query.Dataset
+
+trait SPARQLDatabaseBigData extends SPARQLDatabaseTrait[Sesame, RepositoryConnection] with SesameModule {
+  type DATASET =
+    RepositoryConnection
+  override lazy val dataset: DATASET = {
+    val db = BigData() // initializing the database
+    // TODO expose  write Connection
+    val rc = db.readConnection
+    Logger.info(s"SPARQLDatabaseTrait dataset created $rc")
+    rc
+  }
+}

cityData_server_scala_bigdata/build.properties

@@ -0,0 +1 @@
+sbt.version=0.13.8

cityData_server_scala_bigdata/build.sbt

@@ -0,0 +1,37 @@
+organization := "pixelhumain"
+
+name := "semantic_rest_bigdata"
+
+version := "1.0-SNAPSHOT"
+
+lazy val semantic_rest_core =  RootProject(file("../semantic_rest_core"))
+
+lazy val semantic_rest = (project in file("."))
+	.enablePlugins(PlayScala)
+        .dependsOn(semantic_rest_core)
+
+scalaVersion := "2.11.6"
+
+
+libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.4" % Test
+
+javacOptions ++= Seq("-source","1.7", "-target","1.7")
+
+resolvers += Resolver.url("inthenow-releases", url("http://dl.bintray.com/inthenow/releases"))(Resolver.ivyStylePatterns)
+
+// banana-rdf still has some dependencies that are not yet on Maven Central
+resolvers += Resolver.file("Local repo", file(System.getProperty("user.home") + "/.ivy2/local"))(Resolver.ivyStylePatterns)
+
+
+// BigData specific: the project is split in 3 : the implementation independent core, the Jena TDB project, the BD project
+libraryDependencies += "org.w3" %%  "banana-sesame" % "0.8.1"
+
+resolvers += "apache-repo-releases" at "http://repository.apache.org/content/repositories/releases/"   
+
+// For BigData database
+resolvers += "Bigdata releases" at "http://systap.com/maven/releases/"
+
+// For BigData
+resolvers += "nxparser-repo" at "http://nxparser.googlecode.com/svn/repository/"
+
+libraryDependencies += "com.bigdata" % "bigdata" % "1.5.1"

cityData_server_scala_bigdata/conf/routes

@@ -0,0 +1,5 @@
+# Routes
+# This file defines all application routes (Higher priority routes first)
+# ~~~~
+
+GET     /cities/*path		controllers.ApplicationBigData.displayCity(path: String)

cityData_server_scala_bigdata/project/plugins.sbt

@@ -0,0 +1,9 @@
+// Comment to get more information during initialization
+logLevel := Level.Warn
+
+// The Typesafe repository 
+resolvers += "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/"
+
+// Use the Play sbt plugin for Play projects
+addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.3.8")
+

cityData_server_scala_core/README.md

@@ -0,0 +1,5 @@
+# Introduction
+Here is an abstract web application with Play! framework that wraps a SPARQL database.
+It is a database independent source code, thanks to Banana-RDF.
+
+It is used by [../cityData\_server\_scala\_jena](../cityData_server_scala_jena) .

cityData_server_scala_core/app/controllers/AbstractApplication.scala

@@ -0,0 +1,37 @@
+package controllers
+
+import play.api._
+import play.api.mvc._
+import play.api.mvc.Request
+import play.api.libs.iteratee.Enumerator
+import play.api.libs.iteratee.Iteratee
+import scala.concurrent.Future
+import scala.concurrent.ExecutionContext.Implicits.global
+import models.REST_SPARQL_bridge
+import org.w3.banana.io.JsonLd
+import org.w3.banana.RDF
+
+/** independent of a concrete SPARQL database, thanks to Banana-RDF framework */
+trait AbstractApplication[Rdf <: RDF, DATASET] extends Controller with REST_SPARQL_bridge[Rdf, DATASET] {
+
+  /**
+   * a simple wrapper with simple URL's for a SPARQL database.
+   *   For example for dbPedia it would accept URI's like:
+   *   /baseURL/cities/fra/01600
+   *   and maybe later:
+   *   /baseURL/Settlement?country=France&postalCode=01600
+   *   In fact we are only interested in a small number of classes and properties,
+   *   but it would be nice to have something as generic as possible.
+   *   
+   *  The entry point starts with a country and a postal code.
+   *  The process retrieves corresponding triples from the following sources:
+   *  - local (embedded) dbPedia instance
+   *  - hosted dbPedia instance at endpoint http://dbpedia.org/sparql - TODO
+   */
+  def displayCity(path: String) = {
+    Action { implicit request =>
+    Ok(getJSONLD(request)).
+     as("application/ld+json; charset=utf-8")
+    }
+  }
+}

cityData_server_scala_core/app/models/RDFStoreLocalProvider.scala

@@ -0,0 +1,16 @@
+package models
+
+import scala.util.Try
+import org.w3.banana.RDF
+import org.w3.banana.RDFOpsModule
+import org.w3.banana.RDFStore
+import org.w3.banana.SparqlOpsModule
+
+/**
+ * abstract RDFStore Local Provider
+ * NOTE: same design pattern as for XXXModule in Banana
+ */
+trait RDFStoreLocalProvider[Rdf <: RDF, DATASET] extends RDFOpsModule with SparqlOpsModule {
+  implicit val rdfStore: RDFStore[Rdf, Try, DATASET]
+  val dataset: DATASET
+}

cityData_server_scala_core/app/models/REST_SPARQL_bridge.scala

@@ -0,0 +1,79 @@
+package models
+
+import play.api.mvc.Request
+import play.api.mvc.AnyContent
+import org.w3.banana.RDF
+
+trait REST_SPARQL_bridge[Rdf <: RDF, DATASET] extends SPARQLDatabaseTrait[Rdf, DATASET] {
+
+  val dbo = "http://dbpedia.org/ontology/"
+  val dbp = "http://dbpedia.org/resource/"
+
+  val queryBeginning = """
+    PREFIX dbpedia-owl: <http://dbpedia.org/ontology/>
+    PREFIX dbpedia:     <http://dbpedia.org/resource/>
+    CONSTRUCT {
+      ?CITY ?P ?O .
+    }
+    WHERE {
+      ?CITY a dbpedia-owl:Settlement ;
+    """
+  val queryEnding = """
+              ?P ?O .
+    }
+    """
+
+  /**
+   * For URI's like
+   *   /baseURL/cities/fra/01600
+   *
+   * create SPARQL query: <pre>
+    
+     PREFIX dbpedia-owl: <http://dbpedia.org/ontology/>
+     PREFIX dbpedia:     <http://dbpedia.org/resource/>
+     CONSTRUCT {
+       ?CITY ?P ?O .
+     }
+     WHERE {
+       ?CITY a dbpedia-owl:Settlement ;
+             dbpedia-owl:country dbpedia:France ;
+             dbpedia-owl:postalCode 01600 ;
+             ?P ?O .
+     }
+   </pre>
+   */
+  def getJSONLD(request: Request[AnyContent]): String = {
+    println("getJSONLD: " + request)
+    val conf = RequestRDFConfig(Seq(
+      new RequestRDFPathName(dbo + "country") {
+        val iso3166ToDBPedia = Map(
+          "fra" -> (dbp + "France"))
+        override def map(value: String): String = wrapAsURI(iso3166ToDBPedia.get(value).get)
+      },
+      RequestRDFPathName(dbo + "postalCode")))
+    runQuery(makeSPARQL(request, conf))
+  }
+
+  private def makeSPARQL(request: Request[AnyContent], conf: RequestRDFConfig): String = {
+    val path = request.path
+    val elems = path.split("/").drop(2)
+    // elems contains e.g.  "fr", "01600"
+    val propsAndValues = conf.mappings.zip(elems)
+    val criteria: Seq[String] = propsAndValues.map {
+      x =>
+        wrapAsURI(x._1.uri) + " " + x._1.map(x._2) + " ;\n"
+    }
+    val queryString = queryBeginning + criteria.mkString + queryEnding
+    println("makeSPARQL: " + queryString)
+    queryString
+  }
+
+  def wrapAsURI(uri: String) = s"<$uri>"
+  def wrapAsString(string: String) = s""""${string}""""
+
+  case class RequestRDFPathName(uri: String) {
+    def map(value: String): String = wrapAsString(value)
+  }
+  case class RequestRDFConfig(mappings: Seq[RequestRDFPathName])
+
+}

cityData_server_scala_core/app/models/SPARQLDatabaseTrait.scala

@@ -0,0 +1,29 @@
+package models
+
+import org.w3.banana.SparqlOpsModule
+import org.w3.banana.RDF
+import org.w3.banana.JsonLDWriterModule
+
+trait SPARQLDatabaseTrait[Rdf <: RDF, DATASET]
+extends RDFStoreLocalProvider[Rdf, DATASET]
+with JsonLDWriterModule {
+  import ops._
+  import rdfStore.transactorSyntax._
+  import rdfStore.graphStoreSyntax._
+  import sparqlOps._
+  import rdfStore.sparqlEngineSyntax._
+
+  def runQuery(queryString: String): String = {
+        val query =parseConstruct(queryString)
+
+    val res = dataset.r({
+      val result = for {
+        query <- parseConstruct(queryString)
+        graph <- dataset.executeConstruct(query, Map())
+      } yield graph
+      result
+    })
+    val graph = res.get.get
+    jsonldCompactedWriter.asString(graph, "" ).get
+  }
+}

cityData_server_scala_core/build.properties

@@ -0,0 +1 @@
+sbt.version=0.13.8

cityData_server_scala_core/build.sbt

@@ -0,0 +1,24 @@
+organization := "pixelhumain"
+
+name := "semantic_rest_core"
+
+version := "1.0-SNAPSHOT"
+
+lazy val semantic_rest = (project in file("."))
+	.enablePlugins(PlayScala)
+
+scalaVersion := "2.11.6"
+
+
+libraryDependencies += "org.w3" %%  "banana-rdf" % "0.8.1"
+
+libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.4" % Test
+
+javacOptions ++= Seq("-source","1.7", "-target","1.7")
+
+resolvers += Resolver.url("inthenow-releases", url("http://dl.bintray.com/inthenow/releases"))(Resolver.ivyStylePatterns)
+
+// banana-rdf still has some dependencies that are not yet on Maven Central
+resolvers += Resolver.file("Local repo", file(System.getProperty("user.home") + "/.ivy2/local"))(Resolver.ivyStylePatterns)
+
+