Merge pull request #75 from inrupt/comunica-problem

Attempt at using Comunica

package.json

@@ -4,9 +4,12 @@
   "description": "A central component for Solid servers, handles Web Access Control and Linked Data Platform concerns.",
   "main": "dist/lib/core/app.js",
   "dependencies": {
+    "@comunica/actor-init-sparql": "^1.7.2",
+    "@comunica/actor-init-sparql-rdfjs": "^1.7.2",
     "@types/debug": "^4.1.4",
     "@types/events": "^3.0.0",
     "@types/jsonwebtoken": "^8.3.2",
+    "@types/n3": "^1.0.7",
     "@types/node": "^12.0.0",
     "@types/node-fetch": "^2.3.2",
     "@types/node-rsa": "^1.0.0",
@@ -16,6 +19,7 @@
     "debug": "^4.1.1",
     "jsonwebtoken": "^8.5.1",
     "mocha": "^6.1.4",
+    "n3": "^1.1.1",
     "node-fetch": "^2.5.0",
     "node-rsa": "^1.0.5",
     "rdf-ext": "^1.2.2",

src/lib/core/basicOperations.ts

@@ -55,16 +55,18 @@ async function readBlob (task: WacLdpTask, blob: Blob): Promise<WacLdpResponse>
   result.resourceData = await streamToObject(await blob.getData())
   // TODO: use RdfType enum here
   if (task.asJsonLd) {
-    const rdf = resourceDataToRdf(result.resourceData)
+    const rdf = await resourceDataToRdf(result.resourceData)
     result.resourceData = await rdfToResourceData(rdf, true)
   }
   if (task.sparqlQuery) {
-    debug('reading blob as rdf')
-    const rdf = resourceDataToRdf(result.resourceData)
+    debug('reading blob as rdf', result.resourceData)
+    const rdf = await resourceDataToRdf(result.resourceData)
+    rdf.forEach((quad: any) => { debug('quad', quad.toString()) })
+    debug('done here printing quads')
     debug('applying query', task.sparqlQuery)
-    await applyQuery(rdf, task.sparqlQuery)
-    debug('converting to requested representation')
-    result.resourceData = await rdfToResourceData(rdf, task.asJsonLd)
+    const body: string = await applyQuery(rdf, task.sparqlQuery)
+    debug('converting to requested representation', rdf)
+    result.resourceData = makeResourceData('application/sparql+json', body)
   }
   debug('result.resourceData set to ', result.resourceData)
   if (task.omitBody) {

src/lib/core/executeTask.ts

@@ -141,14 +141,14 @@ async function handleOperation (wacLdpTask: WacLdpTask, storage: BlobTree, appen
 }
 
 export async function executeTask (wacLdpTask: WacLdpTask, aud: string, storage: BlobTree, skipWac: boolean): Promise<WacLdpResponse> {
-  const webId = (wacLdpTask.bearerToken ? await determineWebId(wacLdpTask.bearerToken, aud) : undefined)
-  debug({ webId, path: wacLdpTask.path, isContainer: wacLdpTask.isContainer, origin: wacLdpTask.origin, wacLdpTaskType: wacLdpTask.wacLdpTaskType })
-
   // handle OPTIONS before checking WAC
   if (wacLdpTask.wacLdpTaskType === TaskType.getOptions) {
     return handleOptions(wacLdpTask)
   }
 
+  const webId = (wacLdpTask.bearerToken ? await determineWebId(wacLdpTask.bearerToken, aud) : undefined)
+  debug({ webId, path: wacLdpTask.path, isContainer: wacLdpTask.isContainer, origin: wacLdpTask.origin, wacLdpTaskType: wacLdpTask.wacLdpTaskType })
+
   // may throw if access is denied:
   const appendOnly = await determineAppendOnly(wacLdpTask, webId, storage, skipWac)
 

src/lib/rdf/ResourceDataUtils.ts

@@ -55,12 +55,17 @@ export function objectToStream (obj: any): any {
 }
 
 export async function streamToBuffer (stream: any): Promise<Buffer> {
+  debug(stream)
+  // debug(stream._readableState.buffer.head.data.toString())
   const bufs: Array<Buffer> = []
   return new Promise(resolve => {
     stream.on('data', function (d: Buffer) {
+      debug('got chunk', d)
       bufs.push(d)
     })
+    debug('data event added')
     stream.on('end', function () {
+      debug('got end')
       resolve(Buffer.concat(bufs))
     })
   })

src/lib/rdf/applyQuery.ts

@@ -1,11 +1,50 @@
-// import { newEngine } from '@comunica/actor-init-sparql'
+import rdf from 'rdf-ext'
+import Debug from 'debug'
+import { newEngine } from '@comunica/actor-init-sparql-rdfjs'
+import { Store } from 'n3'
+const debug = Debug('apply-query')
 
-// const myEngine = newEngine();
-export async function applyQuery (store: any, sparqlQuery: string) {
-  // const engine = newEngine()
-  // const result = await engine.query(sparqlQuery, {
+function removeLeadingQuestionMark (str: string) {
+  return str.substring(1)
+}
 
-  // })
-  // const parsedQuery = parser.parse(sparqlQuery)
-  // TODO: implement
+export async function applyQuery (dataset: any, sparqlQuery: string): Promise<string> {
+  const store = new Store()
+  dataset.forEach((quad: any) => {
+    debug('quad', quad.toString())
+    store.addQuad(quad)
+  })
+  const myEngine = newEngine()
+  const result: any = await myEngine.query(sparqlQuery,
+    { sources: [ { type: 'rdfjsSource', value: store } ] })
+  // const sparqlResult = await myEngine.resultToString(result, 'application/json')
+  // debug(sparqlResult)
+  const bindings = await new Promise((resolve) => {
+    const bindings: any = []
+    result.bindingsStream.on('end', () => {
+      resolve(bindings)
+    })
+    result.bindingsStream.on('data', (data: any) => {
+      const binding: any = {}
+      const obj = JSON.parse(JSON.stringify(data))
+      debug(obj)
+      for (const key in obj) {
+        binding[removeLeadingQuestionMark(key)] = {
+          type: obj[key].termType.toLowerCase(),
+          value: obj[key].value
+        }
+      }
+      bindings.push(binding)
+    })
+  })
+  return JSON.stringify({
+    head: {
+      vars: result.variables.map(removeLeadingQuestionMark)
+    },
+    results: {
+      ordered: false,
+      distinct: false,
+      bindings
+    }
+  })
 }

src/lib/rdf/mergeRdfSources.ts

@@ -9,8 +9,8 @@ import { rdfToResourceData } from './rdfToResourceData'
 
 const debug = Debug('mergeRdfSources')
 
-function readAndMerge (rdfSources: { [indexer: string]: ResourceData }): ReadableStream {
-  const dataset = rdf.dataset()
+async function readAndMerge (rdfSources: { [indexer: string]: ResourceData }): Promise<any> {
+  let dataset = rdf.dataset()
   debug('created dataset')
   dataset.forEach((quad: any) => { debug(quad.toString()) })
   for (let i in rdfSources) {
@@ -25,18 +25,19 @@ function readAndMerge (rdfSources: { [indexer: string]: ResourceData }): Readabl
     }
     const bodyStream = convert(Buffer.from(rdfSources[i].body))
     const quadStream = parser.import(bodyStream)
-    dataset.import(quadStream)
+    await dataset.import(quadStream)
     debug('after import', rdfSources[i].body)
     dataset.forEach((quad: any) => { debug(quad.toString()) })
+    debug('done listing quads', dataset)
   }
-  return dataset.toStream()
+  return dataset
 }
 
 export async function mergeRdfSources (rdfSources: { [indexer: string]: ResourceData }, asJsonLd: boolean) {
-  const dataset = readAndMerge(rdfSources)
-  return rdfToResourceData(dataset, asJsonLd)
+  const datasetStream = (await readAndMerge(rdfSources)).toStream()
+  return rdfToResourceData(datasetStream, asJsonLd)
 }
 
-export function resourceDataToRdf (resourceData: ResourceData) {
+export function resourceDataToRdf (resourceData: ResourceData): Promise<any> {
   return readAndMerge({ resourceData })
 }

src/lib/rdf/rdfToResourceData.ts

@@ -6,7 +6,7 @@ import rdf from 'rdf-ext'
 import { Member } from '../storage/Container'
 
 const formats = Formats()
-const debug = Debug('membersListAsResourceData')
+const debug = Debug('rdfToResourceData')
 
 function toFormat (input: ReadableStream, contentType: string): Promise<string> {
   const serializer = formats.serializers[ contentType ]

src/server.ts

@@ -1,3 +1,19 @@
+// START VSCode debugging settings, see https://github.com/visionmedia/debug/issues/641#issuecomment-490706752
+declare global {
+	namespace NodeJS {
+    interface Process {
+      browser: boolean
+    }
+    interface Global {
+      window: object
+    }
+  }
+}
+process.browser = true
+global.window = { process: { type: 'renderer' } }
+process.env.DEBUG = '*'
+// END VSCode debugging settings
+
 import * as http from 'http'
 import Debug from 'debug'
 import { BlobTreeInMem } from './lib/storage/BlobTreeInMem'

test/fixtures/get-query.sparql

@@ -1,5 +1 @@
-PREFIX foaf:  <http://xmlns.com/foaf/0.1/>
-SELECT ?name
-WHERE {
-  ?person foaf:name ?name .
-}
+SELECT ?name { ?person <http://xmlns.com/foaf/0.1/name> ?name. ?person ?z ?name } LIMIT 100

test/fixtures/ldpRs1-2-merge.ttl

@@ -4,3 +4,4 @@
 <http://example.org/#spiderman> <http://www.perceive.net/schemas/relationship/enemyOf> <http://example.org/#green-goblin> .
 <http://example.org/#spiderman> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://xmlns.com/foaf/0.1/Person> .
 <http://example.org/#spiderman> <http://xmlns.com/foaf/0.1/name> "Spiderman" .
+<http://example.org/#spiderman> <http://xmlns.com/foaf/0.1/name> "Человек-паук"@ru .

test/integration/glob-read.test.ts

@@ -47,7 +47,7 @@ test('handles a GET /* request (glob read)', async () => {
         'Accept-Post': 'application/sparql-update',
         'Allow': 'GET, HEAD, POST, PUT, DELETE, PATCH',
         'Content-Type': 'text/turtle',
-        'ETag': '"v/I8yBrDQliUoJxmAw54pw=="',
+        'ETag': '"TmBqjXO24ygE+uQdtQuiOA=="',
         'Link': '<.acl>; rel="acl", <.meta>; rel="describedBy", <http://www.w3.org/ns/ldp#Resource>; rel="type", <http://www.w3.org/ns/ldp#BasicContainer>; rel="type"'
       }
     ]

test/integration/sparql-query-get.test.ts

@@ -18,7 +18,7 @@ beforeEach(async () => {
 
 const handler = makeHandler(storage, 'http://localhost:8080', false)
 
-test.skip('handles a SPARQL query in the GET query parameter', async () => {
+test('handles a SPARQL query in the GET query parameter', async () => {
   const sparqlQuery = fs.readFileSync('test/fixtures/get-query.sparql').toString()
 
   let streamed = false
@@ -33,19 +33,35 @@ test.skip('handles a SPARQL query in the GET query parameter', async () => {
     end: jest.fn(() => { }) // tslint:disable-line: no-empty
   }
   await handler(httpReq, httpRes as unknown as http.ServerResponse)
+  expect(httpRes.end.mock.calls).toEqual([
+    [
+      JSON.stringify({
+        head: {
+          vars: [ 'name' ]
+        },
+        results: {
+          ordered: false,
+          distinct: false,
+          bindings: [
+            {
+              name: { 'type': 'literal', 'value': 'Green Goblin' }
+            }
+          ]
+        }
+      })
+    ]
+  ])
   expect(httpRes.writeHead.mock.calls).toEqual([
     [
       200,
       {
         'Accept-Patch': 'application/sparql-update',
         'Accept-Post': 'application/sparql-update',
         'Allow': 'GET, HEAD, POST, PUT, DELETE, PATCH',
-        'Content-Type': 'text/plain',
+        'Content-Type': 'application/sparql+json',
+        'ETag': '"fTeBCZUGRxPpeUUf4DpHFg=="',
         'Link': '<.acl>; rel="acl", <.meta>; rel="describedBy", <http://www.w3.org/ns/ldp#Resource>; rel="type"'
       }
     ]
   ])
-  expect(httpRes.end.mock.calls).toEqual([
-    [ '' ]
-  ])
 })

tsconfig.json

@@ -15,6 +15,11 @@
     "esModuleInterop": true
   },
   "typeRoots": ["./types"],
+  "typedef": [
+    true,
+    "call-signature",
+    "property-declaration"
+  ],
   "exclude": [
     "test",
     "node_modules",