Merge 968dfc8 into 7ec9654

src/exports.ts

@@ -2,4 +2,5 @@ export { WacLdp, BEARER_PARAM_NAME } from './lib/core/WacLdp'
 export { determineWebIdAndOrigin } from './lib/auth/determineWebIdAndOrigin'
 export { BlobTree, Path } from './lib/storage/BlobTree'
 export { BlobTreeInMem } from './lib/storage/BlobTreeInMem'
+export { QuadAndBlobStore } from './lib/storage/QuadAndBlobStore'
 export { ACL } from './lib/rdf/rdf-constants'

src/lib/core/WacLdp.ts

@@ -1,6 +1,6 @@
 import * as http from 'http'
 import Debug from 'debug'
-import { BlobTree } from '../storage/BlobTree'
+import { QuadAndBlobStore } from '../storage/QuadAndBlobStore'
 import { WacLdpTask } from '../api/http/HttpParser'
 import { sendHttpResponse, WacLdpResponse, ErrorResult, ResultType } from '../api/http/HttpResponder'
 import { optionsHandler } from '../operationHandlers/optionsHandler'
@@ -19,6 +19,7 @@ import { unknownOperationCatchAll } from '../operationHandlers/unknownOperationC
 import { checkAccess, determineRequiredAccessModes, AccessCheckTask } from './checkAccess'
 import { getAppModes } from '../auth/appIsTrustedForMode'
 import { setAppModes } from '../rdf/setAppModes'
+import { BlobTree } from '../storage/BlobTree'
 
 export const BEARER_PARAM_NAME = 'bearer_token'
 
@@ -44,7 +45,7 @@ export class WacLdp extends EventEmitter {
   operationHandlers: Array<OperationHandler>
   idpHost: string
   usesHttps: boolean
-  constructor (storage: BlobTree, aud: string, updatesViaUrl: URL, skipWac: boolean, idpHost: string, usesHttps: boolean) {
+  constructor (storage: QuadAndBlobStore, aud: string, updatesViaUrl: URL, skipWac: boolean, idpHost: string, usesHttps: boolean) {
     super()
     this.rdfLayer = new CachingRdfLayer(aud, storage)
     this.aud = aud
@@ -94,9 +95,6 @@ export class WacLdp extends EventEmitter {
     }
     throw new ErrorResult(ResultType.InternalServerError)
   }
-  async containerExists (url: URL): Promise<boolean> {
-    return this.rdfLayer.localContainerExists(url)
-  }
 
   async handler (httpReq: http.IncomingMessage, httpRes: http.ServerResponse): Promise<void> {
     debug(`\n\n`, httpReq.method, httpReq.url, httpReq.headers)
@@ -171,7 +169,8 @@ export class WacLdp extends EventEmitter {
   }
 }
 
-export function makeHandler (storage: BlobTree, aud: string, updatesViaUrl: URL, skipWac: boolean, idpHost: string, usesHttps: boolean) {
+export function makeHandler (blobTree: BlobTree, aud: string, updatesViaUrl: URL, skipWac: boolean, idpHost: string, usesHttps: boolean) {
+  const storage = new QuadAndBlobStore(blobTree)
   const wacLdp = new WacLdp(storage, aud, updatesViaUrl, skipWac, idpHost, usesHttps)
   return wacLdp.handler.bind(wacLdp)
 }

src/lib/rdf/CachingRdfLayer.ts

@@ -1,6 +1,6 @@
 import { RdfLayer } from './RdfLayer'
 import { Blob } from '../storage/Blob'
-import { BlobTree } from '../storage/BlobTree'
+import { QuadAndBlobStore } from '../storage/QuadAndBlobStore'
 import * as rdflib from 'rdflib'
 import Debug from 'debug'
 import { ResourceData, streamToObject } from './ResourceDataUtils'
@@ -15,7 +15,7 @@ export class CachingRdfLayer extends RdfLayer {
   graphs: { [url: string]: any }
   stores: { [url: string]: any }
   blobs: { [url: string]: Blob }
-  constructor (serverHost: string, storage: BlobTree) {
+  constructor (serverHost: string, storage: QuadAndBlobStore) {
     super(serverHost, storage)
     this.graphs = {}
     this.stores = {}

src/lib/rdf/RdfLayer.ts

@@ -6,12 +6,13 @@ import JsonLdParser from 'rdf-parser-jsonld'
 import convert from 'buffer-to-stream'
 import * as rdflib from 'rdflib'
 
-import { Path, BlobTree, urlToPath } from '../storage/BlobTree'
+import { Path, urlToPath } from '../storage/BlobTree'
 import { Blob } from '../storage/Blob'
 import { ResourceData, streamToObject, determineRdfType, RdfType, makeResourceData, objectToStream } from './ResourceDataUtils'
 import { Container } from '../storage/Container'
 import { setRootAcl, setPublicAcl } from './setRootAcl'
 import { ResultType, ErrorResult } from '../api/http/HttpResponder'
+import { QuadAndBlobStore } from '../storage/QuadAndBlobStore'
 
 const debug = Debug('RdfLayer')
 
@@ -42,7 +43,7 @@ function readRdf (rdfType: RdfType | undefined, bodyStream: ReadableStream) {
   return parser.import(bodyStream)
 }
 
-async function getGraphLocal (blob: Blob): Promise<any> {
+export async function quadStreamFromBlob (blob: Blob): Promise<any> {
   const stream = await blob.getData()
   debug('stream', typeof stream)
   let resourceData
@@ -56,13 +57,18 @@ async function getGraphLocal (blob: Blob): Promise<any> {
   const bodyStream = convert(Buffer.from(resourceData.body))
 
   const quadStream = readRdf(resourceData.rdfType, bodyStream)
+  return quadStream
+}
+
+async function getGraphLocal (blob: Blob): Promise<any> {
+  const quadStream = await quadStreamFromBlob(blob)
   return rdf.dataset().import(quadStream)
 }
 
 export class RdfLayer {
   serverRootDomain: string
-  storage: BlobTree
-  constructor (serverRootDomain: string, storage: BlobTree) {
+  storage: QuadAndBlobStore
+  constructor (serverRootDomain: string, storage: QuadAndBlobStore) {
     this.serverRootDomain = serverRootDomain
     this.storage = storage
   }
@@ -80,10 +86,6 @@ export class RdfLayer {
     const path: Path = urlToPath(url)
     return this.storage.getContainer(path)
   }
-  localContainerExists (url: URL): Promise<boolean> {
-    const path: Path = urlToPath(url)
-    return this.storage.getContainer(path).exists()
-  }
   async fetchGraph (url: URL) {
     if (url.host.endsWith(this.serverRootDomain)) {
       const blob: Blob = this.getLocalBlob(url)

src/lib/rdf/membersListAsResourceData.ts

@@ -7,7 +7,7 @@ import { LDP, RDF } from './rdf-constants'
 
 const debug = Debug('membersListAsResourceData')
 
-function toRdf (containerUrl: URL, membersList: Array<Member>): ReadableStream {
+export function membersListAsQuadStream (containerUrl: URL, membersList: Array<Member>): ReadableStream {
   const dataset = rdf.dataset()
   membersList.map(member => {
     dataset.add(rdf.quad(
@@ -37,6 +37,6 @@ function toRdf (containerUrl: URL, membersList: Array<Member>): ReadableStream {
 
 export async function membersListAsResourceData (containerUrl: URL, membersList: Array<Member>, rdfType: RdfType): Promise<ResourceData> {
   debug('membersListAsResourceData', containerUrl, membersList, rdfType)
-  const dataset = toRdf(containerUrl, membersList)
+  const dataset = membersListAsQuadStream(containerUrl, membersList)
   return rdfToResourceData(dataset, rdfType)
 }

src/lib/rdf/setAppModes.ts

@@ -3,12 +3,13 @@ import * as rdflib from 'rdflib'
 import { BlobTree, urlToPath } from '../storage/BlobTree'
 import { streamToObject, ResourceData, objectToStream, makeResourceData } from './ResourceDataUtils'
 import { ACL } from './rdf-constants'
+import { QuadAndBlobStore } from '../storage/QuadAndBlobStore'
 
 const debug = Debug('setAppModes')
 
 // FIXME: It's weird that setAppModes is in the RDF module, but getAppModes is in the auth module.
 
-export async function setAppModes (webId: URL, origin: string, modes: Array<URL>, storage: BlobTree): Promise<void> {
+export async function setAppModes (webId: URL, origin: string, modes: Array<URL>, storage: QuadAndBlobStore): Promise<void> {
   debug(`Registering app (${origin}) with accessModes ${modes.map(url => url.toString()).join(', ')} for webId ${webId.toString()}`)
   const blob = storage.getBlob(urlToPath(webId))
   const stream = await blob.getData()

src/lib/rdf/setRootAcl.ts

@@ -1,13 +1,14 @@
 import { makeResourceData, bufferToStream } from './ResourceDataUtils'
 import { ACL_SUFFIX } from './RdfLayer'
 import { BlobTree, urlToPath } from '../storage/BlobTree'
+import { QuadAndBlobStore } from '../storage/QuadAndBlobStore'
 
 // TODO: this should be a method on a CachingRdfLayer object,
 // which has access to a `this.storage` variable rather than
 // requiring that as an extra parameter. And it should also
 // use the CachingRdfLayer object's knowledge of how to map
 // URLs to storage paths
-export async function setRootAcl (storage: BlobTree, owner: URL, storageRoot: URL) {
+export async function setRootAcl (storage: QuadAndBlobStore, owner: URL, storageRoot: URL) {
   let rootString = storageRoot.toString()
   if (rootString.substr(-1) !== '/') {
     rootString += '/'
@@ -36,7 +37,7 @@ export async function setRootAcl (storage: BlobTree, owner: URL, storageRoot: UR
 // where this layer needs to know that an inbox is public-append
 // so that also feels like (part of) this function should be at
 // a higher level
-export async function setPublicAcl (storage: BlobTree, owner: URL, inboxUrl: URL, modeName: string) {
+export async function setPublicAcl (storage: QuadAndBlobStore, owner: URL, inboxUrl: URL, modeName: string) {
   let inboxUrlStr = inboxUrl.toString()
   if (inboxUrlStr.substr(-1) !== '/') {
     inboxUrlStr += '/'

src/lib/storage/QuadAndBlobStore.ts

@@ -0,0 +1,51 @@
+import Debug from 'debug'
+import { BlobTree, Path, urlToPath } from './BlobTree'
+import { Member } from './Container'
+import { membersListAsQuadStream } from '../rdf/membersListAsResourceData'
+import { quadStreamFromBlob } from '../rdf/RdfLayer'
+import { rdfToResourceData } from '../rdf/rdfToResourceData'
+import { RdfType, objectToStream } from '../rdf/ResourceDataUtils'
+
+const debug = Debug('quad-and-blob-store')
+
+export class QuadAndBlobStore {
+  storage: BlobTree
+  constructor (storage: BlobTree) {
+    this.storage = storage
+  }
+  getBlob (path: Path) {
+    return this.storage.getBlob(path)
+  }
+  getContainer (path: Path) {
+    return this.storage.getContainer(path)
+  }
+  async getQuadStream (url: URL, preferMinimalContainer?: boolean): Promise<any> {
+    const path = urlToPath(url)
+    if (path.isContainer) {
+      const container = this.storage.getContainer(path)
+      debug(container)
+      let membersList: Array<Member>
+      if (preferMinimalContainer) {
+        membersList = []
+      } else {
+        membersList = await container.getMembers()
+      }
+      debug(membersList)
+      return membersListAsQuadStream(url, membersList)
+    } else {
+      const blob = this.storage.getBlob(path)
+      const stream = await quadStreamFromBlob(blob)
+      return stream
+    }
+  }
+  async setQuadStream (url: URL, quadStream: any): Promise<void> {
+    const path = urlToPath(url)
+    if (path.isContainer) {
+      throw new Error('cannot set QuadStream on Container')
+    } else {
+      const blob = this.storage.getBlob(path)
+      const resourceData = rdfToResourceData(quadStream, RdfType.Turtle)
+      return blob.setData(objectToStream(resourceData))
+    }
+  }
+}

test/unit/auth/determineAllowedAgentsForModes.test.ts

@@ -5,6 +5,7 @@ import { determineAllowedAgentsForModes, ModesCheckTask } from '../../../src/lib
 import { RdfLayer } from '../../../src/lib/rdf/RdfLayer'
 import { BlobTreeInMem } from '../../../src/lib/storage/BlobTreeInMem'
 import { ACL } from '../../../src/lib/rdf/rdf-constants'
+import { QuadAndBlobStore } from '../../../src/lib/storage/QuadAndBlobStore'
 
 test('finds acl:accessTo modes', async () => {
   const bodyStream = fs.createReadStream('test/fixtures/aclDoc-from-NSS-1.ttl')
@@ -18,7 +19,7 @@ test('finds acl:accessTo modes', async () => {
     resourceIsTarget: true,
     contextUrl: new URL('https://example.com'),
     targetUrl: new URL('https://example.com'),
-    rdfLayer: new RdfLayer('https://example.com', new BlobTreeInMem())
+    rdfLayer: new RdfLayer('https://example.com', new QuadAndBlobStore(new BlobTreeInMem()))
   }
   const result = await determineAllowedAgentsForModes(task)
   expect(result).toEqual({
@@ -41,7 +42,7 @@ test('finds acl:default modes', async () => {
     contextUrl: new URL('/.acl', 'https://example.com/'),
     targetUrl: new URL('/', 'https://example.com/'),
     resourceIsTarget: true,
-    rdfLayer: new RdfLayer('https://example.com', new BlobTreeInMem())
+    rdfLayer: new RdfLayer('https://example.com', new QuadAndBlobStore(new BlobTreeInMem()))
   }
   const result = await determineAllowedAgentsForModes(task)
   expect(result).toEqual({
@@ -68,7 +69,7 @@ function testUrlFormat (format: string, target: string, resourceIsTarget: boolea
       resourceIsTarget,
       targetUrl: new URL(target),
       contextUrl: new URL(target + '.acl'),
-      rdfLayer: new RdfLayer('https://example.com', new BlobTreeInMem())
+      rdfLayer: new RdfLayer('https://example.com', new QuadAndBlobStore(new BlobTreeInMem()))
     }
     const result = await determineAllowedAgentsForModes(task)
     expect(result).toEqual({
@@ -105,7 +106,7 @@ test(`acl:default does not imply acl:accessTo`, async () => {
     resourceIsTarget: true,
     targetUrl: new URL('https://example.org/foo/'),
     contextUrl: new URL('https://example.org/foo/.acl'),
-    rdfLayer: new RdfLayer('https://example.com', new BlobTreeInMem())
+    rdfLayer: new RdfLayer('https://example.com', new QuadAndBlobStore(new BlobTreeInMem()))
   }
   const result = await determineAllowedAgentsForModes(task)
   expect(result).toEqual({

test/unit/auth/trustedApps.test.ts

@@ -5,8 +5,8 @@ import { appIsTrustedForMode, OriginCheckTask, getAppModes } from '../../../src/
 import { setAppModes } from '../../../src/lib/rdf/setAppModes'
 import { RdfLayer } from '../../../src/lib/rdf/RdfLayer'
 import { ACL } from '../../../src/lib/rdf/rdf-constants'
-import { BlobTree } from '../../../src/lib/storage/BlobTree'
 import { objectToStream, makeResourceData, streamToObject } from '../../../src/lib/rdf/ResourceDataUtils'
+import { QuadAndBlobStore } from '../../../src/lib/storage/QuadAndBlobStore'
 
 const OWNER_PROFILE_FIXTURE = 'test/fixtures/owner-profile.ttl'
 
@@ -75,7 +75,7 @@ test('setTrustedAppModes existing', async () => {
     new URL('http://www.w3.org/ns/auth/acl#Append'),
     new URL('http://www.w3.org/ns/auth/acl#Control')
   ]
-  await setAppModes(new URL('https://michielbdejong.com/profile/card#me'), 'https://pheyvaer.github.io', modes, storage as BlobTree)
+  await setAppModes(new URL('https://michielbdejong.com/profile/card#me'), 'https://pheyvaer.github.io', modes, storage as QuadAndBlobStore)
 
   expect(stored).toEqual({
     body: [
@@ -121,7 +121,7 @@ test('setTrustedAppModes new', async () => {
     new URL('http://www.w3.org/ns/auth/acl#Append'),
     new URL('http://www.w3.org/ns/auth/acl#Control')
   ]
-  await setAppModes(new URL('https://michielbdejong.com/profile/card#me'), 'https://other.com', modes, storage as BlobTree)
+  await setAppModes(new URL('https://michielbdejong.com/profile/card#me'), 'https://other.com', modes, storage as QuadAndBlobStore)
   expect(stored).toEqual({
     body: [
       '@prefix : <#>.',

test/unit/core/operations.test.ts

@@ -7,7 +7,7 @@ import { makeResourceData, RdfType } from '../../../src/lib/rdf/ResourceDataUtil
 import { Container } from '../../../src/lib/storage/Container'
 import { readContainerHandler } from '../../../src/lib/operationHandlers/readContainerHandler'
 import { RdfLayer } from '../../../src/lib/rdf/RdfLayer'
-import { BlobTree } from '../../../src/lib/storage/BlobTree'
+import { QuadAndBlobStore } from '../../../src/lib/storage/QuadAndBlobStore'
 import { deleteContainerHandler } from '../../../src/lib/operationHandlers/deleteContainerHandler'
 import { readBlobHandler } from '../../../src/lib/operationHandlers/readBlobHandler'
 import { deleteBlobHandler } from '../../../src/lib/operationHandlers/deleteBlobHandler'
@@ -30,7 +30,7 @@ test('delete blob', async () => {
     method: 'DELETE',
     headers: {}
   } as http.IncomingMessage, true)
-  const rdfLayer = new RdfLayer('https://example.com', storage as BlobTree)
+  const rdfLayer = new RdfLayer('https://example.com', storage as QuadAndBlobStore)
   const result: WacLdpResponse = await deleteBlobHandler.handle(task, rdfLayer, 'https://example.com', false, false)
   expect((node as any).delete.mock.calls).toEqual([
     []
@@ -92,7 +92,7 @@ test('delete container', async () => {
     method: 'GET',
     headers: {}
   } as http.IncomingMessage, true)
-  const rdfLayer = new RdfLayer('https://example.com', storage as BlobTree)
+  const rdfLayer = new RdfLayer('https://example.com', storage as QuadAndBlobStore)
   const result: WacLdpResponse = await deleteContainerHandler.handle(task, rdfLayer, 'https://example.com', false, false)
   expect((node as any).delete.mock.calls).toEqual([
     []
@@ -117,7 +117,7 @@ test('read blob (omit body)', async () => {
     url: '/foo',
     method: 'HEAD'
   } as http.IncomingMessage, true)
-  const rdfLayer = new RdfLayer('https://example.com', storage as BlobTree)
+  const rdfLayer = new RdfLayer('https://example.com', storage as QuadAndBlobStore)
   const result: WacLdpResponse = await readBlobHandler.handle(task, rdfLayer, 'https://example.com', false, false)
   // FIXME: Why does it call getData twice?
   expect((node as any).getData.mock.calls).toEqual([
@@ -148,7 +148,7 @@ test('read blob (with body)', async () => {
     url: '/foo',
     method: 'GET'
   } as http.IncomingMessage, true)
-  const rdfLayer = new RdfLayer('https://example.com', storage as BlobTree)
+  const rdfLayer = new RdfLayer('https://example.com', storage as QuadAndBlobStore)
   const result: WacLdpResponse = await readBlobHandler.handle(task, rdfLayer, 'https://example.com', false, false)
   // FIXME: Why does it call getData twice?
   expect((node as any).getData.mock.calls).toEqual([
@@ -180,7 +180,7 @@ test('read container (omit body)', async () => {
     method: 'HEAD',
     headers: {}
   } as http.IncomingMessage, true)
-  const rdfLayer = new RdfLayer('https://example.com', storage as BlobTree)
+  const rdfLayer = new RdfLayer('https://example.com', storage as QuadAndBlobStore)
   const result: WacLdpResponse = await readContainerHandler.handle(task, rdfLayer, 'https://example.com', false, false)
   expect((node as any).getMembers.mock.calls).toEqual([
     []
@@ -217,7 +217,7 @@ test('read container (with body)', async () => {
     method: 'GET',
     headers: {}
   } as http.IncomingMessage, true)
-  const rdfLayer = new RdfLayer('https://example.com', storage as BlobTree)
+  const rdfLayer = new RdfLayer('https://example.com', storage as QuadAndBlobStore)
   const result: WacLdpResponse = await readContainerHandler.handle(task, rdfLayer, 'https://example.com', false, false)
   expect((node as any).getMembers.mock.calls).toEqual([
     []

test/unit/rdf/rdfLayer-fetchGraph.test.ts

@@ -3,9 +3,10 @@ import { BlobTreeInMem } from '../../../src/lib/storage/BlobTreeInMem'
 import { urlToPath } from '../../../src/lib/storage/BlobTree'
 import * as fs from 'fs'
 import { makeResourceData, objectToStream } from '../../../src/lib/rdf/ResourceDataUtils'
+import { QuadAndBlobStore } from '../../../src/lib/storage/QuadAndBlobStore'
 
 test('can fetch a local graph', async () => {
-  const storage = new BlobTreeInMem()
+  const storage = new QuadAndBlobStore(new BlobTreeInMem())
   const blob = storage.getBlob(urlToPath(new URL('https://example.com/profile/card')))
   const body: Buffer = await new Promise(resolve => fs.readFile('./test/fixtures/profile-card.ttl', (err, data) => {
     if (err) throw new Error('failed to read fixture')
@@ -19,7 +20,7 @@ test('can fetch a local graph', async () => {
 })
 
 test('can fetch a remote graph', async () => {
-  const storage = new BlobTreeInMem()
+  const storage = new QuadAndBlobStore(new BlobTreeInMem())
   const rdfLayer = new RdfLayer('example.com', storage)
   const graph = await rdfLayer.fetchGraph(new URL('https://michielbdejong.com/profile/card'))
   expect(graph.length).toEqual(5)