diff --git a/.eslintrc.js b/.eslintrc.js index 92aaa97964..ee3a664c9e 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -5,6 +5,9 @@ module.exports = { tsconfigRootDir: __dirname, // this is the reason this is a .js file project: ['./tsconfig.json'], }, + globals: { + NodeJS: 'readonly' + }, plugins: [ 'tsdoc', 'import', diff --git a/index.ts b/index.ts index e8e914b571..efcb58e7b2 100644 --- a/index.ts +++ b/index.ts @@ -84,6 +84,7 @@ export * from './src/server/HttpResponse'; export * from './src/storage/accessors/DataAccessor'; export * from './src/storage/accessors/FileDataAccessor'; export * from './src/storage/accessors/InMemoryDataAccessor'; +export * from './src/storage/accessors/SparqlDataAccessor'; // Storage/Conversion export * from './src/storage/conversion/ChainedConverter'; diff --git a/package-lock.json b/package-lock.json index 19f4ef577c..b16defb55a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1350,6 +1350,14 @@ "@types/node": "*" } }, + "@types/sparqljs": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/sparqljs/-/sparqljs-3.1.0.tgz", + "integrity": "sha512-Tb+WnG8xLb2XHI3Nub4b6WFWT5gPgGrwhNS9u+jX1uTCBylPMnKSwCcr2UJF2yEZ2Jw8EaNBy27hhmGwhMXccw==", + "requires": { + "@types/rdf-js": "*" + } + }, "@types/stack-utils": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.0.tgz", @@ -1535,6 +1543,15 @@ "eslint-visitor-keys": "^2.0.0" } }, + "JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } + }, "abab": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz", @@ -3991,6 +4008,38 @@ "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz", "integrity": "sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg==" }, + "fetch-sparql-endpoint": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/fetch-sparql-endpoint/-/fetch-sparql-endpoint-1.8.0.tgz", + "integrity": "sha512-r6i3KcsvQBRnQq2CiyE6d1LNzwOYhbmiRgs0IZyWtkP+bczLeCEoZloY5XCmpp/4OWI8CL4fFBsaluizz+E9JA==", + "requires": { + "cross-fetch": "^3.0.6", + "is-stream": "^2.0.0", + "minimist": "^1.2.0", + "n3": "^1.6.3", + "rdf-string": "^1.5.0", + "sparqljs": "^3.1.2", + "sparqljson-parse": "^1.6.0", + "sparqlxml-parse": "^1.4.0", + "stream-to-string": "^1.1.0", + "web-streams-node": "^0.4.0" + }, + "dependencies": { + "is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" + }, + "rdf-string": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/rdf-string/-/rdf-string-1.5.0.tgz", + "integrity": "sha512-3TEJuDIKUADgZrfcZG+zAN4GfVA1Ei2sKA7Z7QVHkAE36wWoRGPJbGihPQMldgzvy9lG2nzZU+CXz+6oGSQNsQ==", + "requires": { + "rdf-data-factory": "^1.0.0" + } + } + } + }, "file-entry-cache": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", @@ -7975,6 +8024,15 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "sax-stream": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sax-stream/-/sax-stream-1.3.0.tgz", + "integrity": "sha512-tcfsAAICAkyNNe4uiKtKmLKxx3C7qPAej13UUoN+7OLYq/P5kHGahZtJhhMVM3fIMndA6TlYHWFlFEzFkv1VGg==", + "requires": { + "debug": "~2", + "sax": "~1" + } + }, "saxes": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", @@ -8369,6 +8427,42 @@ "n3": "^1.6.0" } }, + "sparqljson-parse": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sparqljson-parse/-/sparqljson-parse-1.6.0.tgz", + "integrity": "sha512-alIiURVr3AXIGU6fjuh5k6fwINwGKBQu5QnN9TEpoyIRvukKxZLQE07AHsw/Wxhkxico81tPf8nJTx7H1ira5A==", + "requires": { + "@types/node": "^13.1.0", + "@types/rdf-js": "*", + "JSONStream": "^1.3.3", + "rdf-data-factory": "^1.0.2" + }, + "dependencies": { + "@types/node": { + "version": "13.13.27", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.27.tgz", + "integrity": "sha512-IeZlpkPnUqO45iBxJocIQzwV+K6phdSVaCxRwlvHHQ0YL+Gb1fvuv9GmIMYllZcjyzqoRKDNJeNo6p8dNWSPSQ==" + } + } + }, + "sparqlxml-parse": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/sparqlxml-parse/-/sparqlxml-parse-1.4.0.tgz", + "integrity": "sha512-hKYsRw+KHIF4QXpMtybCSkfVhoQmTdUrUe5WkYnlyyw+3aeskIDnd97TPQi7MNSok2aim02osqkHvWQFNGXm3A==", + "requires": { + "@types/node": "^13.1.0", + "@types/rdf-js": "*", + "rdf-data-factory": "^1.0.2", + "sax-stream": "^1.2.3" + }, + "dependencies": { + "@types/node": { + "version": "13.13.27", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.27.tgz", + "integrity": "sha512-IeZlpkPnUqO45iBxJocIQzwV+K6phdSVaCxRwlvHHQ0YL+Gb1fvuv9GmIMYllZcjyzqoRKDNJeNo6p8dNWSPSQ==" + } + } + }, "spdx-correct": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", @@ -8844,6 +8938,11 @@ "integrity": "sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==", "dev": true }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, "through2": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", diff --git a/package.json b/package.json index 400799d970..490f5f6290 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "@types/n3": "^1.4.4", "@types/node": "^14.10.2", "@types/rdf-js": "^4.0.0", + "@types/sparqljs": "^3.1.0", "@types/streamify-array": "^1.0.0", "@types/uuid": "^8.3.0", "@types/yargs": "^15.0.5", @@ -82,12 +83,14 @@ "componentsjs": "^3.6.0", "cors": "^2.8.5", "express": "^4.17.1", + "fetch-sparql-endpoint": "^1.8.0", "mime-types": "^2.1.27", "n3": "^1.6.3", "rdf-parse": "^1.5.0", "rdf-serialize": "^1.0.0", "rdf-terms": "^1.5.1", "sparqlalgebrajs": "^2.3.1", + "sparqljs": "^3.1.2", "streamify-array": "^1.0.1", "uuid": "^8.3.0", "winston": "^3.3.3", diff --git a/src/storage/RepresentationConvertingStore.ts b/src/storage/RepresentationConvertingStore.ts index 688067741e..8fd746b7fe 100644 --- a/src/storage/RepresentationConvertingStore.ts +++ b/src/storage/RepresentationConvertingStore.ts @@ -9,9 +9,15 @@ import { PassthroughStore } from './PassthroughStore'; import type { ResourceStore } from './ResourceStore'; /** - * Store that overrides the `getRepresentation` function. - * Tries to convert the {@link Representation} it got from the source store - * so it matches one of the given type preferences. + * Store that overrides all functions that take or output a {@link Representation}, + * so `getRepresentation`, `addResource`, and `setRepresentation`. + * + * For incoming representations, they will be converted if an incoming converter and preferences have been set. + * The converted Representation will be passed along. + * + * For outgoing representations, they will be converted if there is an outgoing converter. + * + * Conversions will only happen if required and will not happen if the Representation is already in the correct format. * * In the future this class should take the preferences of the request into account. * Even if there is a match with the output from the store, @@ -20,21 +26,47 @@ import type { ResourceStore } from './ResourceStore'; export class RepresentationConvertingStore extends PassthroughStore { protected readonly logger = getLoggerFor(this); - private readonly converter: RepresentationConverter; + private readonly inConverter?: RepresentationConverter; + private readonly outConverter?: RepresentationConverter; + + private readonly inType?: string; - public constructor(source: T, converter: RepresentationConverter) { + /** + * TODO: This should take RepresentationPreferences instead of a type string when supported by Components.js. + */ + public constructor(source: T, options: { + outConverter?: RepresentationConverter; + inConverter?: RepresentationConverter; + inType?: string; + }) { super(source); - this.converter = converter; + this.inConverter = options.inConverter; + this.outConverter = options.outConverter; + this.inType = options.inType; } public async getRepresentation(identifier: ResourceIdentifier, preferences: RepresentationPreferences, conditions?: Conditions): Promise { const representation = await super.getRepresentation(identifier, preferences, conditions); - if (this.matchesPreferences(representation, preferences)) { + if (!this.outConverter || this.matchesPreferences(representation, preferences)) { return representation; } this.logger.info(`Convert ${identifier.path} from ${representation.metadata.contentType} to ${preferences.type}`); - return this.converter.handleSafe({ identifier, representation, preferences }); + return this.outConverter.handleSafe({ identifier, representation, preferences }); + } + + public async addResource(container: ResourceIdentifier, representation: Representation, + conditions?: Conditions): Promise { + // We can potentially run into problems here if we convert a turtle document where the base IRI is required, + // since we don't know the resource IRI yet at this point. + representation = await this.convertInRepresentation(container, representation); + return this.source.addResource(container, representation, conditions); + } + + public async setRepresentation(identifier: ResourceIdentifier, representation: Representation, + conditions?: Conditions): Promise { + representation = await this.convertInRepresentation(identifier, representation); + return this.source.setRepresentation(identifier, representation, conditions); } private matchesPreferences(representation: Representation, preferences: RepresentationPreferences): boolean { @@ -49,4 +81,16 @@ export class RepresentationConvertingStore { + if (!this.inType) { + return representation; + } + const inPreferences: RepresentationPreferences = { type: [{ value: this.inType, weight: 1 }]}; + if (!inPreferences || !this.inConverter || this.matchesPreferences(representation, inPreferences)) { + return representation; + } + return this.inConverter.handleSafe({ identifier, representation, preferences: inPreferences }); + } } diff --git a/src/storage/accessors/SparqlDataAccessor.ts b/src/storage/accessors/SparqlDataAccessor.ts new file mode 100644 index 0000000000..d0ca683c4b --- /dev/null +++ b/src/storage/accessors/SparqlDataAccessor.ts @@ -0,0 +1,312 @@ +import type { Readable } from 'stream'; +import arrayifyStream from 'arrayify-stream'; +import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint'; +import { DataFactory } from 'n3'; +import type { NamedNode, Quad } from 'rdf-js'; +import type { + ConstructQuery, GraphPattern, + GraphQuads, + InsertDeleteOperation, + SparqlGenerator, + Update, + UpdateOperation, +} from 'sparqljs'; +import { + Generator, +} from 'sparqljs'; +import type { Representation } from '../../ldp/representation/Representation'; +import { RepresentationMetadata } from '../../ldp/representation/RepresentationMetadata'; +import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier'; +import { INTERNAL_QUADS } from '../../util/ContentTypes'; +import { ConflictHttpError } from '../../util/errors/ConflictHttpError'; +import { NotFoundHttpError } from '../../util/errors/NotFoundHttpError'; +import { UnsupportedHttpError } from '../../util/errors/UnsupportedHttpError'; +import { UnsupportedMediaTypeHttpError } from '../../util/errors/UnsupportedMediaTypeHttpError'; +import type { MetadataController } from '../../util/MetadataController'; +import { CONTENT_TYPE, LDP } from '../../util/UriConstants'; +import { toNamedNode } from '../../util/UriUtil'; +import { ensureTrailingSlash } from '../../util/Util'; +import type { ContainerManager } from '../ContainerManager'; +import type { DataAccessor } from './DataAccessor'; + +const { defaultGraph, namedNode, quad, variable } = DataFactory; + +/** + * Stores all data and metadata of resources in a SPARQL backend. + * Communication is done by sending SPARQL queries. + * Queries are constructed in such a way to keep everything consistent, + * such as updating containment triples and deleting old data when it is overwritten. + * + * Since metadata is hidden, no containment triples are stored for metadata files. + * + * All input container metadata is stored in its metadata identifier. + * The containment triples are stored in the graph corresponding to the actual identifier + * so those don't get overwritten. + */ +export class SparqlDataAccessor implements DataAccessor { + private readonly endpoint: string; + private readonly base: string; + private readonly containerManager: ContainerManager; + private readonly metadataController: MetadataController; + private readonly fetcher: SparqlEndpointFetcher; + private readonly generator: SparqlGenerator; + + public constructor(endpoint: string, base: string, containerManager: ContainerManager, + metadataController: MetadataController) { + this.endpoint = endpoint; + this.base = ensureTrailingSlash(base); + this.containerManager = containerManager; + this.metadataController = metadataController; + this.fetcher = new SparqlEndpointFetcher(); + this.generator = new Generator(); + } + + /** + * Only Quad data streams are supported. + */ + public async canHandle(representation: Representation): Promise { + if (representation.binary || representation.metadata.contentType !== INTERNAL_QUADS) { + throw new UnsupportedMediaTypeHttpError('Only Quad data is supported.'); + } + } + + /** + * Returns all triples stored for the corresponding identifier. + * Note that this will not throw a 404 if no results were found. + */ + public async getData(identifier: ResourceIdentifier): Promise { + const name = namedNode(identifier.path); + return this.sendSparqlConstruct(this.sparqlConstruct(name)); + } + + /** + * Returns the metadata for the corresponding identifier. + * Will throw 404 if no metadata was found. + */ + public async getMetadata(identifier: ResourceIdentifier): Promise { + const name = namedNode(identifier.path); + const query = identifier.path.endsWith('/') ? + this.sparqlConstructContainer(name) : + this.sparqlConstruct(this.getMetadataNode(name)); + const stream = await this.sendSparqlConstruct(query); + const quads = await arrayifyStream(stream); + + // Root container will not have metadata if there are no containment triples + if (quads.length === 0 && identifier.path !== this.base) { + throw new NotFoundHttpError(); + } + + const metadata = new RepresentationMetadata(identifier.path).addQuads(quads); + metadata.contentType = INTERNAL_QUADS; + + // Need to generate type metadata for the root container since it's not stored + if (identifier.path === this.base) { + metadata.addQuads(this.metadataController.generateResourceQuads(name, true)); + } + + return metadata; + } + + /** + * Writes the given metadata for the container. + */ + public async writeContainer(identifier: ResourceIdentifier, metadata: RepresentationMetadata): Promise { + const { name, parent } = await this.getRelatedNames(identifier); + return this.sendSparqlUpdate(this.sparqlInsert(name, parent, metadata)); + } + + /** + * Reads the given data stream and stores it together with the metadata. + */ + public async writeDocument(identifier: ResourceIdentifier, data: Readable, metadata: RepresentationMetadata): + Promise { + if (this.isMetadataIdentifier(identifier)) { + throw new ConflictHttpError('Not allowed to create NamedNodes with the metadata extension.'); + } + const { name, parent } = await this.getRelatedNames(identifier); + + const triples = await arrayifyStream(data) as Quad[]; + const def = defaultGraph(); + if (triples.some((triple): boolean => !def.equals(triple.graph))) { + throw new UnsupportedHttpError('Only triples in the default graph are supported.'); + } + + // Not relevant since all content is triples + metadata.removeAll(CONTENT_TYPE); + + return this.sendSparqlUpdate(this.sparqlInsert(name, parent, metadata, triples)); + } + + /** + * Removes all graph data relevant to the given identifier. + */ + public async deleteResource(identifier: ResourceIdentifier): Promise { + const { name, parent } = await this.getRelatedNames(identifier); + return this.sendSparqlUpdate(this.sparqlDelete(name, parent)); + } + + /** + * Helper function to get named nodes corresponding to the identifier and its parent container. + */ + private async getRelatedNames(identifier: ResourceIdentifier): Promise<{ name: NamedNode; parent: NamedNode }> { + const parentIdentifier = await this.containerManager.getContainer(identifier); + const name = namedNode(identifier.path); + const parent = namedNode(parentIdentifier.path); + return { name, parent }; + } + + /** + * Creates the name for the metadata of a resource. + * @param name - Name of the (non-metadata) resource. + */ + private getMetadataNode(name: NamedNode): NamedNode { + return namedNode(`meta:${name.value}`); + } + + /** + * Checks if the given identifier corresponds to the names used for metadata identifiers. + */ + private isMetadataIdentifier(identifier: ResourceIdentifier): boolean { + return identifier.path.startsWith('meta:'); + } + + /** + * Creates a CONSTRUCT query that returns all quads contained within a single resource. + * @param name - Name of the resource to query. + */ + private sparqlConstruct(name: NamedNode): ConstructQuery { + const pattern = quad(variable('s'), variable('p'), variable('o')); + return { + queryType: 'CONSTRUCT', + template: [ pattern ], + where: [ this.sparqlSelectGraph(name, [ pattern ]) ], + type: 'query', + prefixes: {}, + }; + } + + private sparqlConstructContainer(name: NamedNode): ConstructQuery { + const pattern = quad(variable('s'), variable('p'), variable('o')); + return { + queryType: 'CONSTRUCT', + template: [ pattern ], + where: [{ + type: 'union', + patterns: [ + this.sparqlSelectGraph(name, [ pattern ]), + this.sparqlSelectGraph(this.getMetadataNode(name), [ pattern ]), + ], + }], + type: 'query', + prefixes: {}, + }; + } + + private sparqlSelectGraph(name: NamedNode, triples: Quad[]): GraphPattern { + return { + type: 'graph', + name, + patterns: [{ type: 'bgp', triples }], + }; + } + + /** + * Creates an update query that overwrites the data and metadata of a resource. + * If there are no triples we assume it's a container (so don't overwrite the main graph with containment triples). + * @param name - Name of the resource to update. + * @param parent - Name of the parent to update the containment triples. + * @param metadata - New metadata of the resource. + * @param triples - New data of the resource. + */ + private sparqlInsert(name: NamedNode, parent: NamedNode, metadata: RepresentationMetadata, triples?: Quad[]): Update { + const metaName = this.getMetadataNode(name); + + // Insert new metadata and containment triple + const insert: GraphQuads[] = [ + this.sparqlUpdateGraph(metaName, metadata.quads()), + this.sparqlUpdateGraph(parent, [ quad(parent, toNamedNode(LDP.contains), name) ]), + ]; + + // Necessary updates: delete metadata and insert new data + const updates: UpdateOperation[] = [ + this.sparqlUpdateDeleteAll(metaName), + { + updateType: 'insert', + insert, + }, + ]; + + // Only overwrite data triples for documents + if (triples) { + // This needs to be first so it happens before the insert + updates.unshift(this.sparqlUpdateDeleteAll(name)); + insert.push(this.sparqlUpdateGraph(name, triples)); + } + + return { + updates, + type: 'update', + prefixes: {}, + }; + } + + /** + * Creates a query that deletes everything related to the given name. + * @param name - Name of resource to delete. + * @param parent - Parent of the resource to delete so containment triple can be removed. + */ + private sparqlDelete(name: NamedNode, parent: NamedNode): Update { + return { + updates: [ + this.sparqlUpdateDeleteAll(name), + this.sparqlUpdateDeleteAll(this.getMetadataNode(name)), + { + updateType: 'delete', + delete: [ this.sparqlUpdateGraph(parent, [ quad(parent, toNamedNode(LDP.contains), name) ]) ], + }, + ], + type: 'update', + prefixes: {}, + }; + } + + /** + * Helper function for creating SPARQL update queries. + * Creates an operation for deleting all triples in a graph. + * @param name - Name of the graph to delete. + */ + private sparqlUpdateDeleteAll(name: NamedNode): InsertDeleteOperation { + return { + updateType: 'deletewhere', + delete: [ this.sparqlUpdateGraph(name, [ quad(variable(`s`), variable(`p`), variable(`o`)) ]) ], + }; + } + + /** + * Helper function for creating SPARQL update queries. + * Creates a Graph selector with the given triples. + * @param name - Name of the graph. + * @param triples - Triples/triple patterns to select. + */ + private sparqlUpdateGraph(name: NamedNode, triples: Quad[]): GraphQuads { + return { type: 'graph', name, triples }; + } + + /** + * Sends a SPARQL CONSTRUCT query to the endpoint and returns a stream of quads. + * @param sparqlQuery - Query to execute. + */ + private async sendSparqlConstruct(sparqlQuery: ConstructQuery): Promise { + const query = this.generator.stringify(sparqlQuery); + return await this.fetcher.fetchTriples(this.endpoint, query); + } + + /** + * Sends a SPARQL update query to the stored endpoint. + * @param sparqlQuery - Query to send. + */ + private async sendSparqlUpdate(sparqlQuery: Update): Promise { + const query = this.generator.stringify(sparqlQuery); + return await this.fetcher.fetchUpdate(this.endpoint, query); + } +} diff --git a/src/util/Util.ts b/src/util/Util.ts index 72246e7176..3b5b432905 100644 --- a/src/util/Util.ts +++ b/src/util/Util.ts @@ -66,7 +66,7 @@ export const matchingMediaType = (mediaA: string, mediaB: string): boolean => { * * @returns The destination stream. */ -export const pipeStreamsAndErrors = (readable: Readable, destination: T, +export const pipeStreamsAndErrors = (readable: NodeJS.ReadableStream, destination: T, mapError?: (error: Error) => Error): T => { readable.pipe(destination); readable.on('error', (error): boolean => { diff --git a/test/configs/Util.ts b/test/configs/Util.ts index 8d166a8df2..cf58a6bc32 100644 --- a/test/configs/Util.ts +++ b/test/configs/Util.ts @@ -76,11 +76,11 @@ export const getInMemoryResourceStore = (base = BASE): DataAccessorBasedStore => */ export const getConvertingStore = (store: ResourceStore, converters: RepresentationConverter[]): RepresentationConvertingStore => - new RepresentationConvertingStore(store, new CompositeAsyncHandler(converters)); + new RepresentationConvertingStore(store, { outConverter: new CompositeAsyncHandler(converters) }); /** * Gives a patching store based on initial store. - * @param store - Inital resource store. + * @param store - Initial resource store. * * @returns The patching store. */ diff --git a/test/unit/storage/RepresentationConvertingStore.test.ts b/test/unit/storage/RepresentationConvertingStore.test.ts index b44fde42e8..7920acf7c8 100644 --- a/test/unit/storage/RepresentationConvertingStore.test.ts +++ b/test/unit/storage/RepresentationConvertingStore.test.ts @@ -1,3 +1,4 @@ +import type { Representation } from '../../../src/ldp/representation/Representation'; import { RepresentationMetadata } from '../../../src/ldp/representation/RepresentationMetadata'; import type { RepresentationConverter } from '../../../src/storage/conversion/RepresentationConverter'; import { RepresentationConvertingStore } from '../../../src/storage/RepresentationConvertingStore'; @@ -7,19 +8,24 @@ import { CONTENT_TYPE } from '../../../src/util/UriConstants'; describe('A RepresentationConvertingStore', (): void => { let store: RepresentationConvertingStore; let source: ResourceStore; - let handleSafeFn: jest.Mock, []>; - let converter: RepresentationConverter; + let inConverter: RepresentationConverter; + let outConverter: RepresentationConverter; + const inType = 'text/turtle'; const metadata = new RepresentationMetadata({ [CONTENT_TYPE]: 'text/turtle' }); + let representation: Representation; beforeEach(async(): Promise => { source = { getRepresentation: jest.fn(async(): Promise => ({ data: 'data', metadata })), - } as unknown as ResourceStore; + addResource: jest.fn(), + setRepresentation: jest.fn(), + } as any; - handleSafeFn = jest.fn(async(): Promise => 'converter'); - converter = { handleSafe: handleSafeFn } as unknown as RepresentationConverter; + inConverter = { handleSafe: jest.fn(async(): Promise => 'inConvert') } as any; + outConverter = { handleSafe: jest.fn(async(): Promise => 'outConvert') } as any; - store = new RepresentationConvertingStore(source, converter); + store = new RepresentationConvertingStore(source, { inType, inConverter, outConverter }); + representation = { binary: true, data: 'data', metadata } as any; }); it('returns the Representation from the source if no changes are required.', async(): Promise => { @@ -35,7 +41,7 @@ describe('A RepresentationConvertingStore', (): void => { expect(source.getRepresentation).toHaveBeenLastCalledWith( { path: 'path' }, { type: [{ value: 'text/*', weight: 0 }, { value: 'text/turtle', weight: 1 }]}, undefined, ); - expect(handleSafeFn).toHaveBeenCalledTimes(0); + expect(outConverter.handleSafe).toHaveBeenCalledTimes(0); }); it('returns the Representation from the source if there are no preferences.', async(): Promise => { @@ -49,19 +55,47 @@ describe('A RepresentationConvertingStore', (): void => { expect(source.getRepresentation).toHaveBeenLastCalledWith( { path: 'path' }, {}, undefined, ); - expect(handleSafeFn).toHaveBeenCalledTimes(0); + expect(outConverter.handleSafe).toHaveBeenCalledTimes(0); }); it('calls the converter if another output is preferred.', async(): Promise => { await expect(store.getRepresentation({ path: 'path' }, { type: [ { value: 'text/plain', weight: 1 }, { value: 'text/turtle', weight: 0 }, - ]})).resolves.toEqual('converter'); + ]})).resolves.toEqual('outConvert'); expect(source.getRepresentation).toHaveBeenCalledTimes(1); - expect(handleSafeFn).toHaveBeenCalledTimes(1); - expect(handleSafeFn).toHaveBeenLastCalledWith({ + expect(outConverter.handleSafe).toHaveBeenCalledTimes(1); + expect(outConverter.handleSafe).toHaveBeenLastCalledWith({ identifier: { path: 'path' }, representation: { data: 'data', metadata }, preferences: { type: [{ value: 'text/plain', weight: 1 }, { value: 'text/turtle', weight: 0 }]}, }); }); + + it('keeps the representation if the conversion is not required.', async(): Promise => { + const id = { path: 'identifier' }; + + await expect(store.addResource(id, representation, 'conditions' as any)).resolves.toBeUndefined(); + expect(source.addResource).toHaveBeenLastCalledWith(id, representation, 'conditions'); + + await expect(store.setRepresentation(id, representation, 'conditions' as any)).resolves.toBeUndefined(); + expect(inConverter.handleSafe).toHaveBeenCalledTimes(0); + expect(source.setRepresentation).toHaveBeenLastCalledWith(id, representation, 'conditions'); + + store = new RepresentationConvertingStore(source, {}); + await expect(store.addResource(id, representation, 'conditions' as any)).resolves.toBeUndefined(); + expect(source.addResource).toHaveBeenLastCalledWith(id, representation, 'conditions'); + }); + + it('converts the data if it is required.', async(): Promise => { + metadata.contentType = 'text/plain'; + const id = { path: 'identifier' }; + + await expect(store.addResource(id, representation, 'conditions' as any)).resolves.toBeUndefined(); + expect(inConverter.handleSafe).toHaveBeenCalledTimes(1); + expect(source.addResource).toHaveBeenLastCalledWith(id, 'inConvert', 'conditions'); + + await expect(store.setRepresentation(id, representation, 'conditions' as any)).resolves.toBeUndefined(); + expect(inConverter.handleSafe).toHaveBeenCalledTimes(2); + expect(source.setRepresentation).toHaveBeenLastCalledWith(id, 'inConvert', 'conditions'); + }); }); diff --git a/test/unit/storage/accessors/SparqlDataAccessor.test.ts b/test/unit/storage/accessors/SparqlDataAccessor.test.ts new file mode 100644 index 0000000000..8e7d21b979 --- /dev/null +++ b/test/unit/storage/accessors/SparqlDataAccessor.test.ts @@ -0,0 +1,208 @@ +import type { Readable } from 'stream'; +import arrayifyStream from 'arrayify-stream'; +import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint'; +import { DataFactory } from 'n3'; +import type { Quad } from 'rdf-js'; +import streamifyArray from 'streamify-array'; +import { RepresentationMetadata } from '../../../../src/ldp/representation/RepresentationMetadata'; +import { SparqlDataAccessor } from '../../../../src/storage/accessors/SparqlDataAccessor'; +import { UrlContainerManager } from '../../../../src/storage/UrlContainerManager'; +import { INTERNAL_QUADS } from '../../../../src/util/ContentTypes'; +import { ConflictHttpError } from '../../../../src/util/errors/ConflictHttpError'; +import { NotFoundHttpError } from '../../../../src/util/errors/NotFoundHttpError'; +import { UnsupportedHttpError } from '../../../../src/util/errors/UnsupportedHttpError'; +import { UnsupportedMediaTypeHttpError } from '../../../../src/util/errors/UnsupportedMediaTypeHttpError'; +import { MetadataController } from '../../../../src/util/MetadataController'; +import { CONTENT_TYPE, LDP, RDF } from '../../../../src/util/UriConstants'; +import { toNamedNode } from '../../../../src/util/UriUtil'; + +const { literal, namedNode, quad } = DataFactory; + +jest.mock('fetch-sparql-endpoint'); + +const simplifyQuery = (query: string | string[]): string => { + if (Array.isArray(query)) { + query = query.join(' '); + } + return query.replace(/\n/gu, ' ').trim(); +}; + +describe('A SparqlDataAccessor', (): void => { + const endpoint = 'http://test.com/sparql'; + const base = 'http://test.com/'; + let accessor: SparqlDataAccessor; + let metadata: RepresentationMetadata; + let fetchTriples: jest.Mock>; + let fetchUpdate: jest.Mock>; + let triples: Quad[]; + + beforeEach(async(): Promise => { + metadata = new RepresentationMetadata(); + triples = [ quad(namedNode('this'), namedNode('a'), namedNode('triple')) ]; + + // Makes it so the `SparqlEndpointFetcher` will always return the contents of the `bindings` array + fetchTriples = jest.fn(async(): Promise => streamifyArray(triples)); + fetchUpdate = jest.fn(async(): Promise => undefined); + (SparqlEndpointFetcher as any).mockImplementation((): any => ({ + fetchTriples, + fetchUpdate, + })); + + // This needs to be last so the fetcher can be mocked first + accessor = new SparqlDataAccessor(endpoint, base, new UrlContainerManager(base), new MetadataController()); + }); + + it('can only handle quad data.', async(): Promise => { + const data = streamifyArray([]); + await expect(accessor.canHandle({ binary: true, data, metadata })).rejects.toThrow(UnsupportedMediaTypeHttpError); + metadata.contentType = 'newInternalType'; + await expect(accessor.canHandle({ binary: false, data, metadata })).rejects.toThrow(UnsupportedMediaTypeHttpError); + metadata.contentType = INTERNAL_QUADS; + await expect(accessor.canHandle({ binary: false, data, metadata })).resolves.toBeUndefined(); + }); + + it('returns the corresponding quads when data is requested.', async(): Promise => { + const data = await accessor.getData({ path: 'http://identifier' }); + await expect(arrayifyStream(data)).resolves.toBeRdfIsomorphic([ + quad(namedNode('this'), namedNode('a'), namedNode('triple')), + ]); + + expect(fetchTriples).toHaveBeenCalledTimes(1); + expect(fetchTriples.mock.calls[0][0]).toBe(endpoint); + expect(simplifyQuery(fetchTriples.mock.calls[0][1])).toBe(simplifyQuery( + 'CONSTRUCT { ?s ?p ?o. } WHERE { GRAPH { ?s ?p ?o. } }', + )); + }); + + it('returns the corresponding metadata when requested.', async(): Promise => { + metadata = await accessor.getMetadata({ path: 'http://identifier' }); + expect(metadata.quads()).toBeRdfIsomorphic([ + quad(namedNode('this'), namedNode('a'), namedNode('triple')), + quad(namedNode('http://identifier'), toNamedNode(CONTENT_TYPE), literal(INTERNAL_QUADS)), + ]); + + expect(fetchTriples).toHaveBeenCalledTimes(1); + expect(fetchTriples.mock.calls[0][0]).toBe(endpoint); + expect(simplifyQuery(fetchTriples.mock.calls[0][1])).toBe(simplifyQuery( + 'CONSTRUCT { ?s ?p ?o. } WHERE { GRAPH { ?s ?p ?o. } }', + )); + }); + + it('requests container data for generating its metadata.', async(): Promise => { + metadata = await accessor.getMetadata({ path: 'http://container/' }); + expect(metadata.quads()).toBeRdfIsomorphic([ + quad(namedNode('this'), namedNode('a'), namedNode('triple')), + quad(namedNode('http://container/'), toNamedNode(CONTENT_TYPE), literal(INTERNAL_QUADS)), + ]); + + expect(fetchTriples).toHaveBeenCalledTimes(1); + expect(fetchTriples.mock.calls[0][0]).toBe(endpoint); + expect(simplifyQuery(fetchTriples.mock.calls[0][1])).toBe(simplifyQuery([ + 'CONSTRUCT { ?s ?p ?o. } WHERE {', + ' { GRAPH { ?s ?p ?o. } }', + ' UNION', + ' { GRAPH { ?s ?p ?o. } }', + '}', + ])); + }); + + it('generates resource metadata for the root container.', async(): Promise => { + metadata = await accessor.getMetadata({ path: base }); + expect(metadata.quads()).toBeRdfIsomorphic([ + quad(namedNode('this'), namedNode('a'), namedNode('triple')), + quad(namedNode(base), toNamedNode(CONTENT_TYPE), literal(INTERNAL_QUADS)), + quad(namedNode(base), toNamedNode(RDF.type), toNamedNode(LDP.Container)), + quad(namedNode(base), toNamedNode(RDF.type), toNamedNode(LDP.BasicContainer)), + quad(namedNode(base), toNamedNode(RDF.type), toNamedNode(LDP.Resource)), + ]); + + expect(fetchTriples).toHaveBeenCalledTimes(1); + expect(fetchTriples.mock.calls[0][0]).toBe(endpoint); + expect(simplifyQuery(fetchTriples.mock.calls[0][1])).toBe(simplifyQuery([ + 'CONSTRUCT { ?s ?p ?o. } WHERE {', + ` { GRAPH <${base}> { ?s ?p ?o. } }`, + ' UNION', + ` { GRAPH { ?s ?p ?o. } }`, + '}', + ])); + }); + + it('throws 404 if no metadata was found.', async(): Promise => { + // Clear bindings array + triples.splice(0, triples.length); + await expect(accessor.getMetadata({ path: 'http://identifier' })).rejects.toThrow(NotFoundHttpError); + + expect(fetchTriples).toHaveBeenCalledTimes(1); + expect(fetchTriples.mock.calls[0][0]).toBe(endpoint); + expect(simplifyQuery(fetchTriples.mock.calls[0][1])).toBe(simplifyQuery( + 'CONSTRUCT { ?s ?p ?o. } WHERE { GRAPH { ?s ?p ?o. } }', + )); + }); + + it('overwrites the metadata when writing a container and updates parent.', async(): Promise => { + metadata = new RepresentationMetadata('http://test.com/container/', + { [RDF.type]: [ toNamedNode(LDP.Resource), toNamedNode(LDP.Container) ]}); + await expect(accessor.writeContainer({ path: 'http://test.com/container/' }, metadata)).resolves.toBeUndefined(); + + expect(fetchUpdate).toHaveBeenCalledTimes(1); + expect(fetchUpdate.mock.calls[0][0]).toBe(endpoint); + expect(simplifyQuery(fetchUpdate.mock.calls[0][1])).toBe(simplifyQuery([ + 'DELETE WHERE { GRAPH { ?s ?p ?o. } };', + 'INSERT DATA {', + ' GRAPH {', + ' .', + ' .', + ' }', + ' GRAPH { . }', + '}', + ])); + }); + + it('overwrites the data and metadata when writing a resource and updates parent.', async(): Promise => { + const data = streamifyArray([ quad(namedNode('http://name'), namedNode('http://pred'), literal('value')) ]); + metadata = new RepresentationMetadata('http://test.com/container/resource', + { [RDF.type]: [ toNamedNode(LDP.Resource) ]}); + await expect(accessor.writeDocument({ path: 'http://test.com/container/resource' }, data, metadata)) + .resolves.toBeUndefined(); + + expect(fetchUpdate).toHaveBeenCalledTimes(1); + expect(fetchUpdate.mock.calls[0][0]).toBe(endpoint); + expect(simplifyQuery(fetchUpdate.mock.calls[0][1])).toBe(simplifyQuery([ + 'DELETE WHERE { GRAPH { ?s ?p ?o. } };', + 'DELETE WHERE { GRAPH { ?s ?p ?o. } };', + 'INSERT DATA {', + ' GRAPH { . }', + ' GRAPH { . }', + ' GRAPH { "value". }', + '}', + ])); + }); + + it('removes all references when deleting a resource.', async(): Promise => { + metadata = new RepresentationMetadata('http://test.com/container/', + { [RDF.type]: [ toNamedNode(LDP.Resource), toNamedNode(LDP.Container) ]}); + await expect(accessor.deleteResource({ path: 'http://test.com/container/' })).resolves.toBeUndefined(); + + expect(fetchUpdate).toHaveBeenCalledTimes(1); + expect(fetchUpdate.mock.calls[0][0]).toBe(endpoint); + expect(simplifyQuery(fetchUpdate.mock.calls[0][1])).toBe(simplifyQuery([ + 'DELETE WHERE { GRAPH { ?s ?p ?o. } };', + 'DELETE WHERE { GRAPH { ?s ?p ?o. } };', + 'DELETE DATA { GRAPH { . } }', + ])); + }); + + it('errors when trying to write to a metadata document.', async(): Promise => { + const data = streamifyArray([ quad(namedNode('http://name'), namedNode('http://pred'), literal('value')) ]); + await expect(accessor.writeDocument({ path: 'meta:http://test.com/container/resource' }, data, metadata)) + .rejects.toThrow(new ConflictHttpError('Not allowed to create NamedNodes with the metadata extension.')); + }); + + it('errors when writing triples in a non-default graph.', async(): Promise => { + const data = streamifyArray( + [ quad(namedNode('http://name'), namedNode('http://pred'), literal('value'), namedNode('badGraph!')) ], + ); + await expect(accessor.writeDocument({ path: 'http://test.com/container/resource' }, data, metadata)) + .rejects.toThrow(new UnsupportedHttpError('Only triples in the default graph are supported.')); + }); +});