Skip to content

Commit

Permalink
feat: Add support for range headers
Browse files Browse the repository at this point in the history
  • Loading branch information
joachimvh committed Oct 5, 2023
1 parent db66e3d commit 3e9adef
Show file tree
Hide file tree
Showing 25 changed files with 619 additions and 5 deletions.
3 changes: 3 additions & 0 deletions .componentsignore
Expand Up @@ -24,13 +24,16 @@
"NotificationChannelType",
"PermissionMap",
"Promise",
"Readable",
"Readonly",
"RegExp",
"Server",
"SetMultiMap",
"Shorthand",
"Template",
"TemplateEngine",
"Transform",
"TransformOptions",
"ValuePreferencesArg",
"VariableBindings",
"UnionHandler",
Expand Down
5 changes: 5 additions & 0 deletions .eslintrc.js
Expand Up @@ -10,6 +10,7 @@ module.exports = {
ignorePatterns: [ '*.js' ],
globals: {
AsyncIterable: 'readonly',
BufferEncoding: 'readonly',
NodeJS: 'readonly',
RequestInit: 'readonly',
},
Expand Down Expand Up @@ -71,11 +72,15 @@ module.exports = {
// Already checked by @typescript-eslint/no-unused-vars
'no-unused-vars': 'off',
'padding-line-between-statements': 'off',
// Forcing destructuring on existing variables causes clunky code
'prefer-destructuring': 'off',
'prefer-named-capture-group': 'off',
// Already generated by TypeScript
strict: 'off',
'tsdoc/syntax': 'error',
'unicorn/catch-error-name': 'off',
// Can cause some clunky situations if it forces us to assign to an existing variable
'unicorn/consistent-destructuring': 'off',
'unicorn/import-index': 'off',
'unicorn/import-style': 'off',
// The next 2 some functional programming paradigms
Expand Down
6 changes: 5 additions & 1 deletion config/ldp/handler/components/preferences.json
Expand Up @@ -3,7 +3,11 @@
"@graph": [
{
"@id": "urn:solid-server:default:PreferenceParser",
"@type": "AcceptPreferenceParser"
"@type": "UnionPreferenceParser",
"parsers": [
{ "@type": "AcceptPreferenceParser" },
{ "@type": "RangePreferenceParser" }
]
}
]
}
2 changes: 2 additions & 0 deletions config/ldp/metadata-writer/default.json
Expand Up @@ -7,6 +7,7 @@
"css:config/ldp/metadata-writer/writers/link-rel-metadata.json",
"css:config/ldp/metadata-writer/writers/mapped.json",
"css:config/ldp/metadata-writer/writers/modified.json",
"css:config/ldp/metadata-writer/writers/range.json",
"css:config/ldp/metadata-writer/writers/storage-description.json",
"css:config/ldp/metadata-writer/writers/www-auth.json"
],
Expand All @@ -22,6 +23,7 @@
{ "@id": "urn:solid-server:default:MetadataWriter_LinkRelMetadata" },
{ "@id": "urn:solid-server:default:MetadataWriter_Mapped" },
{ "@id": "urn:solid-server:default:MetadataWriter_Modified" },
{ "@id": "urn:solid-server:default:MetadataWriter_Range" },
{ "@id": "urn:solid-server:default:MetadataWriter_StorageDescription" },
{ "@id": "urn:solid-server:default:MetadataWriter_WwwAuth" }
]
Expand Down
10 changes: 10 additions & 0 deletions config/ldp/metadata-writer/writers/range.json
@@ -0,0 +1,10 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^6.0.0/components/context.jsonld",
"@graph": [
{
"comment": "Adds the Content-Range header if necessary.",
"@id": "urn:solid-server:default:MetadataWriter_Range",
"@type": "RangeMetadataWriter"
}
]
}
6 changes: 6 additions & 0 deletions config/storage/middleware/default.json
Expand Up @@ -16,6 +16,12 @@
"comment": "Sets up a stack of utility stores used by most instances.",
"@id": "urn:solid-server:default:ResourceStore",
"@type": "MonitoringStore",
"source": { "@id": "urn:solid-server:default:ResourceStore_BinarySlice" }
},
{
"comment": "Slices part of binary streams based on the range preferences.",
"@id": "urn:solid-server:default:ResourceStore_BinarySlice",
"@type": "BinarySliceResourceStore",
"source": { "@id": "urn:solid-server:default:ResourceStore_Index" }
},
{
Expand Down
2 changes: 1 addition & 1 deletion src/http/input/preferences/AcceptPreferenceParser.ts
Expand Up @@ -11,7 +11,7 @@ import type { RepresentationPreferences } from '../../representation/Representat
import { PreferenceParser } from './PreferenceParser';

const parsers: {
name: keyof RepresentationPreferences;
name: Exclude<keyof RepresentationPreferences, 'range'>;
header: string;
parse: (value: string) => AcceptHeader[];
}[] = [
Expand Down
48 changes: 48 additions & 0 deletions src/http/input/preferences/RangePreferenceParser.ts
@@ -0,0 +1,48 @@
import type { HttpRequest } from '../../../server/HttpRequest';
import { BadRequestHttpError } from '../../../util/errors/BadRequestHttpError';
import type { RepresentationPreferences } from '../../representation/RepresentationPreferences';
import { PreferenceParser } from './PreferenceParser';

/**
* Parses the range header into range preferences.
* If the range corresponds to a suffix-length range, it will be stored in `start` as a negative value.
*/
export class RangePreferenceParser extends PreferenceParser {
public async handle({ request: { headers: { range }}}: { request: HttpRequest }): Promise<RepresentationPreferences> {
if (!range) {
return {};
}

const [ unit, rangeTail ] = range.split('=').map((entry): string => entry.trim());
if (unit.length === 0) {
throw new BadRequestHttpError(`Missing unit value from range header ${range}`);
}
if (!rangeTail) {
throw new BadRequestHttpError(`Invalid range header format ${range}`);
}

const ranges = rangeTail.split(',').map((entry): string => entry.trim());
const parts: { start: number; end?: number }[] = [];
for (const rangeEntry of ranges) {
const [ start, end ] = rangeEntry.split('-').map((entry): string => entry.trim());
// This can actually be undefined if the split results in less than 2 elements
if (typeof end !== 'string') {
throw new BadRequestHttpError(`Invalid range header format ${range}`);
}
if (start.length === 0) {
if (end.length === 0) {
throw new BadRequestHttpError(`Invalid range header format ${range}`);
}
parts.push({ start: -Number.parseInt(end, 10) });
} else {
const part: typeof parts[number] = { start: Number.parseInt(start, 10) };
if (end.length > 0) {
part.end = Number.parseInt(end, 10);
}
parts.push(part);
}
}

return { range: { unit, parts }};
}
}
32 changes: 32 additions & 0 deletions src/http/input/preferences/UnionPreferenceParser.ts
@@ -0,0 +1,32 @@
import { InternalServerError } from '../../../util/errors/InternalServerError';
import { UnionHandler } from '../../../util/handlers/UnionHandler';
import type { RepresentationPreferences } from '../../representation/RepresentationPreferences';
import type { PreferenceParser } from './PreferenceParser';

/**
* Combines the results of multiple {@link PreferenceParser}s.
* Will throw an error if multiple parsers return a range as these can't logically be combined.
*/
export class UnionPreferenceParser extends UnionHandler<PreferenceParser> {
public constructor(parsers: PreferenceParser[]) {
super(parsers, false, false);
}

protected async combine(results: RepresentationPreferences[]): Promise<RepresentationPreferences> {
const rangeCount = results.filter((result): boolean => Boolean(result.range)).length;
if (rangeCount > 1) {
throw new InternalServerError('Found multiple range values. This implies a misconfiguration.');
}

return results.reduce<RepresentationPreferences>((acc, val): RepresentationPreferences => {
for (const key of Object.keys(val) as (keyof RepresentationPreferences)[]) {
if (key === 'range') {
acc[key] = val[key];
} else {
acc[key] = { ...acc[key], ...val[key] };
}
}
return acc;
}, {});
}
}
25 changes: 25 additions & 0 deletions src/http/output/metadata/RangeMetadataWriter.ts
@@ -0,0 +1,25 @@
import type { HttpResponse } from '../../../server/HttpResponse';
import { addHeader } from '../../../util/HeaderUtil';
import { SOLID_HTTP } from '../../../util/Vocabularies';
import type { RepresentationMetadata } from '../../representation/RepresentationMetadata';
import { MetadataWriter } from './MetadataWriter';

/**
* Generates the necessary `content-range` header if there is range metadata.
* If the start or end is unknown, a `*` will be used instead.
* According to the RFC, this is incorrect,
* but is all we can do as long as we don't know the full length of the representation in advance.
* For the same reason, the total length of the representation will always be `*`.
*/
export class RangeMetadataWriter extends MetadataWriter {
public async handle(input: { response: HttpResponse; metadata: RepresentationMetadata }): Promise<void> {
const unit = input.metadata.get(SOLID_HTTP.terms.unit);
if (!unit) {
return;
}
const start = input.metadata.get(SOLID_HTTP.terms.start);
const end = input.metadata.get(SOLID_HTTP.terms.end);

addHeader(input.response, 'Content-Range', `${unit.value} ${start?.value ?? '*'}-${end?.value ?? '*'}/*`);
}
}
6 changes: 4 additions & 2 deletions src/http/output/response/OkResponseDescription.ts
@@ -1,17 +1,19 @@
import type { Readable } from 'stream';
import type { Guarded } from '../../../util/GuardedStream';
import { SOLID_HTTP } from '../../../util/Vocabularies';
import type { RepresentationMetadata } from '../../representation/RepresentationMetadata';
import { ResponseDescription } from './ResponseDescription';

/**
* Corresponds to a 200 response, containing relevant metadata and potentially data.
* Corresponds to a 200 or 206 response, containing relevant metadata and potentially data.
* A 206 will be returned if range metadata is found in the metadata object.
*/
export class OkResponseDescription extends ResponseDescription {
/**
* @param metadata - Metadata concerning the response.
* @param data - Potential data. @ignored
*/
public constructor(metadata: RepresentationMetadata, data?: Guarded<Readable>) {
super(200, metadata, data);
super(metadata.has(SOLID_HTTP.terms.unit) ? 206 : 200, metadata, data);
}
}
2 changes: 2 additions & 0 deletions src/http/representation/RepresentationPreferences.ts
Expand Up @@ -31,4 +31,6 @@ export interface RepresentationPreferences {
datetime?: ValuePreferences;
encoding?: ValuePreferences;
language?: ValuePreferences;
// `start` can be negative and implies the last X of a stream
range?: { unit: string; parts: { start: number; end?: number }[] };
}
6 changes: 6 additions & 0 deletions src/index.ts
Expand Up @@ -77,6 +77,8 @@ export * from './http/input/metadata/SlugParser';
// HTTP/Input/Preferences
export * from './http/input/preferences/AcceptPreferenceParser';
export * from './http/input/preferences/PreferenceParser';
export * from './http/input/preferences/RangePreferenceParser';
export * from './http/input/preferences/UnionPreferenceParser';

// HTTP/Input
export * from './http/input/BasicRequestParser';
Expand Down Expand Up @@ -106,6 +108,7 @@ export * from './http/output/metadata/LinkRelMetadataWriter';
export * from './http/output/metadata/MappedMetadataWriter';
export * from './http/output/metadata/MetadataWriter';
export * from './http/output/metadata/ModifiedMetadataWriter';
export * from './http/output/metadata/RangeMetadataWriter';
export * from './http/output/metadata/StorageDescriptionAdvertiser';
export * from './http/output/metadata/WacAllowMetadataWriter';
export * from './http/output/metadata/WwwAuthMetadataWriter';
Expand Down Expand Up @@ -443,6 +446,7 @@ export * from './storage/validators/QuotaValidator';
export * from './storage/AtomicResourceStore';
export * from './storage/BaseResourceStore';
export * from './storage/BasicConditions';
export * from './storage/BinarySliceResourceStore';
export * from './storage/CachedResourceSet';
export * from './storage/Conditions';
export * from './storage/DataAccessorBasedStore';
Expand Down Expand Up @@ -472,6 +476,7 @@ export * from './util/errors/NotFoundHttpError';
export * from './util/errors/NotImplementedHttpError';
export * from './util/errors/OAuthHttpError';
export * from './util/errors/PreconditionFailedHttpError';
export * from './util/errors/RangeNotSatisfiedHttpError';
export * from './util/errors/RedirectHttpError';
export * from './util/errors/SystemError';
export * from './util/errors/UnauthorizedHttpError';
Expand Down Expand Up @@ -542,6 +547,7 @@ export * from './util/PromiseUtil';
export * from './util/QuadUtil';
export * from './util/RecordObject';
export * from './util/ResourceUtil';
export * from './util/SliceStream';
export * from './util/StreamUtil';
export * from './util/StringUtil';
export * from './util/TermUtil';
Expand Down
66 changes: 66 additions & 0 deletions src/storage/BinarySliceResourceStore.ts
@@ -0,0 +1,66 @@
import type { Representation } from '../http/representation/Representation';
import type { RepresentationPreferences } from '../http/representation/RepresentationPreferences';
import type { ResourceIdentifier } from '../http/representation/ResourceIdentifier';
import { getLoggerFor } from '../logging/LogUtil';
import { InternalServerError } from '../util/errors/InternalServerError';
import { RangeNotSatisfiedHttpError } from '../util/errors/RangeNotSatisfiedHttpError';
import { guardStream } from '../util/GuardedStream';
import { SliceStream } from '../util/SliceStream';
import { toLiteral } from '../util/TermUtil';
import { SOLID_HTTP, XSD } from '../util/Vocabularies';
import type { Conditions } from './Conditions';
import { PassthroughStore } from './PassthroughStore';
import type { ResourceStore } from './ResourceStore';

/**
* Resource store that slices the data stream if there are range preferences.
* Only works for `bytes` range preferences on binary data streams.
* Does not support multipart range requests.
*
* If the slice happens, unit/start/end values will be written to the metadata to indicate such.
* The values are dependent on the preferences we got as an input,
* as we don't know the actual size of the data stream.
*/
export class BinarySliceResourceStore<T extends ResourceStore = ResourceStore> extends PassthroughStore<T> {
protected readonly logger = getLoggerFor(this);

public async getRepresentation(identifier: ResourceIdentifier, preferences: RepresentationPreferences,
conditions?: Conditions): Promise<Representation> {
const result = await this.source.getRepresentation(identifier, preferences, conditions);

if (!preferences.range || preferences.range.unit !== 'bytes' || preferences.range.parts.length === 0) {
return result;
}
if (result.metadata.has(SOLID_HTTP.unit)) {
this.logger.debug('Not slicing stream that has already been sliced.');
return result;
}

if (!result.binary) {
throw new InternalServerError('Trying to slice a non-binary stream.');
}
if (preferences.range.parts.length > 1) {
throw new RangeNotSatisfiedHttpError('Multipart range requests are not supported.');
}

const [{ start, end }] = preferences.range.parts;
result.metadata.set(SOLID_HTTP.terms.unit, preferences.range.unit);
result.metadata.set(SOLID_HTTP.terms.start, toLiteral(start, XSD.terms.integer));
if (typeof end === 'number') {
result.metadata.set(SOLID_HTTP.terms.end, toLiteral(end, XSD.terms.integer));
}

try {
// The reason we don't determine the object mode based on the object mode of the parent stream
// is that `guardedStreamFrom` does not create object streams when inputting streams/buffers.
// Something to potentially update in the future.
result.data = guardStream(new SliceStream(result.data, { start, end, objectMode: false }));
} catch (error: unknown) {
// Creating the slice stream can throw an error if some of the parameters are unacceptable.
// Need to make sure the stream is closed in that case.
result.data.destroy();
throw error;
}
return result;
}
}

0 comments on commit 3e9adef

Please sign in to comment.