From a18e716f9733a7951209b4e472531d09f856fc23 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Thu, 27 Jul 2023 09:54:34 +0200 Subject: [PATCH 01/25] chore: Link to discussions when choosing issue template --- .github/ISSUE_TEMPLATE/config.yml | 5 +++++ .github/ISSUE_TEMPLATE/question.md | 13 ------------- 2 files changed, 5 insertions(+), 13 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/config.yml delete mode 100644 .github/ISSUE_TEMPLATE/question.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..c84f522ae8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: ❓ Question + url: https://github.com/CommunitySolidServer/CommunitySolidServer/discussions/new/choose + about: A question about the server diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md deleted file mode 100644 index 642c2b892b..0000000000 --- a/.github/ISSUE_TEMPLATE/question.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: "❓ Question" -about: A general question -title: '' -labels: '' -assignees: '' - ---- - -#### Question - - - From c2ab32baede584c844b73bb85cf60c0904af34b7 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Thu, 27 Jul 2023 09:55:41 +0200 Subject: [PATCH 02/25] chore: Update description for questions --- .github/ISSUE_TEMPLATE/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index c84f522ae8..d4ca74e45c 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,4 +2,4 @@ blank_issues_enabled: false contact_links: - name: ❓ Question url: https://github.com/CommunitySolidServer/CommunitySolidServer/discussions/new/choose - about: A question about the server + about: A question or discussion about the server From 0f79fa5ffa846377a8c51b78e3bd396bc4dc1816 Mon Sep 17 00:00:00 2001 From: smessie Date: Thu, 17 Aug 2023 08:50:48 +0200 Subject: [PATCH 03/25] ci: Add `linux/arm64/v8` as platform to support Apple Silicon architectures --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 786e13307a..d377d2e962 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -89,6 +89,6 @@ jobs: with: context: . push: true - platforms: linux/amd64,linux/arm/v7,linux/arm/v8 + platforms: linux/amd64,linux/arm/v7,linux/arm/v8,linux/arm64/v8 tags: ${{ needs.docker-meta.outputs.tags }} labels: ${{ needs.docker-meta.outputs.labels }} From d401cc862beaf727dcbdadb2c3d82ccd744d2aef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Aug 2023 00:52:24 +0000 Subject: [PATCH 04/25] chore(deps): bump actions/checkout from 3.5.3 to 3.6.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.3 to 3.6.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.3...v3.6.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/cth-test.yml | 2 +- .github/workflows/docker.yml | 4 ++-- .github/workflows/mkdocs.yml | 6 +++--- .github/workflows/npm-test.yml | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cth-test.yml b/.github/workflows/cth-test.yml index 6c033531bc..aee4a96acf 100644 --- a/.github/workflows/cth-test.yml +++ b/.github/workflows/cth-test.yml @@ -42,7 +42,7 @@ jobs: with: node-version: 16.x - name: Check out the project - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 with: ref: ${{ inputs.branch || github.ref }} - name: Install dependencies and run build scripts diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d377d2e962..09b7f02cba 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,7 +21,7 @@ jobs: tags: ${{ steps.meta-main.outputs.tags || steps.meta-version.outputs.tags }} steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - if: startsWith(github.ref, 'refs/tags/v') || (github.ref == 'refs/heads/main') name: Docker meta edge and version tag id: meta-main @@ -55,7 +55,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml index a0de1486f0..407ce09cdc 100644 --- a/.github/workflows/mkdocs.yml +++ b/.github/workflows/mkdocs.yml @@ -21,7 +21,7 @@ jobs: outputs: major: ${{ steps.tagged_version.outputs.major || steps.current_version.outputs.major }} steps: - - uses: actions/checkout@v3.5.3 + - uses: actions/checkout@v3.6.0 - uses: actions/setup-node@v3 with: node-version: '16.x' @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest needs: mkdocs-prep steps: - - uses: actions/checkout@v3.5.3 + - uses: actions/checkout@v3.6.0 - uses: actions/setup-python@v4 with: python-version: 3.x @@ -63,7 +63,7 @@ jobs: needs: [mkdocs-prep, mkdocs] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.5.3 + - uses: actions/checkout@v3.6.0 - uses: actions/setup-node@v3 with: node-version: '16.x' diff --git a/.github/workflows/npm-test.yml b/.github/workflows/npm-test.yml index cae1eb0e05..45dcbfe5b1 100644 --- a/.github/workflows/npm-test.yml +++ b/.github/workflows/npm-test.yml @@ -7,7 +7,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.5.3 + - uses: actions/checkout@v3.6.0 - uses: actions/setup-node@v3 with: node-version: '16.x' @@ -38,7 +38,7 @@ jobs: - name: Ensure line endings are consistent run: git config --global core.autocrlf input - name: Check out repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Install dependencies and run build scripts run: npm ci - name: Type-check tests @@ -81,7 +81,7 @@ jobs: with: node-version: ${{ matrix.node-version }} - name: Check out repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Install dependencies and run build scripts run: npm ci - name: Run integration tests @@ -105,7 +105,7 @@ jobs: - name: Ensure line endings are consistent run: git config --global core.autocrlf input - name: Check out repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Install dependencies and run build scripts run: npm ci - name: Run integration tests @@ -127,7 +127,7 @@ jobs: with: node-version: '16.x' - name: Check out repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Install dependencies and run build scripts run: npm ci - name: Run deploy tests From 9e682f5c4f8ecd222ae633137ca455b1b9c5ce16 Mon Sep 17 00:00:00 2001 From: surilindur Date: Mon, 28 Aug 2023 11:11:24 +0200 Subject: [PATCH 05/25] feat: Have FixedContentTypeMapper ignore .meta --- src/storage/mapping/BaseFileIdentifierMapper.ts | 8 +++++--- src/storage/mapping/FixedContentTypeMapper.ts | 4 ++-- test/unit/storage/mapping/FixedContentTypeMapper.test.ts | 9 +++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/storage/mapping/BaseFileIdentifierMapper.ts b/src/storage/mapping/BaseFileIdentifierMapper.ts index 2f88c373cf..921ffb6015 100644 --- a/src/storage/mapping/BaseFileIdentifierMapper.ts +++ b/src/storage/mapping/BaseFileIdentifierMapper.ts @@ -24,6 +24,8 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper { protected readonly rootFilepath: string; // Extension to use as a fallback when the media type is not supported (could be made configurable). protected readonly unknownMediaTypeExtension = 'unknown'; + // Path suffix for metadata + private readonly metadataSuffix = '.meta'; public constructor(base: string, rootFilepath: string) { this.baseRequestURI = trimTrailingSlashes(base); @@ -44,7 +46,7 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper { Promise { let path = this.getRelativePath(identifier); if (isMetadata) { - path += '.meta'; + path += this.metadataSuffix; } this.validateRelativePath(path, identifier); @@ -125,7 +127,7 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper { } const isMetadata = this.isMetadataPath(filePath); if (isMetadata) { - url = url.slice(0, -'.meta'.length); + url = url.slice(0, -this.metadataSuffix.length); } return { identifier: { path: url }, filePath, contentType, isMetadata }; } @@ -213,6 +215,6 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper { * Checks if the given path is a metadata path. */ protected isMetadataPath(path: string): boolean { - return path.endsWith('.meta'); + return path.endsWith(this.metadataSuffix); } } diff --git a/src/storage/mapping/FixedContentTypeMapper.ts b/src/storage/mapping/FixedContentTypeMapper.ts index 3ca5556a72..0fb6c15456 100644 --- a/src/storage/mapping/FixedContentTypeMapper.ts +++ b/src/storage/mapping/FixedContentTypeMapper.ts @@ -64,8 +64,8 @@ export class FixedContentTypeMapper extends BaseFileIdentifierMapper { } protected async getDocumentUrl(relative: string): Promise { - // Handle path suffix - if (this.pathSuffix) { + // Handle path suffix, but ignore metadata files + if (this.pathSuffix && !this.isMetadataPath(relative)) { if (relative.endsWith(this.pathSuffix)) { relative = relative.slice(0, -this.pathSuffix.length); } else { diff --git a/test/unit/storage/mapping/FixedContentTypeMapper.test.ts b/test/unit/storage/mapping/FixedContentTypeMapper.test.ts index bc24b83a21..0a2414db30 100644 --- a/test/unit/storage/mapping/FixedContentTypeMapper.test.ts +++ b/test/unit/storage/mapping/FixedContentTypeMapper.test.ts @@ -183,6 +183,15 @@ describe('An FixedContentTypeMapper', (): void => { await expect(mapper.mapFilePathToUrl(`${rootFilepath}test`, false)).rejects.toThrow(NotFoundHttpError); await expect(mapper.mapFilePathToUrl(`${rootFilepath}test.txt`, false)).rejects.toThrow(NotFoundHttpError); }); + + it('returns a generate file path for metadata regardless of the suffix.', async(): Promise => { + await expect(mapper.mapFilePathToUrl(`${rootFilepath}.meta`, false)).resolves.toEqual({ + identifier: { path: `${base}` }, + filePath: `${rootFilepath}.meta`, + contentType: 'text/turtle', + isMetadata: true, + }); + }); }); }); From b1c360409dfa3e0a70bb58595fcb58f14a084dab Mon Sep 17 00:00:00 2001 From: surilindur Date: Tue, 29 Aug 2023 09:27:12 +0200 Subject: [PATCH 06/25] chore: Fix typo in test description --- test/unit/storage/mapping/FixedContentTypeMapper.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/storage/mapping/FixedContentTypeMapper.test.ts b/test/unit/storage/mapping/FixedContentTypeMapper.test.ts index 0a2414db30..575da27c24 100644 --- a/test/unit/storage/mapping/FixedContentTypeMapper.test.ts +++ b/test/unit/storage/mapping/FixedContentTypeMapper.test.ts @@ -184,7 +184,7 @@ describe('An FixedContentTypeMapper', (): void => { await expect(mapper.mapFilePathToUrl(`${rootFilepath}test.txt`, false)).rejects.toThrow(NotFoundHttpError); }); - it('returns a generate file path for metadata regardless of the suffix.', async(): Promise => { + it('returns a generated file path for metadata regardless of the suffix.', async(): Promise => { await expect(mapper.mapFilePathToUrl(`${rootFilepath}.meta`, false)).resolves.toEqual({ identifier: { path: `${base}` }, filePath: `${rootFilepath}.meta`, From a35500314b82bc9b08b1d1f451ff2a78423d9143 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Wed, 30 Aug 2023 09:18:46 +0200 Subject: [PATCH 07/25] chore(release): Release version 6.0.2 of the npm package --- CHANGELOG.md | 18 ++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c01553ef2b..d4e547b2b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,24 @@ All notable changes to this project will be documented in this file. +## [6.0.2](https://github.com/CommunitySolidServer/CommunitySolidServer/compare/v6.0.1...v6.0.2) (2023-08-30) + +### Fixes + +* Have FixedContentTypeMapper ignore .meta ([9e682f5](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/9e682f5c4f8ecd222ae633137ca455b1b9c5ce16)) +* Ignore invalid header parts ([9c2c5ed](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/9c2c5edaf514fe84594024710c03ab3b7b0fbed1)) +* Do not show PUT in Allow header for existing containers ([6f6784a](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/6f6784a28873c1a8a71bc8a6a37b634677109f02)) +* Store activity streams context locally ([a47cc8a](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/a47cc8a5eef4d0dd963f85d0ad0e4746ada48e19)) + +### Testing + +* Clear test data folder before running tests ([6fc3f2c](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/6fc3f2cf4f23ffde40ba88305c4c67bf39b73e10)) +* Enable file locker in notification tests ([f419f2f](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/f419f2f28d664fad6d6e16cf89a6ebbd7d0f0052)) + +### Chores + +* Name HTTP handlers ([937c41f](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/937c41fd17d553ddfc0d8f140867c252ec113ccb)) + ## [6.0.1](https://github.com/CommunitySolidServer/CommunitySolidServer/compare/v6.0.0...v6.0.1) (2023-06-15) ### Fixes diff --git a/package-lock.json b/package-lock.json index a4b7fbb144..f6695e21f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@solid/community-server", - "version": "6.0.1", + "version": "6.0.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@solid/community-server", - "version": "6.0.1", + "version": "6.0.2", "license": "MIT", "dependencies": { "@comunica/context-entries": "^2.6.8", diff --git a/package.json b/package.json index 7ba014c4b7..a1e842c0b1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@solid/community-server", - "version": "6.0.1", + "version": "6.0.2", "description": "Community Solid Server: an open and modular implementation of the Solid specifications", "keywords": [ "solid", From 9bdaa7949eedb25e068b53a36920dc69aeda97b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 00:30:07 +0000 Subject: [PATCH 08/25] chore(deps): bump actions/checkout from 3.6.0 to 4.0.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.6.0 to 4.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.6.0...v4.0.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cth-test.yml | 2 +- .github/workflows/docker.yml | 4 ++-- .github/workflows/mkdocs.yml | 6 +++--- .github/workflows/npm-test.yml | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cth-test.yml b/.github/workflows/cth-test.yml index aee4a96acf..a1a9fd5ec5 100644 --- a/.github/workflows/cth-test.yml +++ b/.github/workflows/cth-test.yml @@ -42,7 +42,7 @@ jobs: with: node-version: 16.x - name: Check out the project - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 with: ref: ${{ inputs.branch || github.ref }} - name: Install dependencies and run build scripts diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 09b7f02cba..c91e6f4015 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,7 +21,7 @@ jobs: tags: ${{ steps.meta-main.outputs.tags || steps.meta-version.outputs.tags }} steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - if: startsWith(github.ref, 'refs/tags/v') || (github.ref == 'refs/heads/main') name: Docker meta edge and version tag id: meta-main @@ -55,7 +55,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml index 407ce09cdc..1f59833ce7 100644 --- a/.github/workflows/mkdocs.yml +++ b/.github/workflows/mkdocs.yml @@ -21,7 +21,7 @@ jobs: outputs: major: ${{ steps.tagged_version.outputs.major || steps.current_version.outputs.major }} steps: - - uses: actions/checkout@v3.6.0 + - uses: actions/checkout@v4.0.0 - uses: actions/setup-node@v3 with: node-version: '16.x' @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest needs: mkdocs-prep steps: - - uses: actions/checkout@v3.6.0 + - uses: actions/checkout@v4.0.0 - uses: actions/setup-python@v4 with: python-version: 3.x @@ -63,7 +63,7 @@ jobs: needs: [mkdocs-prep, mkdocs] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.6.0 + - uses: actions/checkout@v4.0.0 - uses: actions/setup-node@v3 with: node-version: '16.x' diff --git a/.github/workflows/npm-test.yml b/.github/workflows/npm-test.yml index 45dcbfe5b1..c14f94d887 100644 --- a/.github/workflows/npm-test.yml +++ b/.github/workflows/npm-test.yml @@ -7,7 +7,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.6.0 + - uses: actions/checkout@v4.0.0 - uses: actions/setup-node@v3 with: node-version: '16.x' @@ -38,7 +38,7 @@ jobs: - name: Ensure line endings are consistent run: git config --global core.autocrlf input - name: Check out repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Install dependencies and run build scripts run: npm ci - name: Type-check tests @@ -81,7 +81,7 @@ jobs: with: node-version: ${{ matrix.node-version }} - name: Check out repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Install dependencies and run build scripts run: npm ci - name: Run integration tests @@ -105,7 +105,7 @@ jobs: - name: Ensure line endings are consistent run: git config --global core.autocrlf input - name: Check out repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Install dependencies and run build scripts run: npm ci - name: Run integration tests @@ -127,7 +127,7 @@ jobs: with: node-version: '16.x' - name: Check out repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Install dependencies and run build scripts run: npm ci - name: Run deploy tests From b86c3b4deeac8424fcf954acddf9903129f38c3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 00:24:08 +0000 Subject: [PATCH 09/25] chore(deps): bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c91e6f4015..0425be8d45 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -61,7 +61,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From 89e48e7f3fe3d664d03af1d3b43b7434156ad4c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 00:24:12 +0000 Subject: [PATCH 10/25] chore(deps): bump docker/setup-qemu-action from 2 to 3 Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0425be8d45..02ee6b9e9c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -57,7 +57,7 @@ jobs: - name: Checkout uses: actions/checkout@v4.0.0 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Login to DockerHub From 19713513911f7baebf37281926ce935fab361851 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 00:24:16 +0000 Subject: [PATCH 11/25] chore(deps): bump docker/metadata-action from 4 to 5 Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 4 to 5. - [Release notes](https://github.com/docker/metadata-action/releases) - [Upgrade guide](https://github.com/docker/metadata-action/blob/master/UPGRADE.md) - [Commits](https://github.com/docker/metadata-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 02ee6b9e9c..e7c77bd7fd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: - if: startsWith(github.ref, 'refs/tags/v') || (github.ref == 'refs/heads/main') name: Docker meta edge and version tag id: meta-main - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: | solidproject/community-server @@ -39,7 +39,7 @@ jobs: - if: startsWith(github.ref, 'refs/heads/versions/') name: Docker meta next id: meta-version - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: | solidproject/community-server From 9ba85c710b79b37fade9100b367b4cc42cb4e964 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 00:24:20 +0000 Subject: [PATCH 12/25] chore(deps): bump docker/build-push-action from 4 to 5 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4 to 5. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e7c77bd7fd..dcc7ef0d1e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -66,7 +66,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and export to docker - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . load: true @@ -85,7 +85,7 @@ jobs: done <<< "${{ needs.docker-meta.outputs.tags }}"; - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . push: true From 0a7a9b21a223e983d1e064da769bf8a3c7cfc02c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 00:24:24 +0000 Subject: [PATCH 13/25] chore(deps): bump docker/setup-buildx-action from 2 to 3 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dcc7ef0d1e..5d8b7099a3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -59,7 +59,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub uses: docker/login-action@v3 with: From 4f095243df7321de8585a0321eb8700a174bf950 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 00:51:13 +0000 Subject: [PATCH 14/25] chore(deps): bump actions/checkout from 4.0.0 to 4.1.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/cth-test.yml | 2 +- .github/workflows/docker.yml | 4 ++-- .github/workflows/mkdocs.yml | 6 +++--- .github/workflows/npm-test.yml | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cth-test.yml b/.github/workflows/cth-test.yml index a1a9fd5ec5..210f162c51 100644 --- a/.github/workflows/cth-test.yml +++ b/.github/workflows/cth-test.yml @@ -42,7 +42,7 @@ jobs: with: node-version: 16.x - name: Check out the project - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 with: ref: ${{ inputs.branch || github.ref }} - name: Install dependencies and run build scripts diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5d8b7099a3..e4d7b9a403 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,7 +21,7 @@ jobs: tags: ${{ steps.meta-main.outputs.tags || steps.meta-version.outputs.tags }} steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - if: startsWith(github.ref, 'refs/tags/v') || (github.ref == 'refs/heads/main') name: Docker meta edge and version tag id: meta-main @@ -55,7 +55,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml index 1f59833ce7..9dc3ca2866 100644 --- a/.github/workflows/mkdocs.yml +++ b/.github/workflows/mkdocs.yml @@ -21,7 +21,7 @@ jobs: outputs: major: ${{ steps.tagged_version.outputs.major || steps.current_version.outputs.major }} steps: - - uses: actions/checkout@v4.0.0 + - uses: actions/checkout@v4.1.0 - uses: actions/setup-node@v3 with: node-version: '16.x' @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest needs: mkdocs-prep steps: - - uses: actions/checkout@v4.0.0 + - uses: actions/checkout@v4.1.0 - uses: actions/setup-python@v4 with: python-version: 3.x @@ -63,7 +63,7 @@ jobs: needs: [mkdocs-prep, mkdocs] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.0.0 + - uses: actions/checkout@v4.1.0 - uses: actions/setup-node@v3 with: node-version: '16.x' diff --git a/.github/workflows/npm-test.yml b/.github/workflows/npm-test.yml index c14f94d887..d6b95daf8a 100644 --- a/.github/workflows/npm-test.yml +++ b/.github/workflows/npm-test.yml @@ -7,7 +7,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.0.0 + - uses: actions/checkout@v4.1.0 - uses: actions/setup-node@v3 with: node-version: '16.x' @@ -38,7 +38,7 @@ jobs: - name: Ensure line endings are consistent run: git config --global core.autocrlf input - name: Check out repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Install dependencies and run build scripts run: npm ci - name: Type-check tests @@ -81,7 +81,7 @@ jobs: with: node-version: ${{ matrix.node-version }} - name: Check out repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Install dependencies and run build scripts run: npm ci - name: Run integration tests @@ -105,7 +105,7 @@ jobs: - name: Ensure line endings are consistent run: git config --global core.autocrlf input - name: Check out repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Install dependencies and run build scripts run: npm ci - name: Run integration tests @@ -127,7 +127,7 @@ jobs: with: node-version: '16.x' - name: Check out repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Install dependencies and run build scripts run: npm ci - name: Run deploy tests From d31393f4751dd3f023110ead4e47a01ac15da2af Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Tue, 3 Oct 2023 11:19:47 +0200 Subject: [PATCH 15/25] fix: Remove URL encoding from base64 strings before decoding --- src/storage/keyvalue/EncodingPathStorage.ts | 10 +++++++++- .../storage/keyvalue/EncodingPathStorage.test.ts | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/storage/keyvalue/EncodingPathStorage.ts b/src/storage/keyvalue/EncodingPathStorage.ts index da79e351e3..867fda1818 100644 --- a/src/storage/keyvalue/EncodingPathStorage.ts +++ b/src/storage/keyvalue/EncodingPathStorage.ts @@ -62,7 +62,15 @@ export class EncodingPathStorage implements KeyValueStorage { * Converts an internal storage path string into the original path key. */ protected pathToKey(path: string): string { - const buffer = Buffer.from(path.slice(this.basePath.length), 'base64'); + // While the main part of a base64 encoded string is same from any changes from encoding or decoding URL parts, + // the `=` symbol that is used for padding is not. + // This can cause incorrect results when calling these function, + // where the original path contains `YXBwbGU%3D` instead of `YXBwbGU=`. + // This does not create any issues when the source store does not encode the string, so is safe to always call. + // For consistency, we might want to also always encode when creating the path in `keyToPath()`, + // but that would potentially break existing implementations that do not do encoding, + // and is not really necessary to solve any issues. + const buffer = Buffer.from(decodeURIComponent(path.slice(this.basePath.length)), 'base64'); return buffer.toString('utf-8'); } } diff --git a/test/unit/storage/keyvalue/EncodingPathStorage.test.ts b/test/unit/storage/keyvalue/EncodingPathStorage.test.ts index 6e1c7abe5e..0ad2212525 100644 --- a/test/unit/storage/keyvalue/EncodingPathStorage.test.ts +++ b/test/unit/storage/keyvalue/EncodingPathStorage.test.ts @@ -48,4 +48,20 @@ describe('An EncodingPathStorage', (): void => { expect(results).toHaveLength(1); expect(results[0]).toEqual([ 'key', data ]); }); + + it('correctly handles keys that have been encoded by the source storage.', async(): Promise => { + // Base 64 encoding of 'apple' + const encodedKey = 'YXBwbGU='; + const generatedPath = `${relativePath}${encodeURIComponent(encodedKey)}`; + const data = 'data'; + + map.set(generatedPath, data); + + const results = []; + for await (const entry of storage.entries()) { + results.push(entry); + } + expect(results).toHaveLength(1); + expect(results[0]).toEqual([ 'apple', data ]); + }); }); From 1fa6d248a2e500c025794f4e3ed6cc504ed77f10 Mon Sep 17 00:00:00 2001 From: Ruben Verborgh Date: Mon, 2 Oct 2023 11:29:34 +0200 Subject: [PATCH 16/25] docs: Add quick start to README. --- README.md | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 6b9e75bed5..b9e0904fd4 100644 --- a/README.md +++ b/README.md @@ -29,22 +29,27 @@ the Community Solid Server is a great companion: And, of course, for many others who like to experience Solid. -You can install the software locally or on your server -and get started with Solid immediately. +## ⚡ Running the Community Solid Server -## ⚡ Running the server +### 🏃‍♀️ Quickly spin up your Solid server -To run the server, you will need [Node.js](https://nodejs.org/en/). -We support versions 14.14 and up. +Use [Node.js](https://nodejs.org/en/) 14.14 or up and execute: -If you do not use Node.js, -you can run a [Docker](https://www.docker.com/) version instead. +```shell +npx @solid/community-server +``` + +Now visit your brand new server at [http://localhost:3000/](http://localhost:3000/)! + +To persist your pod's contents between restarts, use: + +```shell +npx @solid/community-server -c @css:config/file.json -f data/ +``` -### 💻 Installing and running locally +### 💻 Installing locally -After installing Node.js, -install the latest server version -from the [npm package repository](https://www.npmjs.com/): +Install the npm package globally with: ```shell npm install -g @solid/community-server @@ -59,10 +64,10 @@ community-solid-server # add parameters if needed To run the server with your current folder as storage, use: ```shell -community-solid-server -c @css:config/file.json +community-solid-server -c @css:config/file.json -f data/ ``` -### 📃 Installing and running from source +### 📃 Running from source If you rather prefer to run the latest source code version, or if you want to try a specific [branch](https://www.npmjs.com/) of the code, From e424b8488261bc8942d82a7fe2d92a94650e93b9 Mon Sep 17 00:00:00 2001 From: Ruben Verborgh Date: Mon, 2 Oct 2023 11:43:37 +0200 Subject: [PATCH 17/25] docs: Add starting guide. --- documentation/markdown/README.md | 1 + .../markdown/usage/starting-server.md | 121 ++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 documentation/markdown/usage/starting-server.md diff --git a/documentation/markdown/README.md b/documentation/markdown/README.md index 5a83ef2976..1c85006216 100644 --- a/documentation/markdown/README.md +++ b/documentation/markdown/README.md @@ -29,6 +29,7 @@ the [changelog](https://github.com/CommunitySolidServer/CommunitySolidServer/blo ## Using the server +* [Quickly starting the server](usage/starting-server.md) * [Basic example HTTP requests](usage/example-requests.md) * [Editing the metadata of a resource](usage/metadata.md) * [How to use the Identity Provider](usage/identity-provider.md) diff --git a/documentation/markdown/usage/starting-server.md b/documentation/markdown/usage/starting-server.md new file mode 100644 index 0000000000..fa8863e13c --- /dev/null +++ b/documentation/markdown/usage/starting-server.md @@ -0,0 +1,121 @@ +# Starting the Community Solid Server + +## Quickly spinning up a server + +Use [Node.js](https://nodejs.org/en/) 14.14 or up and execute: + +```shell +npx @solid/community-server +``` + +Now visit your brand new server at [http://localhost:3000/](http://localhost:3000/)! + +To persist your pod's contents between restarts, use: + +```shell +npx @solid/community-server -c @css:config/file.json -f data/ +``` + +## Local installation + +Install the npm package globally with: + +```shell +npm install -g @solid/community-server +``` + +To run the server with in-memory storage, use: + +```shell +community-solid-server # add parameters if needed +``` + +To run the server with your current folder as storage, use: + +```shell +community-solid-server -c @css:config/file.json -f data/ +``` + +## Configuring the server + +The Community Solid Server is designed to be flexible +such that people can easily run different configurations. +This is useful for customizing the server with plugins, +testing applications in different setups, +or developing new parts for the server +without needing to change its base code. + +An easy way to customize the server is +by passing parameters to the server command. +These parameters give you direct access +to some commonly used settings: + +| parameter name | default value | description | +|-------------------------|----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `--port, -p` | `3000` | The TCP port on which the server should listen. | +| `--baseUrl, -b` | `http://localhost:$PORT/` | The base URL used internally to generate URLs. Change this if your server does not run on `http://localhost:$PORT/`. | +| `--socket` | | The Unix Domain Socket on which the server should listen. `--baseUrl` must be set if this option is provided | +| `--loggingLevel, -l` | `info` | The detail level of logging; useful for debugging problems. Use `debug` for full information. | +| `--config, -c` | `@css:config/default.json` | The configuration(s) for the server. The default only stores data in memory; to persist to your filesystem, use `@css:config/file.json` | +| `--rootFilePath, -f` | `./` | Root folder where the server stores data, when using a file-based configuration. | +| `--sparqlEndpoint, -s` | | URL of the SPARQL endpoint, when using a quadstore-based configuration. | +| `--showStackTrace, -t` | false | Enables detailed logging on error output. | +| `--podConfigJson` | `./pod-config.json` | Path to the file that keeps track of dynamic Pod configurations. Only relevant when using `@css:config/dynamic.json`. | +| `--seededPodConfigJson` | | Path to the file that keeps track of seeded Pod configurations. | +| `--mainModulePath, -m` | | Path from where Components.js will start its lookup when initializing configurations. | +| `--workers, -w` | `1` | Run in multithreaded mode using workers. Special values are `-1` (scale to `num_cores-1`), `0` (scale to `num_cores`) and 1 (singlethreaded). | + +Parameters can also be passed through environment variables. + +They are prefixed with `CSS_` and converted from `camelCase` to `CAMEL_CASE` + +> eg. `--showStackTrace` => `CSS_SHOW_STACK_TRACE` + +Command-line arguments will always override environment variables. + +## Alternative ways to run the server + +### From source + +If you rather prefer to run the latest source code version, +or if you want to try a specific [branch](https://www.npmjs.com/) of the code, +you can use: + +```shell +git clone https://github.com/CommunitySolidServer/CommunitySolidServer.git +cd CommunitySolidServer +npm ci +npm start -- # add parameters if needed +``` + +### Via Docker + +Docker allows you to run the server without having Node.js installed. Images are built on each tagged version and hosted +on [Docker Hub](https://hub.docker.com/r/solidproject/community-server). + +```shell +# Clone the repo to get access to the configs +git clone https://github.com/CommunitySolidServer/CommunitySolidServer.git +cd CommunitySolidServer +# Run the image, serving your `~/Solid` directory on `http://localhost:3000` +docker run --rm -v ~/Solid:/data -p 3000:3000 -it solidproject/community-server:latest +# Or use one of the built-in configurations +docker run --rm -p 3000:3000 -it solidproject/community-server -c config/default.json +# Or use your own configuration mapped to the right directory +docker run --rm -v ~/solid-config:/config -p 3000:3000 -it solidproject/community-server -c /config/my-config.json +# Or use environment variables to configure your css instance +docker run --rm -v ~/Solid:/data -p 3000:3000 -it -e CSS_CONFIG=config/file-no-setup.json -e CSS_LOGGING_LEVEL=debug solidproject/community-server +``` + +### Using a Helm Chart + +The official [Helm](https://helm.sh/) Chart for Kubernetes deployment is maintained at +[CommunitySolidServer/css-helm-chart](https://github.com/CommunitySolidServer/css-helm-chart) and published on +[ArtifactHUB](https://artifacthub.io/packages/helm/community-solid-server/community-solid-server). +There you will find complete installation instructions. + +```shell +# Summary +helm repo add community-solid-server https://communitysolidserver.github.io/css-helm-chart/charts/ +helm install my-css community-solid-server/community-solid-server +``` From d618f9781af80b1697d5fe23f50e3f186954792b Mon Sep 17 00:00:00 2001 From: Ruben Verborgh Date: Mon, 2 Oct 2023 11:43:59 +0200 Subject: [PATCH 18/25] docs: Simplify README by pointing to our docs. --- README.md | 132 +++--------------------------------------------------- 1 file changed, 7 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index b9e0904fd4..5b93a0c7f1 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ the Community Solid Server is a great companion: - 🧑🏽 **for people** who want to try out having their own Pod -- 👨🏿‍💻 **for developers** who want to create and test Solid apps +- 👨🏿‍💻 **for developers** who want to quickly create and test Solid apps - 👩🏻‍🔬 **for researchers** who want to design new features for Solid @@ -31,8 +31,6 @@ And, of course, for many others who like to experience Solid. ## ⚡ Running the Community Solid Server -### 🏃‍♀️ Quickly spin up your Solid server - Use [Node.js](https://nodejs.org/en/) 14.14 or up and execute: ```shell @@ -47,137 +45,21 @@ To persist your pod's contents between restarts, use: npx @solid/community-server -c @css:config/file.json -f data/ ``` -### 💻 Installing locally - -Install the npm package globally with: - -```shell -npm install -g @solid/community-server -``` +Find more ways to run the server in the [documentation](https://communitysolidserver.github.io/CommunitySolidServer/6.x/usage/starting-server/). -To run the server with in-memory storage, use: +## 🔧 Configure your server -```shell -community-solid-server # add parameters if needed -``` - -To run the server with your current folder as storage, use: - -```shell -community-solid-server -c @css:config/file.json -f data/ -``` - -### 📃 Running from source - -If you rather prefer to run the latest source code version, -or if you want to try a specific [branch](https://www.npmjs.com/) of the code, -you can use: - -```shell -git clone https://github.com/CommunitySolidServer/CommunitySolidServer.git -cd CommunitySolidServer -npm ci -npm start -- # add parameters if needed -``` - -### 📦 Running via Docker - -Docker allows you to run the server without having Node.js installed. Images are built on each tagged version and hosted -on [Docker Hub](https://hub.docker.com/r/solidproject/community-server). - -```shell -# Clone the repo to get access to the configs -git clone https://github.com/CommunitySolidServer/CommunitySolidServer.git -cd CommunitySolidServer -# Run the image, serving your `~/Solid` directory on `http://localhost:3000` -docker run --rm -v ~/Solid:/data -p 3000:3000 -it solidproject/community-server:latest -# Or use one of the built-in configurations -docker run --rm -p 3000:3000 -it solidproject/community-server -c config/default.json -# Or use your own configuration mapped to the right directory -docker run --rm -v ~/solid-config:/config -p 3000:3000 -it solidproject/community-server -c /config/my-config.json -# Or use environment variables to configure your css instance -docker run --rm -v ~/Solid:/data -p 3000:3000 -it -e CSS_CONFIG=config/file-no-setup.json -e CSS_LOGGING_LEVEL=debug solidproject/community-server -``` - -### 🗃️ Helm Chart - -The official [Helm](https://helm.sh/) Chart for Kubernetes deployment is maintained at -[CommunitySolidServer/css-helm-chart](https://github.com/CommunitySolidServer/css-helm-chart) and published on -[ArtifactHUB](https://artifacthub.io/packages/helm/community-solid-server/community-solid-server). -There you will find complete installation instructions. - -```shell -# Summary -helm repo add community-solid-server https://communitysolidserver.github.io/css-helm-chart/charts/ -helm install my-css community-solid-server/community-solid-server -``` - -## 🔧 Configuring the server - -The Community Solid Server is designed to be flexible -such that people can easily run different configurations. -This is useful for customizing the server with plugins, -testing applications in different setups, -or developing new parts for the server -without needing to change its base code. - -### ⏱ Parameters - -An easy way to customize the server is -by passing parameters to the server command. -These parameters give you direct access -to some commonly used settings: - -| parameter name | default value | description | -|-------------------------|----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| -| `--port, -p` | `3000` | The TCP port on which the server should listen. | -| `--baseUrl, -b` | `http://localhost:$PORT/` | The base URL used internally to generate URLs. Change this if your server does not run on `http://localhost:$PORT/`. | -| `--socket` | | The Unix Domain Socket on which the server should listen. `--baseUrl` must be set if this option is provided | -| `--loggingLevel, -l` | `info` | The detail level of logging; useful for debugging problems. Use `debug` for full information. | -| `--config, -c` | `@css:config/default.json` | The configuration(s) for the server. The default only stores data in memory; to persist to your filesystem, use `@css:config/file.json` | -| `--rootFilePath, -f` | `./` | Root folder where the server stores data, when using a file-based configuration. | -| `--sparqlEndpoint, -s` | | URL of the SPARQL endpoint, when using a quadstore-based configuration. | -| `--showStackTrace, -t` | false | Enables detailed logging on error output. | -| `--podConfigJson` | `./pod-config.json` | Path to the file that keeps track of dynamic Pod configurations. Only relevant when using `@css:config/dynamic.json`. | -| `--seededPodConfigJson` | | Path to the file that keeps track of seeded Pod configurations. | -| `--mainModulePath, -m` | | Path from where Components.js will start its lookup when initializing configurations. | -| `--workers, -w` | `1` | Run in multithreaded mode using workers. Special values are `-1` (scale to `num_cores-1`), `0` (scale to `num_cores`) and 1 (singlethreaded). | - -### 🔀 Multithreading - -The Community Solid Server can be started in multithreaded mode with any config. The config must only contain components -that are threadsafe though. If a non-threadsafe component is used in multithreaded mode, the server will describe with -an error which class is the culprit. - -```shell -# Running multithreaded with autoscaling to number of logical cores minus 1 -npm start -- -c config/file.json -w -1 -``` - -### 🖥️ Environment variables - -Parameters can also be passed through environment variables. - -They are prefixed with `CSS_` and converted from `camelCase` to `CAMEL_CASE` - -> eg. `--showStackTrace` => `CSS_SHOW_STACK_TRACE` - -**Note: command-line arguments will always override environment variables!** - -### 🧶 Custom configurations - -More substantial changes to server behavior can be achieved -by writing new configuration files in JSON-LD. +Substantial changes to server behavior can be achieved via JSON configuration files. The Community Solid Server uses [Components.js](https://componentsjs.readthedocs.io/en/latest/) to specify how modules and components need to be wired together at runtime. -Examples and guidance on configurations +Recipes for configuring the server can be found at [CommunitySolidServer/recipes](https://github.com/CommunitySolidServer/recipes). + +Examples and guidance on custom configurations are available in the [`config` folder](https://github.com/CommunitySolidServer/CommunitySolidServer/tree/main/config), and the [configurations tutorial](https://github.com/CommunitySolidServer/tutorials/blob/main/custom-configurations.md). There is also a [configuration generator](https://communitysolidserver.github.io/configuration-generator/). -Recipes for configuring the server can be found at [CommunitySolidServer/recipes](https://github.com/CommunitySolidServer/recipes). - ## 👩🏽‍💻 Developing server code The server allows writing and plugging in custom modules From db66e3df7559d78d129da4628e1cff360fa0602f Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Thu, 5 Oct 2023 09:46:09 +0200 Subject: [PATCH 19/25] test: Consistently clean up folder before stopping server There seems to be some integration tests that sometimes fail on tests where the order is reversed. Hopefully this resolves the problem. --- test/integration/Quota.test.ts | 4 ++-- test/integration/ResourceLockCleanup.test.ts | 5 +---- test/integration/SeedingPods.test.ts | 2 +- test/integration/WebHookChannel2023.test.ts | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/test/integration/Quota.test.ts b/test/integration/Quota.test.ts index 81b429db5b..66ebaf55ad 100644 --- a/test/integration/Quota.test.ts +++ b/test/integration/Quota.test.ts @@ -112,8 +112,8 @@ describe('A quota server', (): void => { }); afterAll(async(): Promise => { - await app.stop(); await removeFolder(rootFilePath); + await app.stop(); }); // Test quota in the first pod @@ -198,8 +198,8 @@ describe('A quota server', (): void => { }); afterAll(async(): Promise => { - await app.stop(); await removeFolder(rootFilePath); + await app.stop(); }); it('should return 413 when global quota is exceeded.', async(): Promise => { diff --git a/test/integration/ResourceLockCleanup.test.ts b/test/integration/ResourceLockCleanup.test.ts index b6ba484f8c..4a054dd750 100644 --- a/test/integration/ResourceLockCleanup.test.ts +++ b/test/integration/ResourceLockCleanup.test.ts @@ -53,11 +53,8 @@ void => { }); afterAll(async(): Promise => { - // Stop the server - await app.stop(); - - // Execute the configured teardown await teardown(); + await app.stop(); }); it('should not be affected by dangling locks.', async(): Promise => { diff --git a/test/integration/SeedingPods.test.ts b/test/integration/SeedingPods.test.ts index 349739fd27..9a49e411e9 100644 --- a/test/integration/SeedingPods.test.ts +++ b/test/integration/SeedingPods.test.ts @@ -45,8 +45,8 @@ describe('A server with seeded pods', (): void => { }); afterAll(async(): Promise => { - await app.stop(); await removeFolder(rootFilePath); + await app.stop(); }); it('has created the requested pods.', async(): Promise => { diff --git a/test/integration/WebHookChannel2023.test.ts b/test/integration/WebHookChannel2023.test.ts index 3a78d128a8..9cea59371e 100644 --- a/test/integration/WebHookChannel2023.test.ts +++ b/test/integration/WebHookChannel2023.test.ts @@ -74,8 +74,8 @@ describe.each(stores)('A server supporting WebHookChannel2023 using %s', (name, afterAll(async(): Promise => { clientServer.close(); - await app.stop(); await teardown(); + await app.stop(); }); it('links to the storage description.', async(): Promise => { From 3e9adef4cf00d0776c0d371f835a31511db7427b Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Fri, 29 Sep 2023 16:32:07 +0200 Subject: [PATCH 20/25] feat: Add support for range headers --- .componentsignore | 3 + .eslintrc.js | 5 ++ .../ldp/handler/components/preferences.json | 6 +- config/ldp/metadata-writer/default.json | 2 + config/ldp/metadata-writer/writers/range.json | 10 +++ config/storage/middleware/default.json | 6 ++ .../preferences/AcceptPreferenceParser.ts | 2 +- .../preferences/RangePreferenceParser.ts | 48 ++++++++++ .../preferences/UnionPreferenceParser.ts | 32 +++++++ .../output/metadata/RangeMetadataWriter.ts | 25 ++++++ .../output/response/OkResponseDescription.ts | 6 +- .../RepresentationPreferences.ts | 2 + src/index.ts | 6 ++ src/storage/BinarySliceResourceStore.ts | 66 ++++++++++++++ src/util/SliceStream.ts | 87 +++++++++++++++++++ src/util/Vocabularies.ts | 4 + src/util/errors/RangeNotSatisfiedHttpError.ts | 19 ++++ .../integration/LdpHandlerWithoutAuth.test.ts | 25 +++++- .../preferences/RangePreferenceParser.test.ts | 40 +++++++++ .../preferences/UnionPreferenceParser.test.ts | 54 ++++++++++++ .../unit/http/ldp/GetOperationHandler.test.ts | 13 +++ .../metadata/RangeMetadataWriter.test.ts | 40 +++++++++ .../storage/BinarySliceResourceStore.test.ts | 80 +++++++++++++++++ test/unit/util/SliceStream.test.ts | 41 +++++++++ test/unit/util/errors/HttpError.test.ts | 2 + 25 files changed, 619 insertions(+), 5 deletions(-) create mode 100644 config/ldp/metadata-writer/writers/range.json create mode 100644 src/http/input/preferences/RangePreferenceParser.ts create mode 100644 src/http/input/preferences/UnionPreferenceParser.ts create mode 100644 src/http/output/metadata/RangeMetadataWriter.ts create mode 100644 src/storage/BinarySliceResourceStore.ts create mode 100644 src/util/SliceStream.ts create mode 100644 src/util/errors/RangeNotSatisfiedHttpError.ts create mode 100644 test/unit/http/input/preferences/RangePreferenceParser.test.ts create mode 100644 test/unit/http/input/preferences/UnionPreferenceParser.test.ts create mode 100644 test/unit/http/output/metadata/RangeMetadataWriter.test.ts create mode 100644 test/unit/storage/BinarySliceResourceStore.test.ts create mode 100644 test/unit/util/SliceStream.test.ts diff --git a/.componentsignore b/.componentsignore index 74a2041874..5aa385c02d 100644 --- a/.componentsignore +++ b/.componentsignore @@ -24,6 +24,7 @@ "NotificationChannelType", "PermissionMap", "Promise", + "Readable", "Readonly", "RegExp", "Server", @@ -31,6 +32,8 @@ "Shorthand", "Template", "TemplateEngine", + "Transform", + "TransformOptions", "ValuePreferencesArg", "VariableBindings", "UnionHandler", diff --git a/.eslintrc.js b/.eslintrc.js index 23acb12684..f0c5ab123e 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -10,6 +10,7 @@ module.exports = { ignorePatterns: [ '*.js' ], globals: { AsyncIterable: 'readonly', + BufferEncoding: 'readonly', NodeJS: 'readonly', RequestInit: 'readonly', }, @@ -71,11 +72,15 @@ module.exports = { // Already checked by @typescript-eslint/no-unused-vars 'no-unused-vars': 'off', 'padding-line-between-statements': 'off', + // Forcing destructuring on existing variables causes clunky code + 'prefer-destructuring': 'off', 'prefer-named-capture-group': 'off', // Already generated by TypeScript strict: 'off', 'tsdoc/syntax': 'error', 'unicorn/catch-error-name': 'off', + // Can cause some clunky situations if it forces us to assign to an existing variable + 'unicorn/consistent-destructuring': 'off', 'unicorn/import-index': 'off', 'unicorn/import-style': 'off', // The next 2 some functional programming paradigms diff --git a/config/ldp/handler/components/preferences.json b/config/ldp/handler/components/preferences.json index d619701d78..ca7f456727 100644 --- a/config/ldp/handler/components/preferences.json +++ b/config/ldp/handler/components/preferences.json @@ -3,7 +3,11 @@ "@graph": [ { "@id": "urn:solid-server:default:PreferenceParser", - "@type": "AcceptPreferenceParser" + "@type": "UnionPreferenceParser", + "parsers": [ + { "@type": "AcceptPreferenceParser" }, + { "@type": "RangePreferenceParser" } + ] } ] } diff --git a/config/ldp/metadata-writer/default.json b/config/ldp/metadata-writer/default.json index 4ce5706f11..f90bf63443 100644 --- a/config/ldp/metadata-writer/default.json +++ b/config/ldp/metadata-writer/default.json @@ -7,6 +7,7 @@ "css:config/ldp/metadata-writer/writers/link-rel-metadata.json", "css:config/ldp/metadata-writer/writers/mapped.json", "css:config/ldp/metadata-writer/writers/modified.json", + "css:config/ldp/metadata-writer/writers/range.json", "css:config/ldp/metadata-writer/writers/storage-description.json", "css:config/ldp/metadata-writer/writers/www-auth.json" ], @@ -22,6 +23,7 @@ { "@id": "urn:solid-server:default:MetadataWriter_LinkRelMetadata" }, { "@id": "urn:solid-server:default:MetadataWriter_Mapped" }, { "@id": "urn:solid-server:default:MetadataWriter_Modified" }, + { "@id": "urn:solid-server:default:MetadataWriter_Range" }, { "@id": "urn:solid-server:default:MetadataWriter_StorageDescription" }, { "@id": "urn:solid-server:default:MetadataWriter_WwwAuth" } ] diff --git a/config/ldp/metadata-writer/writers/range.json b/config/ldp/metadata-writer/writers/range.json new file mode 100644 index 0000000000..0406669eff --- /dev/null +++ b/config/ldp/metadata-writer/writers/range.json @@ -0,0 +1,10 @@ +{ + "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^6.0.0/components/context.jsonld", + "@graph": [ + { + "comment": "Adds the Content-Range header if necessary.", + "@id": "urn:solid-server:default:MetadataWriter_Range", + "@type": "RangeMetadataWriter" + } + ] +} diff --git a/config/storage/middleware/default.json b/config/storage/middleware/default.json index df7e741158..61e8a4cf78 100644 --- a/config/storage/middleware/default.json +++ b/config/storage/middleware/default.json @@ -16,6 +16,12 @@ "comment": "Sets up a stack of utility stores used by most instances.", "@id": "urn:solid-server:default:ResourceStore", "@type": "MonitoringStore", + "source": { "@id": "urn:solid-server:default:ResourceStore_BinarySlice" } + }, + { + "comment": "Slices part of binary streams based on the range preferences.", + "@id": "urn:solid-server:default:ResourceStore_BinarySlice", + "@type": "BinarySliceResourceStore", "source": { "@id": "urn:solid-server:default:ResourceStore_Index" } }, { diff --git a/src/http/input/preferences/AcceptPreferenceParser.ts b/src/http/input/preferences/AcceptPreferenceParser.ts index 90fcd8ee99..09e4b08dd7 100644 --- a/src/http/input/preferences/AcceptPreferenceParser.ts +++ b/src/http/input/preferences/AcceptPreferenceParser.ts @@ -11,7 +11,7 @@ import type { RepresentationPreferences } from '../../representation/Representat import { PreferenceParser } from './PreferenceParser'; const parsers: { - name: keyof RepresentationPreferences; + name: Exclude; header: string; parse: (value: string) => AcceptHeader[]; }[] = [ diff --git a/src/http/input/preferences/RangePreferenceParser.ts b/src/http/input/preferences/RangePreferenceParser.ts new file mode 100644 index 0000000000..2efcf2cb1f --- /dev/null +++ b/src/http/input/preferences/RangePreferenceParser.ts @@ -0,0 +1,48 @@ +import type { HttpRequest } from '../../../server/HttpRequest'; +import { BadRequestHttpError } from '../../../util/errors/BadRequestHttpError'; +import type { RepresentationPreferences } from '../../representation/RepresentationPreferences'; +import { PreferenceParser } from './PreferenceParser'; + +/** + * Parses the range header into range preferences. + * If the range corresponds to a suffix-length range, it will be stored in `start` as a negative value. + */ +export class RangePreferenceParser extends PreferenceParser { + public async handle({ request: { headers: { range }}}: { request: HttpRequest }): Promise { + if (!range) { + return {}; + } + + const [ unit, rangeTail ] = range.split('=').map((entry): string => entry.trim()); + if (unit.length === 0) { + throw new BadRequestHttpError(`Missing unit value from range header ${range}`); + } + if (!rangeTail) { + throw new BadRequestHttpError(`Invalid range header format ${range}`); + } + + const ranges = rangeTail.split(',').map((entry): string => entry.trim()); + const parts: { start: number; end?: number }[] = []; + for (const rangeEntry of ranges) { + const [ start, end ] = rangeEntry.split('-').map((entry): string => entry.trim()); + // This can actually be undefined if the split results in less than 2 elements + if (typeof end !== 'string') { + throw new BadRequestHttpError(`Invalid range header format ${range}`); + } + if (start.length === 0) { + if (end.length === 0) { + throw new BadRequestHttpError(`Invalid range header format ${range}`); + } + parts.push({ start: -Number.parseInt(end, 10) }); + } else { + const part: typeof parts[number] = { start: Number.parseInt(start, 10) }; + if (end.length > 0) { + part.end = Number.parseInt(end, 10); + } + parts.push(part); + } + } + + return { range: { unit, parts }}; + } +} diff --git a/src/http/input/preferences/UnionPreferenceParser.ts b/src/http/input/preferences/UnionPreferenceParser.ts new file mode 100644 index 0000000000..32a23f84ce --- /dev/null +++ b/src/http/input/preferences/UnionPreferenceParser.ts @@ -0,0 +1,32 @@ +import { InternalServerError } from '../../../util/errors/InternalServerError'; +import { UnionHandler } from '../../../util/handlers/UnionHandler'; +import type { RepresentationPreferences } from '../../representation/RepresentationPreferences'; +import type { PreferenceParser } from './PreferenceParser'; + +/** + * Combines the results of multiple {@link PreferenceParser}s. + * Will throw an error if multiple parsers return a range as these can't logically be combined. + */ +export class UnionPreferenceParser extends UnionHandler { + public constructor(parsers: PreferenceParser[]) { + super(parsers, false, false); + } + + protected async combine(results: RepresentationPreferences[]): Promise { + const rangeCount = results.filter((result): boolean => Boolean(result.range)).length; + if (rangeCount > 1) { + throw new InternalServerError('Found multiple range values. This implies a misconfiguration.'); + } + + return results.reduce((acc, val): RepresentationPreferences => { + for (const key of Object.keys(val) as (keyof RepresentationPreferences)[]) { + if (key === 'range') { + acc[key] = val[key]; + } else { + acc[key] = { ...acc[key], ...val[key] }; + } + } + return acc; + }, {}); + } +} diff --git a/src/http/output/metadata/RangeMetadataWriter.ts b/src/http/output/metadata/RangeMetadataWriter.ts new file mode 100644 index 0000000000..d67d229eee --- /dev/null +++ b/src/http/output/metadata/RangeMetadataWriter.ts @@ -0,0 +1,25 @@ +import type { HttpResponse } from '../../../server/HttpResponse'; +import { addHeader } from '../../../util/HeaderUtil'; +import { SOLID_HTTP } from '../../../util/Vocabularies'; +import type { RepresentationMetadata } from '../../representation/RepresentationMetadata'; +import { MetadataWriter } from './MetadataWriter'; + +/** + * Generates the necessary `content-range` header if there is range metadata. + * If the start or end is unknown, a `*` will be used instead. + * According to the RFC, this is incorrect, + * but is all we can do as long as we don't know the full length of the representation in advance. + * For the same reason, the total length of the representation will always be `*`. + */ +export class RangeMetadataWriter extends MetadataWriter { + public async handle(input: { response: HttpResponse; metadata: RepresentationMetadata }): Promise { + const unit = input.metadata.get(SOLID_HTTP.terms.unit); + if (!unit) { + return; + } + const start = input.metadata.get(SOLID_HTTP.terms.start); + const end = input.metadata.get(SOLID_HTTP.terms.end); + + addHeader(input.response, 'Content-Range', `${unit.value} ${start?.value ?? '*'}-${end?.value ?? '*'}/*`); + } +} diff --git a/src/http/output/response/OkResponseDescription.ts b/src/http/output/response/OkResponseDescription.ts index c2599184fc..b8ae1d9e46 100644 --- a/src/http/output/response/OkResponseDescription.ts +++ b/src/http/output/response/OkResponseDescription.ts @@ -1,10 +1,12 @@ import type { Readable } from 'stream'; import type { Guarded } from '../../../util/GuardedStream'; +import { SOLID_HTTP } from '../../../util/Vocabularies'; import type { RepresentationMetadata } from '../../representation/RepresentationMetadata'; import { ResponseDescription } from './ResponseDescription'; /** - * Corresponds to a 200 response, containing relevant metadata and potentially data. + * Corresponds to a 200 or 206 response, containing relevant metadata and potentially data. + * A 206 will be returned if range metadata is found in the metadata object. */ export class OkResponseDescription extends ResponseDescription { /** @@ -12,6 +14,6 @@ export class OkResponseDescription extends ResponseDescription { * @param data - Potential data. @ignored */ public constructor(metadata: RepresentationMetadata, data?: Guarded) { - super(200, metadata, data); + super(metadata.has(SOLID_HTTP.terms.unit) ? 206 : 200, metadata, data); } } diff --git a/src/http/representation/RepresentationPreferences.ts b/src/http/representation/RepresentationPreferences.ts index 6eb5565dde..985d5d2ac5 100644 --- a/src/http/representation/RepresentationPreferences.ts +++ b/src/http/representation/RepresentationPreferences.ts @@ -31,4 +31,6 @@ export interface RepresentationPreferences { datetime?: ValuePreferences; encoding?: ValuePreferences; language?: ValuePreferences; + // `start` can be negative and implies the last X of a stream + range?: { unit: string; parts: { start: number; end?: number }[] }; } diff --git a/src/index.ts b/src/index.ts index 217958b0ec..ad6e7d0e7e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -77,6 +77,8 @@ export * from './http/input/metadata/SlugParser'; // HTTP/Input/Preferences export * from './http/input/preferences/AcceptPreferenceParser'; export * from './http/input/preferences/PreferenceParser'; +export * from './http/input/preferences/RangePreferenceParser'; +export * from './http/input/preferences/UnionPreferenceParser'; // HTTP/Input export * from './http/input/BasicRequestParser'; @@ -106,6 +108,7 @@ export * from './http/output/metadata/LinkRelMetadataWriter'; export * from './http/output/metadata/MappedMetadataWriter'; export * from './http/output/metadata/MetadataWriter'; export * from './http/output/metadata/ModifiedMetadataWriter'; +export * from './http/output/metadata/RangeMetadataWriter'; export * from './http/output/metadata/StorageDescriptionAdvertiser'; export * from './http/output/metadata/WacAllowMetadataWriter'; export * from './http/output/metadata/WwwAuthMetadataWriter'; @@ -443,6 +446,7 @@ export * from './storage/validators/QuotaValidator'; export * from './storage/AtomicResourceStore'; export * from './storage/BaseResourceStore'; export * from './storage/BasicConditions'; +export * from './storage/BinarySliceResourceStore'; export * from './storage/CachedResourceSet'; export * from './storage/Conditions'; export * from './storage/DataAccessorBasedStore'; @@ -472,6 +476,7 @@ export * from './util/errors/NotFoundHttpError'; export * from './util/errors/NotImplementedHttpError'; export * from './util/errors/OAuthHttpError'; export * from './util/errors/PreconditionFailedHttpError'; +export * from './util/errors/RangeNotSatisfiedHttpError'; export * from './util/errors/RedirectHttpError'; export * from './util/errors/SystemError'; export * from './util/errors/UnauthorizedHttpError'; @@ -542,6 +547,7 @@ export * from './util/PromiseUtil'; export * from './util/QuadUtil'; export * from './util/RecordObject'; export * from './util/ResourceUtil'; +export * from './util/SliceStream'; export * from './util/StreamUtil'; export * from './util/StringUtil'; export * from './util/TermUtil'; diff --git a/src/storage/BinarySliceResourceStore.ts b/src/storage/BinarySliceResourceStore.ts new file mode 100644 index 0000000000..9ddac934a2 --- /dev/null +++ b/src/storage/BinarySliceResourceStore.ts @@ -0,0 +1,66 @@ +import type { Representation } from '../http/representation/Representation'; +import type { RepresentationPreferences } from '../http/representation/RepresentationPreferences'; +import type { ResourceIdentifier } from '../http/representation/ResourceIdentifier'; +import { getLoggerFor } from '../logging/LogUtil'; +import { InternalServerError } from '../util/errors/InternalServerError'; +import { RangeNotSatisfiedHttpError } from '../util/errors/RangeNotSatisfiedHttpError'; +import { guardStream } from '../util/GuardedStream'; +import { SliceStream } from '../util/SliceStream'; +import { toLiteral } from '../util/TermUtil'; +import { SOLID_HTTP, XSD } from '../util/Vocabularies'; +import type { Conditions } from './Conditions'; +import { PassthroughStore } from './PassthroughStore'; +import type { ResourceStore } from './ResourceStore'; + +/** + * Resource store that slices the data stream if there are range preferences. + * Only works for `bytes` range preferences on binary data streams. + * Does not support multipart range requests. + * + * If the slice happens, unit/start/end values will be written to the metadata to indicate such. + * The values are dependent on the preferences we got as an input, + * as we don't know the actual size of the data stream. + */ +export class BinarySliceResourceStore extends PassthroughStore { + protected readonly logger = getLoggerFor(this); + + public async getRepresentation(identifier: ResourceIdentifier, preferences: RepresentationPreferences, + conditions?: Conditions): Promise { + const result = await this.source.getRepresentation(identifier, preferences, conditions); + + if (!preferences.range || preferences.range.unit !== 'bytes' || preferences.range.parts.length === 0) { + return result; + } + if (result.metadata.has(SOLID_HTTP.unit)) { + this.logger.debug('Not slicing stream that has already been sliced.'); + return result; + } + + if (!result.binary) { + throw new InternalServerError('Trying to slice a non-binary stream.'); + } + if (preferences.range.parts.length > 1) { + throw new RangeNotSatisfiedHttpError('Multipart range requests are not supported.'); + } + + const [{ start, end }] = preferences.range.parts; + result.metadata.set(SOLID_HTTP.terms.unit, preferences.range.unit); + result.metadata.set(SOLID_HTTP.terms.start, toLiteral(start, XSD.terms.integer)); + if (typeof end === 'number') { + result.metadata.set(SOLID_HTTP.terms.end, toLiteral(end, XSD.terms.integer)); + } + + try { + // The reason we don't determine the object mode based on the object mode of the parent stream + // is that `guardedStreamFrom` does not create object streams when inputting streams/buffers. + // Something to potentially update in the future. + result.data = guardStream(new SliceStream(result.data, { start, end, objectMode: false })); + } catch (error: unknown) { + // Creating the slice stream can throw an error if some of the parameters are unacceptable. + // Need to make sure the stream is closed in that case. + result.data.destroy(); + throw error; + } + return result; + } +} diff --git a/src/util/SliceStream.ts b/src/util/SliceStream.ts new file mode 100644 index 0000000000..9d8e4b76ef --- /dev/null +++ b/src/util/SliceStream.ts @@ -0,0 +1,87 @@ +import type { Readable, TransformCallback, TransformOptions } from 'stream'; +import { Transform } from 'stream'; +import { RangeNotSatisfiedHttpError } from './errors/RangeNotSatisfiedHttpError'; +import { pipeSafely } from './StreamUtil'; + +/** + * A stream that slices a part out of another stream. + * `start` and `end` are inclusive. + * If `end` is not defined it is until the end of the stream. + * Does not support negative `start` values which would indicate slicing the end of the stream off, + * since we don't know the length of the input stream. + * + * Both object and non-object streams are supported. + * This needs to be explicitly specified, + * as the class makes no assumptions based on the object mode of the source stream. + */ +export class SliceStream extends Transform { + protected readonly source: Readable; + protected remainingSkip: number; + protected remainingRead: number; + + public constructor(source: Readable, options: TransformOptions & { start: number; end?: number }) { + super(options); + const end = options.end ?? Number.POSITIVE_INFINITY; + if (options.start < 0) { + throw new RangeNotSatisfiedHttpError('Slicing data at the end of a stream is not supported.'); + } + if (options.start >= end) { + throw new RangeNotSatisfiedHttpError('Range start should be less than end.'); + } + this.remainingSkip = options.start; + // End value is inclusive + this.remainingRead = end - options.start + 1; + + this.source = source; + pipeSafely(source, this); + } + + // eslint-disable-next-line @typescript-eslint/naming-convention + public _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { + this.source.pause(); + if (this.writableObjectMode) { + this.objectSlice(chunk); + } else { + this.binarySlice(chunk); + } + // eslint-disable-next-line callback-return + callback(); + this.source.resume(); + } + + protected binarySlice(chunk: Buffer): void { + let length = chunk.length; + if (this.remainingSkip > 0) { + chunk = chunk.slice(this.remainingSkip); + this.remainingSkip -= length - chunk.length; + length = chunk.length; + } + if (length > 0 && this.remainingSkip <= 0) { + chunk = chunk.slice(0, this.remainingRead); + this.push(chunk); + this.remainingRead -= length; + this.checkEnd(); + } + } + + protected objectSlice(chunk: unknown): void { + if (this.remainingSkip > 0) { + this.remainingSkip -= 1; + } else { + this.remainingRead -= 1; + this.push(chunk); + this.checkEnd(); + } + } + + /** + * Stop piping the source stream and close everything once the slice is finished. + */ + protected checkEnd(): void { + if (this.remainingRead <= 0) { + this.source.unpipe(); + this.end(); + this.source.destroy(); + } + } +} diff --git a/src/util/Vocabularies.ts b/src/util/Vocabularies.ts index 71f04ffa1b..6a642c207c 100644 --- a/src/util/Vocabularies.ts +++ b/src/util/Vocabularies.ts @@ -258,8 +258,12 @@ export const SOLID_ERROR = createVocabulary('urn:npm:solid:community-server:erro ); export const SOLID_HTTP = createVocabulary('urn:npm:solid:community-server:http:', + // Unit, start, and end are used for range headers + 'end', 'location', + 'start', 'slug', + 'unit', ); export const SOLID_META = createVocabulary('urn:npm:solid:community-server:meta:', diff --git a/src/util/errors/RangeNotSatisfiedHttpError.ts b/src/util/errors/RangeNotSatisfiedHttpError.ts new file mode 100644 index 0000000000..9678f7487b --- /dev/null +++ b/src/util/errors/RangeNotSatisfiedHttpError.ts @@ -0,0 +1,19 @@ +import type { HttpErrorOptions } from './HttpError'; +import { generateHttpErrorClass } from './HttpError'; + +// eslint-disable-next-line @typescript-eslint/naming-convention +const BaseHttpError = generateHttpErrorClass(416, 'RangeNotSatisfiedHttpError'); + +/** + * An error thrown when the requested range is not supported. + */ +export class RangeNotSatisfiedHttpError extends BaseHttpError { + /** + * Default message is 'The requested range is not supported.'. + * @param message - Optional, more specific, message. + * @param options - Optional error options. + */ + public constructor(message?: string, options?: HttpErrorOptions) { + super(message ?? 'The requested range is not supported.', options); + } +} diff --git a/test/integration/LdpHandlerWithoutAuth.test.ts b/test/integration/LdpHandlerWithoutAuth.test.ts index 215731a05b..d82dffa214 100644 --- a/test/integration/LdpHandlerWithoutAuth.test.ts +++ b/test/integration/LdpHandlerWithoutAuth.test.ts @@ -2,7 +2,7 @@ import { createReadStream } from 'fs'; import fetch from 'cross-fetch'; import type { Quad } from 'n3'; import { DataFactory, Parser, Store } from 'n3'; -import { joinFilePath, PIM, RDF } from '../../src/'; +import { joinFilePath, joinUrl, PIM, RDF } from '../../src/'; import type { App } from '../../src/'; import { LDP } from '../../src/util/Vocabularies'; import { @@ -726,4 +726,27 @@ describe.each(stores)('An LDP handler allowing all requests %s', (name, { storeC // DELETE await deleteResource(resourceUrl); }); + + it('supports range requests.', async(): Promise => { + const resourceUrl = joinUrl(baseUrl, 'range'); + await putResource(resourceUrl, { contentType: 'text/plain', body: '0123456789' }); + + let response = await fetch(resourceUrl, { headers: { range: 'bytes=0-5' }}); + expect(response.status).toBe(206); + expect(response.headers.get('content-range')).toBe('bytes 0-5/*'); + await expect(response.text()).resolves.toBe('012345'); + + response = await fetch(resourceUrl, { headers: { range: 'bytes=5-' }}); + expect(response.status).toBe(206); + expect(response.headers.get('content-range')).toBe('bytes 5-*/*'); + await expect(response.text()).resolves.toBe('56789'); + + response = await fetch(resourceUrl, { headers: { range: 'bytes=5-15' }}); + expect(response.status).toBe(206); + expect(response.headers.get('content-range')).toBe('bytes 5-15/*'); + await expect(response.text()).resolves.toBe('56789'); + + response = await fetch(resourceUrl, { headers: { range: 'bytes=-5' }}); + expect(response.status).toBe(416); + }); }); diff --git a/test/unit/http/input/preferences/RangePreferenceParser.test.ts b/test/unit/http/input/preferences/RangePreferenceParser.test.ts new file mode 100644 index 0000000000..abc98fa404 --- /dev/null +++ b/test/unit/http/input/preferences/RangePreferenceParser.test.ts @@ -0,0 +1,40 @@ +import { RangePreferenceParser } from '../../../../../src/http/input/preferences/RangePreferenceParser'; +import { BadRequestHttpError } from '../../../../../src/util/errors/BadRequestHttpError'; + +describe('A RangePreferenceParser', (): void => { + const parser = new RangePreferenceParser(); + + it('parses range headers.', async(): Promise => { + await expect(parser.handle({ request: { headers: { range: 'bytes=5-10' }}} as any)) + .resolves.toEqual({ range: { unit: 'bytes', parts: [{ start: 5, end: 10 }]}}); + + await expect(parser.handle({ request: { headers: { range: 'bytes=5-' }}} as any)) + .resolves.toEqual({ range: { unit: 'bytes', parts: [{ start: 5 }]}}); + + await expect(parser.handle({ request: { headers: { range: 'bytes=-5' }}} as any)) + .resolves.toEqual({ range: { unit: 'bytes', parts: [{ start: -5 }]}}); + + await expect(parser.handle({ request: { headers: { range: 'bytes=5-10, 11-20, 21-99' }}} as any)) + .resolves.toEqual({ range: { unit: 'bytes', + parts: [{ start: 5, end: 10 }, { start: 11, end: 20 }, { start: 21, end: 99 }]}}); + }); + + it('returns an empty object if there is no header.', async(): Promise => { + await expect(parser.handle({ request: { headers: {}}} as any)).resolves.toEqual({}); + }); + + it('rejects invalid range headers.', async(): Promise => { + await expect(parser.handle({ request: { headers: { range: '=5-10' }}} as any)) + .rejects.toThrow(BadRequestHttpError); + await expect(parser.handle({ request: { headers: { range: 'bytes' }}} as any)) + .rejects.toThrow(BadRequestHttpError); + await expect(parser.handle({ request: { headers: { range: 'bytes=' }}} as any)) + .rejects.toThrow(BadRequestHttpError); + await expect(parser.handle({ request: { headers: { range: 'bytes=-' }}} as any)) + .rejects.toThrow(BadRequestHttpError); + await expect(parser.handle({ request: { headers: { range: 'bytes=5' }}} as any)) + .rejects.toThrow(BadRequestHttpError); + await expect(parser.handle({ request: { headers: { range: 'bytes=5-10, 99' }}} as any)) + .rejects.toThrow(BadRequestHttpError); + }); +}); diff --git a/test/unit/http/input/preferences/UnionPreferenceParser.test.ts b/test/unit/http/input/preferences/UnionPreferenceParser.test.ts new file mode 100644 index 0000000000..206f75faa5 --- /dev/null +++ b/test/unit/http/input/preferences/UnionPreferenceParser.test.ts @@ -0,0 +1,54 @@ +import { PreferenceParser } from '../../../../../src/http/input/preferences/PreferenceParser'; +import { UnionPreferenceParser } from '../../../../../src/http/input/preferences/UnionPreferenceParser'; +import { InternalServerError } from '../../../../../src/util/errors/InternalServerError'; + +describe('A UnionPreferenceParser', (): void => { + let parsers: jest.Mocked[]; + let parser: UnionPreferenceParser; + + beforeEach(async(): Promise => { + parsers = [ + { + canHandle: jest.fn(), + handle: jest.fn().mockResolvedValue({}), + } satisfies Partial as any, + { + canHandle: jest.fn(), + handle: jest.fn().mockResolvedValue({}), + } satisfies Partial as any, + ]; + + parser = new UnionPreferenceParser(parsers); + }); + + it('combines the outputs.', async(): Promise => { + parsers[0].handle.mockResolvedValue({ + type: { 'text/turtle': 1 }, + range: { unit: 'bytes', parts: [{ start: 3, end: 5 }]}, + }); + parsers[1].handle.mockResolvedValue({ + type: { 'text/plain': 0.9 }, + language: { nl: 0.8 }, + }); + + await expect(parser.handle({} as any)).resolves.toEqual({ + type: { 'text/turtle': 1, 'text/plain': 0.9 }, + language: { nl: 0.8 }, + range: { unit: 'bytes', parts: [{ start: 3, end: 5 }]}, + }); + }); + + it('throws an error if multiple parsers return a range.', async(): Promise => { + parsers[0].handle.mockResolvedValue({ + type: { 'text/turtle': 1 }, + range: { unit: 'bytes', parts: [{ start: 3, end: 5 }]}, + }); + parsers[1].handle.mockResolvedValue({ + type: { 'text/plain': 0.9 }, + language: { nl: 0.8 }, + range: { unit: 'bytes', parts: [{ start: 3, end: 5 }]}, + }); + + await expect(parser.handle({} as any)).rejects.toThrow(InternalServerError); + }); +}); diff --git a/test/unit/http/ldp/GetOperationHandler.test.ts b/test/unit/http/ldp/GetOperationHandler.test.ts index 20044c6e13..65503cfddc 100644 --- a/test/unit/http/ldp/GetOperationHandler.test.ts +++ b/test/unit/http/ldp/GetOperationHandler.test.ts @@ -8,6 +8,7 @@ import { BasicConditions } from '../../../../src/storage/BasicConditions'; import type { ResourceStore } from '../../../../src/storage/ResourceStore'; import { NotImplementedHttpError } from '../../../../src/util/errors/NotImplementedHttpError'; import { NotModifiedHttpError } from '../../../../src/util/errors/NotModifiedHttpError'; +import { SOLID_HTTP } from '../../../../src/util/Vocabularies'; describe('A GetOperationHandler', (): void => { let operation: Operation; @@ -45,6 +46,18 @@ describe('A GetOperationHandler', (): void => { expect(store.getRepresentation).toHaveBeenLastCalledWith(operation.target, preferences, conditions); }); + it('returns 206 if the result is a partial stream.', async(): Promise => { + metadata.set(SOLID_HTTP.terms.unit, 'bytes'); + metadata.set(SOLID_HTTP.terms.start, '5'); + metadata.set(SOLID_HTTP.terms.end, '7'); + const result = await handler.handle({ operation }); + expect(result.statusCode).toBe(206); + expect(result.metadata).toBe(metadata); + expect(result.data).toBe(data); + expect(store.getRepresentation).toHaveBeenCalledTimes(1); + expect(store.getRepresentation).toHaveBeenLastCalledWith(operation.target, preferences, conditions); + }); + it('returns a 304 if the conditions do not match.', async(): Promise => { operation.conditions = { matchesMetadata: (): boolean => false, diff --git a/test/unit/http/output/metadata/RangeMetadataWriter.test.ts b/test/unit/http/output/metadata/RangeMetadataWriter.test.ts new file mode 100644 index 0000000000..f2839092c5 --- /dev/null +++ b/test/unit/http/output/metadata/RangeMetadataWriter.test.ts @@ -0,0 +1,40 @@ +import { createResponse } from 'node-mocks-http'; +import { RangeMetadataWriter } from '../../../../../src/http/output/metadata/RangeMetadataWriter'; +import { RepresentationMetadata } from '../../../../../src/http/representation/RepresentationMetadata'; +import type { HttpResponse } from '../../../../../src/server/HttpResponse'; +import { SOLID_HTTP } from '../../../../../src/util/Vocabularies'; + +describe('RangeMetadataWriter', (): void => { + let metadata: RepresentationMetadata; + let response: HttpResponse; + let writer: RangeMetadataWriter; + + beforeEach(async(): Promise => { + metadata = new RepresentationMetadata(); + response = createResponse(); + writer = new RangeMetadataWriter(); + }); + + it('adds the content-range header.', async(): Promise => { + metadata.set(SOLID_HTTP.terms.unit, 'bytes'); + metadata.set(SOLID_HTTP.terms.start, '1'); + metadata.set(SOLID_HTTP.terms.end, '5'); + await expect(writer.handle({ response, metadata })).resolves.toBeUndefined(); + expect(response.getHeaders()).toEqual({ + 'content-range': 'bytes 1-5/*', + }); + }); + + it('uses * if the value is unknown.', async(): Promise => { + metadata.set(SOLID_HTTP.terms.unit, 'bytes'); + await expect(writer.handle({ response, metadata })).resolves.toBeUndefined(); + expect(response.getHeaders()).toEqual({ + 'content-range': 'bytes *-*/*', + }); + }); + + it('does nothing if there is no range metadata.', async(): Promise => { + await expect(writer.handle({ response, metadata })).resolves.toBeUndefined(); + expect(response.getHeaders()).toEqual({ }); + }); +}); diff --git a/test/unit/storage/BinarySliceResourceStore.test.ts b/test/unit/storage/BinarySliceResourceStore.test.ts new file mode 100644 index 0000000000..494d5c6df8 --- /dev/null +++ b/test/unit/storage/BinarySliceResourceStore.test.ts @@ -0,0 +1,80 @@ +import { BasicRepresentation } from '../../../src/http/representation/BasicRepresentation'; +import type { Representation } from '../../../src/http/representation/Representation'; +import { BinarySliceResourceStore } from '../../../src/storage/BinarySliceResourceStore'; +import { ResourceStore } from '../../../src/storage/ResourceStore'; +import { InternalServerError } from '../../../src/util/errors/InternalServerError'; +import { RangeNotSatisfiedHttpError } from '../../../src/util/errors/RangeNotSatisfiedHttpError'; +import { readableToString } from '../../../src/util/StreamUtil'; +import { SOLID_HTTP } from '../../../src/util/Vocabularies'; + +describe('A BinarySliceResourceStore', (): void => { + const identifier = { path: 'path' }; + let representation: Representation; + let source: jest.Mocked; + let store: BinarySliceResourceStore; + + beforeEach(async(): Promise => { + representation = new BasicRepresentation('0123456789', 'text/plain'); + + source = { + getRepresentation: jest.fn().mockResolvedValue(representation), + } satisfies Partial as any; + + store = new BinarySliceResourceStore(source); + }); + + it('slices the data stream and stores the metadata.', async(): Promise => { + const result = await store.getRepresentation(identifier, { range: { unit: 'bytes', parts: [{ start: 1, end: 4 }]}}); + await expect(readableToString(result.data)).resolves.toBe('1234'); + expect(result.metadata.get(SOLID_HTTP.terms.unit)?.value).toBe('bytes'); + expect(result.metadata.get(SOLID_HTTP.terms.start)?.value).toBe('1'); + expect(result.metadata.get(SOLID_HTTP.terms.end)?.value).toBe('4'); + }); + + it('does not add end metadata if there is none.', async(): Promise => { + const result = await store.getRepresentation(identifier, { range: { unit: 'bytes', parts: [{ start: 5 }]}}); + await expect(readableToString(result.data)).resolves.toBe('56789'); + expect(result.metadata.get(SOLID_HTTP.terms.unit)?.value).toBe('bytes'); + expect(result.metadata.get(SOLID_HTTP.terms.start)?.value).toBe('5'); + expect(result.metadata.get(SOLID_HTTP.terms.end)).toBeUndefined(); + }); + + it('returns the original data if there is no valid range request.', async(): Promise => { + let result = await store.getRepresentation(identifier, {}); + await expect(readableToString(result.data)).resolves.toBe('0123456789'); + + source.getRepresentation.mockResolvedValue(new BasicRepresentation('0123456789', 'text/plain')); + result = await store.getRepresentation(identifier, { range: { unit: 'triples', parts: []}}); + await expect(readableToString(result.data)).resolves.toBe('0123456789'); + + source.getRepresentation.mockResolvedValue(new BasicRepresentation('0123456789', 'text/plain')); + result = await store.getRepresentation(identifier, { range: { unit: 'bytes', parts: []}}); + await expect(readableToString(result.data)).resolves.toBe('0123456789'); + }); + + it('returns the original data if there already is slice metadata.', async(): Promise => { + representation.metadata.set(SOLID_HTTP.terms.unit, 'triples'); + + const result = await store.getRepresentation(identifier, { range: { unit: 'bytes', parts: [{ start: 5 }]}}); + await expect(readableToString(result.data)).resolves.toBe('0123456789'); + }); + + it('only supports binary streams.', async(): Promise => { + representation.binary = false; + await expect(store.getRepresentation(identifier, { range: { unit: 'bytes', parts: [{ start: 5 }]}})) + .rejects.toThrow(InternalServerError); + }); + + it('does not support multipart ranges.', async(): Promise => { + await expect(store.getRepresentation(identifier, + { range: { unit: 'bytes', parts: [{ start: 5, end: 6 }, { start: 7, end: 8 }]}})) + .rejects.toThrow(RangeNotSatisfiedHttpError); + }); + + it('closes the source stream if there was an error creating the SliceStream.', async(): Promise => { + representation.data.destroy = jest.fn(); + await expect(store.getRepresentation(identifier, { range: { unit: 'bytes', parts: [{ start: -5 }]}})) + .rejects.toThrow(RangeNotSatisfiedHttpError); + expect(representation.data.destroy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/test/unit/util/SliceStream.test.ts b/test/unit/util/SliceStream.test.ts new file mode 100644 index 0000000000..8a9a68dda6 --- /dev/null +++ b/test/unit/util/SliceStream.test.ts @@ -0,0 +1,41 @@ +import { Readable } from 'stream'; +import { RangeNotSatisfiedHttpError } from '../../../src/util/errors/RangeNotSatisfiedHttpError'; +import { SliceStream } from '../../../src/util/SliceStream'; +import { readableToString } from '../../../src/util/StreamUtil'; + +describe('A SliceStream', (): void => { + it('does not support suffix slicing.', async(): Promise => { + expect((): unknown => new SliceStream(Readable.from('0123456789'), { start: -5 })) + .toThrow(RangeNotSatisfiedHttpError); + }); + + it('requires the end to be more than the start.', async(): Promise => { + expect((): unknown => new SliceStream(Readable.from('0123456789'), { start: 5, end: 4 })) + .toThrow(RangeNotSatisfiedHttpError); + expect((): unknown => new SliceStream(Readable.from('0123456789'), { start: 5, end: 5 })) + .toThrow(RangeNotSatisfiedHttpError); + }); + + it('can slice binary streams.', async(): Promise => { + await expect(readableToString(new SliceStream(Readable.from('0123456789', { objectMode: false }), + { start: 3, end: 7, objectMode: false }))).resolves.toBe('34567'); + + await expect(readableToString(new SliceStream(Readable.from('0123456789', { objectMode: false }), + { start: 3, objectMode: false }))).resolves.toBe('3456789'); + + await expect(readableToString(new SliceStream(Readable.from('0123456789', { objectMode: false }), + { start: 3, end: 20, objectMode: false }))).resolves.toBe('3456789'); + }); + + it('can slice object streams.', async(): Promise => { + const arr = [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' ]; + await expect(readableToString(new SliceStream(Readable.from(arr, { objectMode: true }), + { start: 3, end: 7, objectMode: true }))).resolves.toBe('34567'); + + await expect(readableToString(new SliceStream(Readable.from(arr, { objectMode: true }), + { start: 3, objectMode: true }))).resolves.toBe('3456789'); + + await expect(readableToString(new SliceStream(Readable.from(arr, { objectMode: true }), + { start: 3, end: 20, objectMode: true }))).resolves.toBe('3456789'); + }); +}); diff --git a/test/unit/util/errors/HttpError.test.ts b/test/unit/util/errors/HttpError.test.ts index 4b2726dd22..786382f380 100644 --- a/test/unit/util/errors/HttpError.test.ts +++ b/test/unit/util/errors/HttpError.test.ts @@ -12,6 +12,7 @@ import { NotImplementedHttpError } from '../../../../src/util/errors/NotImplemen import { NotModifiedHttpError } from '../../../../src/util/errors/NotModifiedHttpError'; import { PayloadHttpError } from '../../../../src/util/errors/PayloadHttpError'; import { PreconditionFailedHttpError } from '../../../../src/util/errors/PreconditionFailedHttpError'; +import { RangeNotSatisfiedHttpError } from '../../../../src/util/errors/RangeNotSatisfiedHttpError'; import { UnauthorizedHttpError } from '../../../../src/util/errors/UnauthorizedHttpError'; import { UnprocessableEntityHttpError } from '../../../../src/util/errors/UnprocessableEntityHttpError'; import { UnsupportedMediaTypeHttpError } from '../../../../src/util/errors/UnsupportedMediaTypeHttpError'; @@ -30,6 +31,7 @@ describe('HttpError', (): void => { [ 'PreconditionFailedHttpError', 412, PreconditionFailedHttpError ], [ 'PayloadHttpError', 413, PayloadHttpError ], [ 'UnsupportedMediaTypeHttpError', 415, UnsupportedMediaTypeHttpError ], + [ 'RangeNotSatisfiedHttpError', 416, RangeNotSatisfiedHttpError ], [ 'UnprocessableEntityHttpError', 422, UnprocessableEntityHttpError ], [ 'InternalServerError', 500, InternalServerError ], [ 'NotImplementedHttpError', 501, NotImplementedHttpError ], From 71e55690f3418be3d08e35d2cd3aeae5a0634654 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Mon, 2 Oct 2023 13:40:19 +0200 Subject: [PATCH 21/25] feat: Track binary size of resources when possible --- config/http/middleware/handlers/cors.json | 1 + .../output/metadata/RangeMetadataWriter.ts | 35 +++++++++++++--- src/storage/BinarySliceResourceStore.ts | 6 ++- src/storage/accessors/DataAccessor.ts | 3 ++ src/storage/accessors/InMemoryDataAccessor.ts | 14 ++++++- src/storage/conversion/ChainedConverter.ts | 8 ++++ src/util/QuadUtil.ts | 14 ++++++- src/util/SliceStream.ts | 32 ++++++++++++--- .../integration/LdpHandlerWithoutAuth.test.ts | 15 ++++--- .../metadata/RangeMetadataWriter.test.ts | 40 +++++++++++++++++-- .../storage/BinarySliceResourceStore.test.ts | 10 ++++- .../accessors/InMemoryDataAccessor.test.ts | 13 ++++-- .../conversion/ChainedConverter.test.ts | 10 ++++- test/unit/util/QuadUtil.test.ts | 13 +++++- test/unit/util/SliceStream.test.ts | 13 +++++- 15 files changed, 193 insertions(+), 34 deletions(-) diff --git a/config/http/middleware/handlers/cors.json b/config/http/middleware/handlers/cors.json index bc0239dab4..c4f9f33f34 100644 --- a/config/http/middleware/handlers/cors.json +++ b/config/http/middleware/handlers/cors.json @@ -21,6 +21,7 @@ "Accept-Post", "Accept-Put", "Allow", + "Content-Range", "ETag", "Last-Modified", "Link", diff --git a/src/http/output/metadata/RangeMetadataWriter.ts b/src/http/output/metadata/RangeMetadataWriter.ts index d67d229eee..ed74efd1d1 100644 --- a/src/http/output/metadata/RangeMetadataWriter.ts +++ b/src/http/output/metadata/RangeMetadataWriter.ts @@ -1,6 +1,8 @@ +import { getLoggerFor } from '../../../logging/LogUtil'; import type { HttpResponse } from '../../../server/HttpResponse'; import { addHeader } from '../../../util/HeaderUtil'; -import { SOLID_HTTP } from '../../../util/Vocabularies'; +import { termToInt } from '../../../util/QuadUtil'; +import { POSIX, SOLID_HTTP } from '../../../util/Vocabularies'; import type { RepresentationMetadata } from '../../representation/RepresentationMetadata'; import { MetadataWriter } from './MetadataWriter'; @@ -10,16 +12,39 @@ import { MetadataWriter } from './MetadataWriter'; * According to the RFC, this is incorrect, * but is all we can do as long as we don't know the full length of the representation in advance. * For the same reason, the total length of the representation will always be `*`. + * + * This class also adds the content-length header. + * This will contain either the full size for standard requests, + * or the size of the slice for range requests. */ export class RangeMetadataWriter extends MetadataWriter { + protected readonly logger = getLoggerFor(this); + public async handle(input: { response: HttpResponse; metadata: RepresentationMetadata }): Promise { - const unit = input.metadata.get(SOLID_HTTP.terms.unit); + const size = termToInt(input.metadata.get(POSIX.terms.size)); + const unit = input.metadata.get(SOLID_HTTP.terms.unit)?.value; if (!unit) { + if (typeof size === 'number') { + addHeader(input.response, 'Content-Length', `${size}`); + } return; } - const start = input.metadata.get(SOLID_HTTP.terms.start); - const end = input.metadata.get(SOLID_HTTP.terms.end); - addHeader(input.response, 'Content-Range', `${unit.value} ${start?.value ?? '*'}-${end?.value ?? '*'}/*`); + let start = termToInt(input.metadata.get(SOLID_HTTP.terms.start)); + if (typeof start === 'number' && start < 0 && typeof size === 'number') { + start = size + start; + } + let end = termToInt(input.metadata.get(SOLID_HTTP.terms.end)); + if (typeof end !== 'number' && typeof size === 'number') { + end = size - 1; + } + + const rangeHeader = `${unit} ${start ?? '*'}-${end ?? '*'}/${size ?? '*'}`; + addHeader(input.response, 'Content-Range', rangeHeader); + if (typeof start === 'number' && typeof end === 'number') { + addHeader(input.response, 'Content-Length', `${end - start + 1}`); + } else { + this.logger.warn(`Generating invalid content-range header due to missing size information: ${rangeHeader}`); + } } } diff --git a/src/storage/BinarySliceResourceStore.ts b/src/storage/BinarySliceResourceStore.ts index 9ddac934a2..d2e1be93bf 100644 --- a/src/storage/BinarySliceResourceStore.ts +++ b/src/storage/BinarySliceResourceStore.ts @@ -5,9 +5,10 @@ import { getLoggerFor } from '../logging/LogUtil'; import { InternalServerError } from '../util/errors/InternalServerError'; import { RangeNotSatisfiedHttpError } from '../util/errors/RangeNotSatisfiedHttpError'; import { guardStream } from '../util/GuardedStream'; +import { termToInt } from '../util/QuadUtil'; import { SliceStream } from '../util/SliceStream'; import { toLiteral } from '../util/TermUtil'; -import { SOLID_HTTP, XSD } from '../util/Vocabularies'; +import { POSIX, SOLID_HTTP, XSD } from '../util/Vocabularies'; import type { Conditions } from './Conditions'; import { PassthroughStore } from './PassthroughStore'; import type { ResourceStore } from './ResourceStore'; @@ -51,10 +52,11 @@ export class BinarySliceResourceStore e } try { + const size = termToInt(result.metadata.get(POSIX.terms.size)); // The reason we don't determine the object mode based on the object mode of the parent stream // is that `guardedStreamFrom` does not create object streams when inputting streams/buffers. // Something to potentially update in the future. - result.data = guardStream(new SliceStream(result.data, { start, end, objectMode: false })); + result.data = guardStream(new SliceStream(result.data, { start, end, size, objectMode: false })); } catch (error: unknown) { // Creating the slice stream can throw an error if some of the parameters are unacceptable. // Need to make sure the stream is closed in that case. diff --git a/src/storage/accessors/DataAccessor.ts b/src/storage/accessors/DataAccessor.ts index 6caa00d1b5..f15560ee18 100644 --- a/src/storage/accessors/DataAccessor.ts +++ b/src/storage/accessors/DataAccessor.ts @@ -32,6 +32,9 @@ export interface DataAccessor { /** * Returns the metadata corresponding to the identifier. + * If possible, it is suggested to add a `posix:size` triple to the metadata indicating the binary size. + * This is necessary for range requests. + * * @param identifier - Identifier for which the metadata is requested. */ getMetadata: (identifier: ResourceIdentifier) => Promise; diff --git a/src/storage/accessors/InMemoryDataAccessor.ts b/src/storage/accessors/InMemoryDataAccessor.ts index ab95c1c320..353a51aa4a 100644 --- a/src/storage/accessors/InMemoryDataAccessor.ts +++ b/src/storage/accessors/InMemoryDataAccessor.ts @@ -8,6 +8,8 @@ import { NotFoundHttpError } from '../../util/errors/NotFoundHttpError'; import type { Guarded } from '../../util/GuardedStream'; import type { IdentifierStrategy } from '../../util/identifiers/IdentifierStrategy'; import { guardedStreamFrom } from '../../util/StreamUtil'; +import { POSIX } from '../../util/Vocabularies'; +import { isInternalContentType } from '../conversion/ConversionUtil'; import type { DataAccessor } from './DataAccessor'; interface DataEntry { @@ -59,9 +61,17 @@ export class InMemoryDataAccessor implements DataAccessor, SingleThreaded { public async writeDocument(identifier: ResourceIdentifier, data: Guarded, metadata: RepresentationMetadata): Promise { const parent = this.getParentEntry(identifier); + // Drain original stream and create copy + const dataArray = await arrayifyStream(data); + + // Only add the size for binary streams, which are all streams that do not have an internal type. + if (metadata.contentType && !isInternalContentType(metadata.contentType)) { + const size = dataArray.reduce((total, chunk: Buffer): number => total + chunk.length, 0); + metadata.set(POSIX.terms.size, `${size}`); + } + parent.entries[identifier.path] = { - // Drain original stream and create copy - data: await arrayifyStream(data), + data: dataArray, metadata, }; } diff --git a/src/storage/conversion/ChainedConverter.ts b/src/storage/conversion/ChainedConverter.ts index 9415b9d968..ba657265e3 100644 --- a/src/storage/conversion/ChainedConverter.ts +++ b/src/storage/conversion/ChainedConverter.ts @@ -5,6 +5,7 @@ import type { ValuePreferences } from '../../http/representation/RepresentationP import { getLoggerFor } from '../../logging/LogUtil'; import { BadRequestHttpError } from '../../util/errors/BadRequestHttpError'; import { NotImplementedHttpError } from '../../util/errors/NotImplementedHttpError'; +import { POSIX } from '../../util/Vocabularies'; import { cleanPreferences, getBestPreference, getTypeWeight, preferencesToString } from './ConversionUtil'; import type { RepresentationConverterArgs } from './RepresentationConverter'; import { RepresentationConverter } from './RepresentationConverter'; @@ -100,6 +101,13 @@ export class ChainedConverter extends RepresentationConverter { args.preferences = { type: { [outTypes[i]]: 1 }}; args.representation = await match.converters[i].handle(args); } + + // For now, we assume any kind of conversion invalidates the stored byte length. + // In the future, we could let converters handle this individually, as some might know the size of the result. + if (match.converters.length > 0) { + args.representation.metadata.removeAll(POSIX.terms.size); + } + return args.representation; } diff --git a/src/util/QuadUtil.ts b/src/util/QuadUtil.ts index dfba2e4af3..45f7c70269 100644 --- a/src/util/QuadUtil.ts +++ b/src/util/QuadUtil.ts @@ -3,7 +3,7 @@ import type { NamedNode } from '@rdfjs/types'; import arrayifyStream from 'arrayify-stream'; import type { ParserOptions } from 'n3'; import { StreamParser, StreamWriter } from 'n3'; -import type { Quad } from 'rdf-js'; +import type { Quad, Term } from 'rdf-js'; import type { Guarded } from './GuardedStream'; import { guardedStreamFrom, pipeSafely } from './StreamUtil'; import { toNamedTerm } from './TermUtil'; @@ -45,6 +45,18 @@ export function uniqueQuads(quads: Quad[]): Quad[] { }, []); } +/** + * Converts a term to a number. Returns undefined if the term was undefined. + * + * @param term - Term to parse. + * @param radix - Radix to use when parsing. Default is 10. + */ +export function termToInt(term?: Term, radix = 10): number | undefined { + if (term) { + return Number.parseInt(term.value, radix); + } +} + /** * Represents a triple pattern to be used as a filter. */ diff --git a/src/util/SliceStream.ts b/src/util/SliceStream.ts index 9d8e4b76ef..4f1689c100 100644 --- a/src/util/SliceStream.ts +++ b/src/util/SliceStream.ts @@ -3,12 +3,19 @@ import { Transform } from 'stream'; import { RangeNotSatisfiedHttpError } from './errors/RangeNotSatisfiedHttpError'; import { pipeSafely } from './StreamUtil'; +export interface SliceStreamOptions extends TransformOptions { + start: number; + end?: number; + size?: number; +} + /** * A stream that slices a part out of another stream. * `start` and `end` are inclusive. * If `end` is not defined it is until the end of the stream. - * Does not support negative `start` values which would indicate slicing the end of the stream off, - * since we don't know the length of the input stream. + * + * Negative `start` values can be used to instead slice that many streams off the end of the stream. + * This requires the `size` field to be defined. * * Both object and non-object streams are supported. * This needs to be explicitly specified, @@ -19,16 +26,29 @@ export class SliceStream extends Transform { protected remainingSkip: number; protected remainingRead: number; - public constructor(source: Readable, options: TransformOptions & { start: number; end?: number }) { + public constructor(source: Readable, options: SliceStreamOptions) { super(options); + let start = options.start; const end = options.end ?? Number.POSITIVE_INFINITY; if (options.start < 0) { - throw new RangeNotSatisfiedHttpError('Slicing data at the end of a stream is not supported.'); + if (typeof options.size !== 'number') { + throw new RangeNotSatisfiedHttpError('Slicing data at the end of a stream requires a known size.'); + } else { + // `start` is a negative number here so need to add + start = options.size + start; + } } - if (options.start >= end) { + + if (start >= end) { throw new RangeNotSatisfiedHttpError('Range start should be less than end.'); } - this.remainingSkip = options.start; + + // Not using `end` variable as that could be infinity + if (typeof options.end === 'number' && typeof options.size === 'number' && options.end >= options.size) { + throw new RangeNotSatisfiedHttpError('Range end should be less than the total size.'); + } + + this.remainingSkip = start; // End value is inclusive this.remainingRead = end - options.start + 1; diff --git a/test/integration/LdpHandlerWithoutAuth.test.ts b/test/integration/LdpHandlerWithoutAuth.test.ts index d82dffa214..258d660523 100644 --- a/test/integration/LdpHandlerWithoutAuth.test.ts +++ b/test/integration/LdpHandlerWithoutAuth.test.ts @@ -733,20 +733,23 @@ describe.each(stores)('An LDP handler allowing all requests %s', (name, { storeC let response = await fetch(resourceUrl, { headers: { range: 'bytes=0-5' }}); expect(response.status).toBe(206); - expect(response.headers.get('content-range')).toBe('bytes 0-5/*'); + expect(response.headers.get('content-range')).toBe('bytes 0-5/10'); + expect(response.headers.get('content-length')).toBe('6'); await expect(response.text()).resolves.toBe('012345'); response = await fetch(resourceUrl, { headers: { range: 'bytes=5-' }}); expect(response.status).toBe(206); - expect(response.headers.get('content-range')).toBe('bytes 5-*/*'); + expect(response.headers.get('content-range')).toBe('bytes 5-9/10'); + expect(response.headers.get('content-length')).toBe('5'); await expect(response.text()).resolves.toBe('56789'); - response = await fetch(resourceUrl, { headers: { range: 'bytes=5-15' }}); + response = await fetch(resourceUrl, { headers: { range: 'bytes=-4' }}); expect(response.status).toBe(206); - expect(response.headers.get('content-range')).toBe('bytes 5-15/*'); - await expect(response.text()).resolves.toBe('56789'); + expect(response.headers.get('content-range')).toBe('bytes 6-9/10'); + expect(response.headers.get('content-length')).toBe('4'); + await expect(response.text()).resolves.toBe('6789'); - response = await fetch(resourceUrl, { headers: { range: 'bytes=-5' }}); + response = await fetch(resourceUrl, { headers: { range: 'bytes=5-15' }}); expect(response.status).toBe(416); }); }); diff --git a/test/unit/http/output/metadata/RangeMetadataWriter.test.ts b/test/unit/http/output/metadata/RangeMetadataWriter.test.ts index f2839092c5..5256ff7e1c 100644 --- a/test/unit/http/output/metadata/RangeMetadataWriter.test.ts +++ b/test/unit/http/output/metadata/RangeMetadataWriter.test.ts @@ -2,7 +2,7 @@ import { createResponse } from 'node-mocks-http'; import { RangeMetadataWriter } from '../../../../../src/http/output/metadata/RangeMetadataWriter'; import { RepresentationMetadata } from '../../../../../src/http/representation/RepresentationMetadata'; import type { HttpResponse } from '../../../../../src/server/HttpResponse'; -import { SOLID_HTTP } from '../../../../../src/util/Vocabularies'; +import { POSIX, SOLID_HTTP } from '../../../../../src/util/Vocabularies'; describe('RangeMetadataWriter', (): void => { let metadata: RepresentationMetadata; @@ -15,17 +15,19 @@ describe('RangeMetadataWriter', (): void => { writer = new RangeMetadataWriter(); }); - it('adds the content-range header.', async(): Promise => { + it('adds the content-range and content-length header.', async(): Promise => { metadata.set(SOLID_HTTP.terms.unit, 'bytes'); metadata.set(SOLID_HTTP.terms.start, '1'); metadata.set(SOLID_HTTP.terms.end, '5'); + metadata.set(POSIX.terms.size, '10'); await expect(writer.handle({ response, metadata })).resolves.toBeUndefined(); expect(response.getHeaders()).toEqual({ - 'content-range': 'bytes 1-5/*', + 'content-range': 'bytes 1-5/10', + 'content-length': '5', }); }); - it('uses * if the value is unknown.', async(): Promise => { + it('uses * if a value is unknown.', async(): Promise => { metadata.set(SOLID_HTTP.terms.unit, 'bytes'); await expect(writer.handle({ response, metadata })).resolves.toBeUndefined(); expect(response.getHeaders()).toEqual({ @@ -37,4 +39,34 @@ describe('RangeMetadataWriter', (): void => { await expect(writer.handle({ response, metadata })).resolves.toBeUndefined(); expect(response.getHeaders()).toEqual({ }); }); + + it('adds a content-length header if the size is known.', async(): Promise => { + metadata.set(POSIX.terms.size, '10'); + await expect(writer.handle({ response, metadata })).resolves.toBeUndefined(); + expect(response.getHeaders()).toEqual({ + 'content-length': '10', + }); + }); + + it('correctly deduces end values if the size is known.', async(): Promise => { + metadata.set(SOLID_HTTP.terms.unit, 'bytes'); + metadata.set(SOLID_HTTP.terms.start, '4'); + metadata.set(POSIX.terms.size, '10'); + await expect(writer.handle({ response, metadata })).resolves.toBeUndefined(); + expect(response.getHeaders()).toEqual({ + 'content-range': 'bytes 4-9/10', + 'content-length': '6', + }); + }); + + it('correctly handles negative start values.', async(): Promise => { + metadata.set(SOLID_HTTP.terms.unit, 'bytes'); + metadata.set(SOLID_HTTP.terms.start, '-4'); + metadata.set(POSIX.terms.size, '10'); + await expect(writer.handle({ response, metadata })).resolves.toBeUndefined(); + expect(response.getHeaders()).toEqual({ + 'content-range': 'bytes 6-9/10', + 'content-length': '4', + }); + }); }); diff --git a/test/unit/storage/BinarySliceResourceStore.test.ts b/test/unit/storage/BinarySliceResourceStore.test.ts index 494d5c6df8..9d23151032 100644 --- a/test/unit/storage/BinarySliceResourceStore.test.ts +++ b/test/unit/storage/BinarySliceResourceStore.test.ts @@ -5,7 +5,7 @@ import { ResourceStore } from '../../../src/storage/ResourceStore'; import { InternalServerError } from '../../../src/util/errors/InternalServerError'; import { RangeNotSatisfiedHttpError } from '../../../src/util/errors/RangeNotSatisfiedHttpError'; import { readableToString } from '../../../src/util/StreamUtil'; -import { SOLID_HTTP } from '../../../src/util/Vocabularies'; +import { POSIX, SOLID_HTTP } from '../../../src/util/Vocabularies'; describe('A BinarySliceResourceStore', (): void => { const identifier = { path: 'path' }; @@ -31,6 +31,14 @@ describe('A BinarySliceResourceStore', (): void => { expect(result.metadata.get(SOLID_HTTP.terms.end)?.value).toBe('4'); }); + it('uses the stream size when slicing if available.', async(): Promise => { + representation.metadata.set(POSIX.terms.size, '10'); + const result = await store.getRepresentation(identifier, { range: { unit: 'bytes', parts: [{ start: -4 }]}}); + await expect(readableToString(result.data)).resolves.toBe('6789'); + expect(result.metadata.get(SOLID_HTTP.terms.unit)?.value).toBe('bytes'); + expect(result.metadata.get(SOLID_HTTP.terms.start)?.value).toBe('-4'); + }); + it('does not add end metadata if there is none.', async(): Promise => { const result = await store.getRepresentation(identifier, { range: { unit: 'bytes', parts: [{ start: 5 }]}}); await expect(readableToString(result.data)).resolves.toBe('56789'); diff --git a/test/unit/storage/accessors/InMemoryDataAccessor.test.ts b/test/unit/storage/accessors/InMemoryDataAccessor.test.ts index 3c4780e990..f9e0078504 100644 --- a/test/unit/storage/accessors/InMemoryDataAccessor.test.ts +++ b/test/unit/storage/accessors/InMemoryDataAccessor.test.ts @@ -9,7 +9,7 @@ import { NotFoundHttpError } from '../../../../src/util/errors/NotFoundHttpError import type { Guarded } from '../../../../src/util/GuardedStream'; import { BaseIdentifierStrategy } from '../../../../src/util/identifiers/BaseIdentifierStrategy'; import { guardedStreamFrom, readableToString } from '../../../../src/util/StreamUtil'; -import { LDP, RDF } from '../../../../src/util/Vocabularies'; +import { CONTENT_TYPE, LDP, POSIX, RDF } from '../../../../src/util/Vocabularies'; const { namedNode } = DataFactory; class DummyStrategy extends BaseIdentifierStrategy { @@ -104,13 +104,18 @@ describe('An InMemoryDataAccessor', (): void => { it('adds stored metadata when requesting document metadata.', async(): Promise => { const identifier = { path: `${base}resource` }; - const inputMetadata = new RepresentationMetadata(identifier, { [RDF.type]: LDP.terms.Resource }); + const inputMetadata = new RepresentationMetadata(identifier, { + [RDF.type]: LDP.terms.Resource, + [CONTENT_TYPE]: 'text/turtle', + }); await expect(accessor.writeDocument(identifier, data, inputMetadata)).resolves.toBeUndefined(); metadata = await accessor.getMetadata(identifier); expect(metadata.identifier.value).toBe(`${base}resource`); const quads = metadata.quads(); - expect(quads).toHaveLength(1); - expect(quads[0].object.value).toBe(LDP.Resource); + expect(quads).toHaveLength(3); + expect(metadata.get(RDF.terms.type)).toEqual(LDP.terms.Resource); + expect(metadata.contentType).toBe('text/turtle'); + expect(metadata.get(POSIX.terms.size)?.value).toBe('4'); }); it('adds stored metadata when requesting container metadata.', async(): Promise => { diff --git a/test/unit/storage/conversion/ChainedConverter.test.ts b/test/unit/storage/conversion/ChainedConverter.test.ts index 01d1dbf4e9..df0b7bc6ff 100644 --- a/test/unit/storage/conversion/ChainedConverter.test.ts +++ b/test/unit/storage/conversion/ChainedConverter.test.ts @@ -8,7 +8,7 @@ import { BaseTypedRepresentationConverter } from '../../../../src/storage/conver import { ChainedConverter } from '../../../../src/storage/conversion/ChainedConverter'; import { matchesMediaType } from '../../../../src/storage/conversion/ConversionUtil'; import type { RepresentationConverterArgs } from '../../../../src/storage/conversion/RepresentationConverter'; -import { CONTENT_TYPE } from '../../../../src/util/Vocabularies'; +import { CONTENT_TYPE, POSIX } from '../../../../src/util/Vocabularies'; class DummyConverter extends BaseTypedRepresentationConverter { private readonly inTypes: ValuePreferences; @@ -47,6 +47,7 @@ describe('A ChainedConverter', (): void => { beforeEach(async(): Promise => { const metadata = new RepresentationMetadata('a/a'); + metadata.set(POSIX.terms.size, '500'); representation = { metadata } as Representation; preferences = { type: { 'x/x': 1, 'x/*': 0.8 }}; args = { representation, preferences, identifier: { path: 'path' }}; @@ -81,6 +82,7 @@ describe('A ChainedConverter', (): void => { const result = await converter.handle(args); expect(result.metadata.contentType).toBe('b/b'); + expect(result.metadata.get(POSIX.terms.size)?.value).toBe('500'); }); it('converts input matching the output preferences if a better output can be found.', async(): Promise => { @@ -91,6 +93,7 @@ describe('A ChainedConverter', (): void => { const result = await converter.handle(args); expect(result.metadata.contentType).toBe('x/x'); + expect(result.metadata.get(POSIX.terms.size)).toBeUndefined(); }); it('interprets no preferences as */*.', async(): Promise => { @@ -101,10 +104,12 @@ describe('A ChainedConverter', (): void => { let result = await converter.handle(args); expect(result.metadata.contentType).toBe('b/b'); + expect(result.metadata.get(POSIX.terms.size)?.value).toBe('500'); args.preferences.type = { }; result = await converter.handle(args); expect(result.metadata.contentType).toBe('b/b'); + expect(result.metadata.get(POSIX.terms.size)?.value).toBe('500'); }); it('can find paths of length 1.', async(): Promise => { @@ -113,6 +118,7 @@ describe('A ChainedConverter', (): void => { const result = await converter.handle(args); expect(result.metadata.contentType).toBe('x/x'); + expect(result.metadata.get(POSIX.terms.size)).toBeUndefined(); }); it('can find longer paths.', async(): Promise => { @@ -126,6 +132,7 @@ describe('A ChainedConverter', (): void => { const result = await converter.handle(args); expect(result.metadata.contentType).toBe('x/x'); + expect(result.metadata.get(POSIX.terms.size)).toBeUndefined(); }); it('will use the shortest path among the best found.', async(): Promise => { @@ -147,6 +154,7 @@ describe('A ChainedConverter', (): void => { } const result = await converter.handle(args); expect(result.metadata.contentType).toBe('x/x'); + expect(result.metadata.get(POSIX.terms.size)).toBeUndefined(); expect(converters[0].handle).toHaveBeenCalledTimes(0); expect(converters[1].handle).toHaveBeenCalledTimes(0); expect(converters[2].handle).toHaveBeenCalledTimes(1); diff --git a/test/unit/util/QuadUtil.test.ts b/test/unit/util/QuadUtil.test.ts index 26856960ca..11c3413d53 100644 --- a/test/unit/util/QuadUtil.test.ts +++ b/test/unit/util/QuadUtil.test.ts @@ -1,6 +1,6 @@ import 'jest-rdf'; import { DataFactory } from 'n3'; -import { parseQuads, serializeQuads, uniqueQuads } from '../../../src/util/QuadUtil'; +import { parseQuads, serializeQuads, termToInt, uniqueQuads } from '../../../src/util/QuadUtil'; import { guardedStreamFrom, readableToString } from '../../../src/util/StreamUtil'; const { literal, namedNode, quad } = DataFactory; @@ -50,4 +50,15 @@ describe('QuadUtil', (): void => { ]); }); }); + + describe('#termToInt', (): void => { + it('returns undefined if the input is undefined.', async(): Promise => { + expect(termToInt()).toBeUndefined(); + }); + + it('converts the term to a number.', async(): Promise => { + expect(termToInt(namedNode('5'))).toBe(5); + expect(termToInt(namedNode('0xF'), 16)).toBe(15); + }); + }); }); diff --git a/test/unit/util/SliceStream.test.ts b/test/unit/util/SliceStream.test.ts index 8a9a68dda6..27989b0db5 100644 --- a/test/unit/util/SliceStream.test.ts +++ b/test/unit/util/SliceStream.test.ts @@ -4,7 +4,7 @@ import { SliceStream } from '../../../src/util/SliceStream'; import { readableToString } from '../../../src/util/StreamUtil'; describe('A SliceStream', (): void => { - it('does not support suffix slicing.', async(): Promise => { + it('does not support suffix slicing if the size is unknown.', async(): Promise => { expect((): unknown => new SliceStream(Readable.from('0123456789'), { start: -5 })) .toThrow(RangeNotSatisfiedHttpError); }); @@ -16,6 +16,11 @@ describe('A SliceStream', (): void => { .toThrow(RangeNotSatisfiedHttpError); }); + it('requires the end to be less than the size.', async(): Promise => { + expect((): unknown => new SliceStream(Readable.from('0123456789'), { start: 5, end: 6, size: 6 })) + .toThrow(RangeNotSatisfiedHttpError); + }); + it('can slice binary streams.', async(): Promise => { await expect(readableToString(new SliceStream(Readable.from('0123456789', { objectMode: false }), { start: 3, end: 7, objectMode: false }))).resolves.toBe('34567'); @@ -25,6 +30,9 @@ describe('A SliceStream', (): void => { await expect(readableToString(new SliceStream(Readable.from('0123456789', { objectMode: false }), { start: 3, end: 20, objectMode: false }))).resolves.toBe('3456789'); + + await expect(readableToString(new SliceStream(Readable.from('0123456789', { objectMode: false }), + { start: -3, size: 10, objectMode: false }))).resolves.toBe('789'); }); it('can slice object streams.', async(): Promise => { @@ -37,5 +45,8 @@ describe('A SliceStream', (): void => { await expect(readableToString(new SliceStream(Readable.from(arr, { objectMode: true }), { start: 3, end: 20, objectMode: true }))).resolves.toBe('3456789'); + + await expect(readableToString(new SliceStream(Readable.from(arr, { objectMode: true }), + { start: -3, size: 10, objectMode: true }))).resolves.toBe('789'); }); }); From 44f731d7e7251d4636faf39588f64f04b560e403 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Thu, 5 Oct 2023 11:04:01 +0200 Subject: [PATCH 22/25] test: Initialize empty pods when testing quota Another attempt to help fix random CI integration test failures --- test/integration/Quota.test.ts | 26 +++-------------------- test/integration/config/quota-global.json | 9 ++++++++ test/integration/config/quota-pod.json | 9 ++++++++ 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/test/integration/Quota.test.ts b/test/integration/Quota.test.ts index 66ebaf55ad..6f575e9d5c 100644 --- a/test/integration/Quota.test.ts +++ b/test/integration/Quota.test.ts @@ -1,9 +1,8 @@ -import { promises as fsPromises } from 'fs'; import type { Stats } from 'fs'; import fetch from 'cross-fetch'; import type { Response } from 'cross-fetch'; -import { ensureDir, pathExists } from 'fs-extra'; -import { joinFilePath, joinUrl } from '../../src'; +import { ensureDir, pathExists, stat } from 'fs-extra'; +import { joinUrl } from '../../src'; import type { App } from '../../src'; import { getPort } from '../util/Util'; import { getDefaultVariables, getTestConfigPath, getTestFolder, instantiateFromConfig, removeFolder } from './Config'; @@ -45,23 +44,6 @@ async function registerTestPods(baseUrl: string, pods: string[]): Promise } } -/* We just want a container with the correct metadata, everything else can be removed */ -async function clearInitialFiles(rootFilePath: string, pods: string[]): Promise { - for (const pod of pods) { - const fileList = await fsPromises.readdir(joinFilePath(rootFilePath, pod)); - for (const file of fileList) { - if (file !== '.meta') { - const path = joinFilePath(rootFilePath, pod, file); - if ((await fsPromises.stat(path)).isDirectory()) { - await fsPromises.rm(path, { recursive: true }); - } else { - await fsPromises.unlink(path); - } - } - } - } -} - describe('A quota server', (): void => { // The allowed quota depends on what filesystem/OS you are using. // For example: an empty folder is reported as @@ -74,7 +56,7 @@ describe('A quota server', (): void => { // We want to use an empty folder as on APFS/Mac folder sizes vary a lot const tempFolder = getTestFolder('quota-temp'); await ensureDir(tempFolder); - folderSizeTest = await fsPromises.stat(tempFolder); + folderSizeTest = await stat(tempFolder); await removeFolder(tempFolder); }); const podName1 = 'arthur'; @@ -108,7 +90,6 @@ describe('A quota server', (): void => { // Initialize 2 pods await registerTestPods(baseUrl, [ podName1, podName2 ]); - await clearInitialFiles(rootFilePath, [ podName1, podName2 ]); }); afterAll(async(): Promise => { @@ -194,7 +175,6 @@ describe('A quota server', (): void => { // Initialize 2 pods await registerTestPods(baseUrl, [ podName1, podName2 ]); - await clearInitialFiles(rootFilePath, [ podName1, podName2 ]); }); afterAll(async(): Promise => { diff --git a/test/integration/config/quota-global.json b/test/integration/config/quota-global.json index 604fe1b0a2..a0a4c02183 100644 --- a/test/integration/config/quota-global.json +++ b/test/integration/config/quota-global.json @@ -54,6 +54,15 @@ "@type": "FileSizeReporter", "ignoreFolders": [ "^/\\.internal$" ] }, + { + "comment": "Use an empty pod for quota tests", + "@type": "Override", + "overrideInstance": { "@id": "urn:solid-server:default:PodResourcesGenerator" }, + "overrideParameters": { + "@type": "StaticFolderGenerator", + "templateFolder": "@css:templates/root/empty" + } + }, { "@id": "urn:solid-server:test:Instances", "@type": "RecordObject", diff --git a/test/integration/config/quota-pod.json b/test/integration/config/quota-pod.json index 6488dffa35..6e8e3fc281 100644 --- a/test/integration/config/quota-pod.json +++ b/test/integration/config/quota-pod.json @@ -49,6 +49,15 @@ }, "limit_unit": "bytes" }, + { + "comment": "Use an empty pod for quota tests", + "@type": "Override", + "overrideInstance": { "@id": "urn:solid-server:default:PodResourcesGenerator" }, + "overrideParameters": { + "@type": "StaticFolderGenerator", + "templateFolder": "@css:templates/root/empty" + } + }, { "@id": "urn:solid-server:test:Instances", "@type": "RecordObject", From da46becf7a087118e7d682a193d00a3ca6c32eab Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Thu, 5 Oct 2023 13:04:33 +0200 Subject: [PATCH 23/25] fix: Prevent error when creating a root pod --- templates/identity/email-password/register-partial.html.ejs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/identity/email-password/register-partial.html.ejs b/templates/identity/email-password/register-partial.html.ejs index 748d0223ad..4fe214c013 100644 --- a/templates/identity/email-password/register-partial.html.ejs +++ b/templates/identity/email-password/register-partial.html.ejs @@ -59,7 +59,9 @@ Create a new Pod with my WebID as owner<% if (locals.allowRootPod) { %> in the root<% } %>.
    - <% if (!locals.allowRootPod) { %> + <% if (locals.allowRootPod) { %> + + <% } else { %>
  1. From fdb39f6df403c262e01555b796579f11f365382c Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Thu, 5 Oct 2023 11:32:22 +0200 Subject: [PATCH 24/25] docs: Add v6.1.0 features to RELEASE_NOTES.md --- CHANGELOG.md | 18 ++++++++++++++++++ RELEASE_NOTES.md | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4e547b2b4..e0be676716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,24 @@ All notable changes to this project will be documented in this file. +## [6.1.0](https://github.com/CommunitySolidServer/CommunitySolidServer/compare/v6.0.2...v6.1.0) (2023-10-05) + +### Features + +* Track binary size of resources when possible ([71e5569](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/71e55690f3418be3d08e35d2cd3aeae5a0634654)) +* Add support for range headers ([3e9adef](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/3e9adef4cf00d0776c0d371f835a31511db7427b)) + +### Fixes + +* Prevent error when creating a root pod([da46bec](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/da46becf7a087118e7d682a193d00a3ca6c32eab)) +* Remove URL encoding from base64 strings before decoding ([d31393f](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/d31393f4751dd3f023110ead4e47a01ac15da2af)) + +### Documentation + +* Simplify README by pointing to our docs. ([d618f97](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/d618f9781af80b1697d5fe23f50e3f186954792b)) +* Add starting guide. ([e424b84](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/e424b8488261bc8942d82a7fe2d92a94650e93b9)) +* Add quick start to README. ([1fa6d24](https://github.com/CommunitySolidServer/CommunitySolidServer/commit/1fa6d248a2e500c025794f4e3ed6cc504ed77f10)) + ## [6.0.2](https://github.com/CommunitySolidServer/CommunitySolidServer/compare/v6.0.1...v6.0.2) (2023-08-30) ### Fixes diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index cdbffebd0c..4e66000f67 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,11 @@ # Community Solid Server release notes +## v6.1.0 + +### New features + +- Added support for HTTP Range headers. + ## v6.0.0 ### New features From 5c0c54c31cbfe4d3528a1bfe4f1044f7ed2a8066 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Thu, 5 Oct 2023 11:32:53 +0200 Subject: [PATCH 25/25] chore(release): Release version 6.1.0 of the npm package --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index f6695e21f7..6420a02c82 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@solid/community-server", - "version": "6.0.2", + "version": "6.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@solid/community-server", - "version": "6.0.2", + "version": "6.1.0", "license": "MIT", "dependencies": { "@comunica/context-entries": "^2.6.8", diff --git a/package.json b/package.json index a1e842c0b1..fbc1a93a61 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@solid/community-server", - "version": "6.0.2", + "version": "6.1.0", "description": "Community Solid Server: an open and modular implementation of the Solid specifications", "keywords": [ "solid",