fix: Small fixes

RELEASE_NOTES.md

@@ -4,6 +4,7 @@
 ### New features
 - The Identity Provider now uses the `webid` scope as required for Solid-OIDC.
 - The `VoidLocker` can be used to disable locking for development/testing purposes. This can be enabled by changing the `/config/util/resource-locker/` import to `debug-void.json`
+- Added support for setting a quota on the server. See the `config/quota-file.json` config for an example.
 
 ### Configuration changes
 You might need to make changes to your v2 configuration if you use a custom config.

config/quota-file.json

@@ -33,14 +33,12 @@
   ],
   "@graph": [
     {
-      "comment": "A single-pod server that stores its resources on disk while enforcing quota."
+      "comment": "A server that stores its resources on disk while enforcing quota."
     },
     {
       "@id": "urn:solid-server:default:QuotaStrategy",
       "PodQuotaStrategy:_limit_amount": 7000,
-      "PodQuotaStrategy:_limit_unit": "bytes",
-      "GlobalQuotaStrategy:_limit_amount": 10000,
-      "GlobalQuotaStrategy:_limit_unit": "bytes"
+      "PodQuotaStrategy:_limit_unit": "bytes"
     },
     {
       "@id": "urn:solid-server:default:SizeReporter",

config/storage/README.md

@@ -5,7 +5,9 @@ Options related to how data and resources are stored.
 The final part of the ResourceStore chain that handles data access.
 * *dynamic*: The routing store used here is needed when using dynamic pod creation.
 * *file*: Default setup with a file backend.
+* *global-quota-file*: File backend with a global quota over the entire server.
 * *memory*: Default setup with a memory backend.
+* *pod-quota-file*: File backend with a max quota per pod.
 * *regex*: Uses a different backend based on the container that is being used.
 * *sparql*: Default setup with a SPARQL endpoint backend.
   Also updates the converting store so all incoming data is transformed into quads.

config/storage/backend/global-quota-file.json

@@ -1,11 +1,12 @@
 {
   "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^2.0.0/components/context.jsonld",
   "import": [
-    "files-scs:config/storage/backend/quota/global-quota-file.json"
+    "files-scs:config/storage/backend/quota/global-quota-file.json",
+    "files-scs:config/storage/backend/quota/quota-file.json"
   ],
   "@graph": [
     {
-      "comment": "A default store setup with a file system backend.",
+      "comment": "A global quota store setup with a file system backend.",
       "@id": "urn:solid-server:default:ResourceStore_Backend",
       "@type": "DataAccessorBasedStore",
       "identifierStrategy": { "@id": "urn:solid-server:default:IdentifierStrategy" },

config/storage/backend/pod-quota-file.json

@@ -1,11 +1,12 @@
 {
   "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^2.0.0/components/context.jsonld",
   "import": [
-    "files-scs:config/storage/backend/quota/pod-quota-file.json"
+    "files-scs:config/storage/backend/quota/pod-quota-file.json",
+    "files-scs:config/storage/backend/quota/quota-file.json"
   ],
   "@graph": [
     {
-      "comment": "A default store setup with a file system backend.",
+      "comment": "A pod quota store setup with a file system backend.",
       "@id": "urn:solid-server:default:ResourceStore_Backend",
       "@type": "DataAccessorBasedStore",
       "identifierStrategy": { "@id": "urn:solid-server:default:IdentifierStrategy" },

config/storage/backend/quota/global-quota-file.json

@@ -1,8 +1,5 @@
 {
   "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^2.0.0/components/context.jsonld",
-  "import": [
-    "files-scs:config/storage/backend/quota/quota-file.json"
-  ],
   "comment": "Configuration of a GlobalQuotaStrategy to enforce quota globally on the server.",
   "@graph": [
     {

config/storage/backend/quota/pod-quota-file.json

@@ -1,8 +1,5 @@
 {
   "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^2.0.0/components/context.jsonld",
-  "import": [
-    "files-scs:config/storage/backend/quota/quota-file.json"
-  ],
   "comment": "Configuration of a PodQuotaStrategy to enforce pod quotas on the server.",
   "@graph": [
     {

src/storage/accessors/AtomicFileDataAccessor.ts

@@ -1,10 +1,10 @@
 import { mkdirSync, promises as fsPromises } from 'fs';
-import { join } from 'path';
 import type { Readable } from 'stream';
 import { v4 } from 'uuid';
 import type { RepresentationMetadata } from '../../http/representation/RepresentationMetadata';
 import type { ResourceIdentifier } from '../../http/representation/ResourceIdentifier';
 import type { Guarded } from '../../util/GuardedStream';
+import { joinFilePath } from '../../util/PathUtil';
 import type { FileIdentifierMapper } from '../mapping/FileIdentifierMapper';
 import type { AtomicDataAccessor } from './AtomicDataAccessor';
 import { FileDataAccessor } from './FileDataAccessor';
@@ -19,7 +19,7 @@ export class AtomicFileDataAccessor extends FileDataAccessor implements AtomicDa
 
   public constructor(resourceMapper: FileIdentifierMapper, rootFilePath: string, tempFilePath: string) {
     super(resourceMapper);
-    this.tempFilePath = join(rootFilePath, tempFilePath);
+    this.tempFilePath = joinFilePath(rootFilePath, tempFilePath);
     // Cannot use fsPromises in constructor
     mkdirSync(this.tempFilePath, { recursive: true });
   }
@@ -36,7 +36,7 @@ export class AtomicFileDataAccessor extends FileDataAccessor implements AtomicDa
     const link = await this.resourceMapper.mapUrlToFilePath(identifier, false, metadata.contentType);
 
     // Generate temporary file name
-    const tempFilePath = join(this.tempFilePath, `temp-${v4()}.txt`);
+    const tempFilePath = joinFilePath(this.tempFilePath, `temp-${v4()}.txt`);
 
     try {
       await this.writeDataFile(tempFilePath, data);

src/storage/quota/QuotaStrategy.ts

@@ -55,10 +55,10 @@ export abstract class QuotaStrategy {
   }
 
   /**
-   * Get the currently used/occupied space
+   * Get the currently used/occupied space.
    *
    * @param identifier - the identifier that should be used to calculate the total
-   * @returns a Size object containing the the requested value.
+   * @returns a Size object containing the requested value.
    * If quota is not relevant for this identifier, Size.amount should be Number.MAX_SAFE_INTEGER
    */
   protected abstract getTotalSpaceUsed(identifier: ResourceIdentifier): Promise<Size>;
@@ -80,8 +80,6 @@ export abstract class QuotaStrategy {
    * Like other Passthrough instances this will simply pass on the chunks, when the quota isn't exceeded.
    *
    * @param identifier - the identifier of the resource in question
-   * @param data - the Readable stream that belongs to the identifier
-   * @param metadata - the RepresentationMetadata that belongs to the identifier
    * @returns a Passthrough instance that errors when quota is exceeded
    */
   public async createQuotaGuard(identifier: ResourceIdentifier): Promise<Guarded<PassThrough>> {

test/integration/Quota.test.ts

@@ -1,8 +1,8 @@
 import { promises as fsPromises } from 'fs';
 import type { Stats } from 'fs';
-import { join } from 'path';
 import fetch from 'cross-fetch';
 import type { Response } from 'cross-fetch';
+import { joinFilePath, joinUrl } from '../../src';
 import type { App } from '../../src';
 import { getPort } from '../util/Util';
 import { getDefaultVariables, getTestConfigPath, getTestFolder, instantiateFromConfig, removeFolder } from './Config';
@@ -44,12 +44,13 @@ async function registerTestPods(baseUrl: string, pods: string[]): Promise<void>
   }
 }
 
+/* We just want a container with the correct metadata, everything else can be removed */
 async function clearInitialFiles(rootFilePath: string, pods: string[]): Promise<void> {
   for (const pod of pods) {
-    const fileList = await fsPromises.readdir(join(rootFilePath, pod));
+    const fileList = await fsPromises.readdir(joinFilePath(rootFilePath, pod));
     for (const file of fileList) {
       if (file !== '.meta') {
-        const path = join(rootFilePath, pod, file);
+        const path = joinFilePath(rootFilePath, pod, file);
         if ((await fsPromises.stat(path)).isDirectory()) {
           await fsPromises.rmdir(path, { recursive: true });
         } else {
@@ -60,31 +61,30 @@ async function clearInitialFiles(rootFilePath: string, pods: string[]): Promise<
   }
 }
 
-describe('A quota server with', (): void => {
+describe('A quota server', (): void => {
   // The allowed quota depends on what filesystem/OS you are using.
   // For example: an empty folder is reported as
-  //  0KB on NTFS (most of the times, milage may vary)
-  //  0-...KB on APFS (depending on its contents and settings)
-  //  4O96KB on FAT
-  // While I am running these tests on a macBook, Github runs them on a
-  // mounted FAT drive and you might be running them on Windows/NTFS.
+  //  - 0KB on NTFS (most of the time, mileage may vary)
+  //  - 0-...KB on APFS (depending on its contents and settings)
+  //  - 4O96KB on FAT
+  // This is why we need to determine the size of a folder on the current system.
   let folderSizeTest: Stats;
   beforeAll(async(): Promise<void> => {
     // We want to use an empty folder as on APFS/Mac folder sizes vary a lot
-    const tempFolder = join(process.cwd(), 'tempFolderForTest');
+    const tempFolder = getTestFolder('quota-temp');
     await fsPromises.mkdir(tempFolder);
     folderSizeTest = await fsPromises.stat(tempFolder);
-    await fsPromises.rmdir(tempFolder, { recursive: true });
+    await removeFolder(tempFolder);
   });
   const podName1 = 'arthur';
   const podName2 = 'abel';
 
   /** Test the general functionality of the server using pod quota */
-  describe('pod quota enabled', (): void => {
+  describe('with pod quota enabled', (): void => {
     const port = getPort('PodQuota');
     const baseUrl = `http://localhost:${port}/`;
-    const pod1 = `${baseUrl}${podName1}`;
-    const pod2 = `${baseUrl}${podName2}`;
+    const pod1 = joinUrl(baseUrl, podName1);
+    const pod2 = joinUrl(baseUrl, podName2);
     const rootFilePath = getTestFolder('quota-pod');
 
     let app: App;
@@ -124,7 +124,7 @@ describe('A quota server with', (): void => {
       await expect(response1).resolves.toBeDefined();
       expect((await response1).status).toEqual(201);
 
-      const response2 = performSimplePutWithLength(testFile2, 2000);
+      const response2 = performSimplePutWithLength(testFile2, 2500);
       await expect(response2).resolves.toBeDefined();
       expect((await response2).status).toEqual(413);
     });
@@ -143,18 +143,18 @@ describe('A quota server with', (): void => {
       const testFile1 = `${pod1}/test2.txt`;
       const testFile2 = `${pod2}/test2.txt`;
 
-      const response1 = performSimplePutWithLength(testFile1, 2000);
+      const response1 = performSimplePutWithLength(testFile1, 2500);
       await expect(response1).resolves.toBeDefined();
       expect((await response1).status).toEqual(413);
 
-      const response2 = performSimplePutWithLength(testFile2, 2000);
+      const response2 = performSimplePutWithLength(testFile2, 2500);
       await expect(response2).resolves.toBeDefined();
       expect((await response2).status).toEqual(413);
     });
   });
 
   /** Test the general functionality of the server using global quota */
-  describe('global quota enabled', (): void => {
+  describe('with global quota enabled', (): void => {
     const port = getPort('GlobalQuota');
     const baseUrl = `http://localhost:${port}/`;
     const pod1 = `${baseUrl}${podName1}`;
@@ -198,7 +198,7 @@ describe('A quota server with', (): void => {
       const awaitedRes1 = await response1;
       expect(awaitedRes1.status).toEqual(201);
 
-      const response2 = performSimplePutWithLength(testFile2, 2000);
+      const response2 = performSimplePutWithLength(testFile2, 2500);
       await expect(response2).resolves.toBeDefined();
       const awaitedRes2 = await response2;
       expect(awaitedRes2.status).toEqual(413);
@@ -208,12 +208,12 @@ describe('A quota server with', (): void => {
       const testFile1 = `${pod1}/test3.txt`;
       const testFile2 = `${pod2}/test4.txt`;
 
-      const response1 = performSimplePutWithLength(testFile1, 2000);
+      const response1 = performSimplePutWithLength(testFile1, 2500);
       await expect(response1).resolves.toBeDefined();
       const awaitedRes1 = await response1;
       expect(awaitedRes1.status).toEqual(413);
 
-      const response2 = performSimplePutWithLength(testFile2, 2000);
+      const response2 = performSimplePutWithLength(testFile2, 2500);
       await expect(response2).resolves.toBeDefined();
       const awaitedRes2 = await response2;
       expect(awaitedRes2.status).toEqual(413);

test/unit/quota/PodQuotaStrategy.test.ts

@@ -66,7 +66,7 @@ describe('PodQuotaStrategy', (): void => {
       await expect(result).resolves.toEqual(expect.objectContaining({ amount: mockSize.amount }));
       expect(mockReporter.getSize).toHaveBeenCalledTimes(2);
     });
-    it('should throw when looking for pimStorage errors.', async(): Promise<void> => {
+    it('should throw when looking for pim:Storage errors.', async(): Promise<void> => {
       accessor.getMetadata.mockImplementationOnce((): any => {
         throw new Error('error');
       });