feat(*): implement the base logic for uploading files and handling disks

Still need to implement delete and transfer and write some tests for the repository functionality
before this is ready to ship.

Author: bericp1

.eslintrc.json

@@ -32,6 +32,7 @@
         "node/prefer-global/url": ["error", "always"],
         "node/no-unsupported-features/es-syntax": "off",
         "@typescript-eslint/explicit-function-return-type": "error",
+        "@typescript-eslint/no-explicit-any": "off",
         "curly": ["error", "all"],
         "max-len": ["error", { "code": 140, "ignoreUrls": true }],
         "no-undefined": "error",

README.md

@@ -20,6 +20,16 @@ npm install --save @carimus/node-uploads
 
 TODO
 
+## TODO
+
+-   [ ] Usage docs
+-   [ ] Detailed API docs
+-   [ ] Do URL generation for publicly available disks.
+-   [ ] Support deleting files was @carimus/node-disks supports it
+-   [ ] Support temporary URLs (e.g. presigned URLs for S3 buckets) for disks that support it
+-   [ ] Support transfer logic for transferring single uploads from one disk to another and in bulk.
+-   [ ] Support `getTemporaryFile` to copy an upload file to the local filesystem tmp directory for direct manipulation
+
 ## Development
 
 This project is based on the `carimus-node-ts-package-template`. Check out the

package.json

@@ -74,5 +74,8 @@
         "extends": [
             "@commitlint/config-conventional"
         ]
+    },
+    "dependencies": {
+        "@carimus/node-disks": "^1.2.0"
     }
 }

src/errors/InvalidConfigError.ts

@@ -0,0 +1,9 @@
+export class InvalidConfigError extends Error {
+    public constructor(missing: string[] = []) {
+        super(
+            missing.length === 0
+                ? 'Invalid configuration provided.'
+                : `Missing required configuration: ${missing.join(', ')}`,
+        );
+    }
+}

src/errors/PathNotUniqueError.ts

@@ -0,0 +1,14 @@
+export class PathNotUniqueError extends Error {
+    public constructor(
+        disk: string,
+        oldPath: string,
+        newPath: string,
+        operation: string,
+    ) {
+        super(
+            `Generated path is not unique on the same disk during ${operation}: ` +
+                `'${oldPath}' -> '${newPath}'. If you provided a custom \`generatePath\` function, ensure it` +
+                `always generates unique paths (i.e. include a timestamp).`,
+        );
+    }
+}

src/errors/index.ts

@@ -0,0 +1,2 @@
+export { InvalidConfigError } from './InvalidConfigError';
+export { PathNotUniqueError } from './PathNotUniqueError';

src/index.test.ts

@@ -1,4 +1,7 @@
-export function main(): boolean {
-    console.log('this is a test');
-    return true;
-}
+// Export types
+export * from './types';
+
+// Export classes and other public interfaces
+export * from './lib/Uploads';
+export * from './lib/defaults';
+export * from './errors';

src/index.ts

@@ -0,0 +1,240 @@
+import { Readable } from 'stream';
+import { DiskManager } from '@carimus/node-disks';
+import { InvalidConfigError } from '../errors/InvalidConfigError';
+import {
+    Upload,
+    UploadedFile,
+    UploadMeta,
+    UploadRepository,
+    UploadsConfig,
+} from '../types';
+import { trimPath } from './utils';
+import { defaultSanitizeFilename, defaultGeneratePath } from './defaults';
+import { PathNotUniqueError } from '../errors/PathNotUniqueError';
+
+/**
+ * A service for handling uploaded files.
+ *
+ * TODO Do URL generation for publicly available disks.
+ * TODO Support deleting files was @carimus/node-disks supports it
+ * TODO Support temporary URLs (e.g. presigned URLs for S3 buckets) for disks that support it
+ * TODO Support transfer logic for transferring single uploads from one disk to another and in bulk.
+ * TODO Support `getTemporaryFile` to copy an upload file to the local filesystem tmp directory for direct manipulation
+ */
+export class Uploads {
+    private config: UploadsConfig;
+    private repository: UploadRepository;
+    private disks: DiskManager;
+
+    public constructor(config: UploadsConfig) {
+        this.config = config;
+        this.repository = config.repository;
+        if (typeof config.disks === 'object') {
+            this.disks =
+                config.disks instanceof DiskManager
+                    ? config.disks
+                    : new DiskManager(config.disks);
+        } else {
+            throw new InvalidConfigError(['disks']);
+        }
+    }
+
+    /**
+     * Sanitize a client provided filename before storing on the disk.
+     *
+     * @param uploadedAs
+     */
+    public sanitizeFilename(uploadedAs: string): string {
+        return this.config.sanitizeFilename
+            ? this.config.sanitizeFilename(uploadedAs)
+            : defaultSanitizeFilename(uploadedAs);
+    }
+
+    /**
+     * Generate a timestamped unique path and filename based on the client-provided filename.
+     *
+     * @param sanitizedUploadedAs
+     */
+    public generatePath(sanitizedUploadedAs: string): string {
+        return this.config.generatePath
+            ? this.config.generatePath(sanitizedUploadedAs)
+            : defaultGeneratePath(sanitizedUploadedAs);
+    }
+
+    /**
+     * Get the full final storage path on the disk for an upload being stored based off of the sanitized filename
+     * and the configuration path prefix.
+     *
+     * @param sanitizedUploadedAs
+     */
+    public generateStoragePath(sanitizedUploadedAs: string): string {
+        const path = this.generatePath(sanitizedUploadedAs);
+        return `/${trimPath(this.config.pathPrefix || '')}/${trimPath(path)}`;
+    }
+
+    /**
+     * Get the default disk name based on config.
+     */
+    private getDefaultDiskName(): string {
+        return this.config.defaultDisk || 'default';
+    }
+
+    /**
+     * Take the file data and raw client-provided filename and place that file on the disk in the proper
+     * location by sanitizing the filename and generating a unique path for it.
+     *
+     * @param fileData
+     * @param uploadedAs
+     */
+    public async place(
+        fileData: Buffer | Readable,
+        uploadedAs: string,
+    ): Promise<UploadedFile> {
+        // Generate a filename and path.
+        const sanitizedUploadedAs = this.sanitizeFilename(uploadedAs);
+        const path = this.generateStoragePath(sanitizedUploadedAs);
+        // Grab a disk instance and write the data to the disk.
+        const diskName = this.getDefaultDiskName();
+        const disk = this.disks.getDisk(diskName);
+        await disk.write(path, fileData);
+        // Return a record representing the uploaded file and where it lives on the disk.
+        return {
+            disk: diskName,
+            path,
+            uploadedAs: sanitizedUploadedAs,
+        };
+    }
+
+    /**
+     * Take an uploaded file and copy it to a new unique generated location on the default disk, regardless of the
+     * source disk.
+     *
+     * TODO Once @carimus/node-disks supports the copy operation, use that when oldDisk === newDisk
+     *
+     * @param originalFile
+     */
+    public async copy(originalFile: UploadedFile): Promise<UploadedFile> {
+        // Get the current disk
+        const newDiskName = this.getDefaultDiskName();
+
+        // Clone the original file upload info, setting the new disk and regenerating the path.
+        const newFile: UploadedFile = {
+            ...originalFile,
+            disk: newDiskName,
+            path: this.generateStoragePath(originalFile.uploadedAs),
+        };
+
+        // Throw if the locations are exactly the same.
+        if (
+            originalFile.disk === newFile.disk &&
+            originalFile.path === newFile.path
+        ) {
+            throw new PathNotUniqueError(
+                originalFile.disk,
+                originalFile.path,
+                newFile.path,
+                'copy',
+            );
+        }
+
+        // Resolve the disks
+        const originalDisk = this.disks.getDisk(originalFile.disk);
+        const newDisk = this.disks.getDisk(newFile.disk);
+
+        // Perform the copy
+        await newDisk.write(
+            originalFile.path,
+            await originalDisk.createReadStream(newFile.path),
+        );
+
+        // Return the newly uploaded file
+        return newFile;
+    }
+
+    /**
+     * Place an uploaded file on the disk and create it in the repository.
+     *
+     * @param fileData
+     * @param uploadedAs
+     * @param meta
+     */
+    public async upload(
+        fileData: Buffer | Readable,
+        uploadedAs: string,
+        meta: UploadMeta = {},
+    ): Promise<Upload> {
+        const uploadedFile = await this.place(fileData, uploadedAs);
+        return this.repository.create(uploadedFile, meta);
+    }
+
+    /**
+     * Update an existing upload with a new uploaded file by placing the new file and deleting the old file, updating
+     * the upload itself in the repository.
+     *
+     * @param upload
+     * @param fileData
+     * @param uploadedAs
+     * @param meta
+     */
+    public async update(
+        upload: Upload,
+        fileData: Buffer | Readable,
+        uploadedAs: string,
+        meta: UploadMeta = {},
+    ): Promise<Upload> {
+        const uploadedFile = await this.place(fileData, uploadedAs);
+        return this.repository.update(upload, uploadedFile, meta);
+    }
+
+    /**
+     * Duplicate an existing upload to the default disk no matter what the current uploads disk is and create it in
+     * the repository.
+     *
+     * This will regenerate the path even if the upload is remaining on the same disk.
+     *
+     * The original upload will not be touched.
+     *
+     *
+     * @param original
+     * @param meta
+     */
+    public async duplicate(
+        original: Upload,
+        meta: UploadMeta = {},
+    ): Promise<Upload> {
+        // Ask the repository for info on where and how the original upload file is stored.
+        const originalFile = await this.repository.getUploadedFileInfo(
+            original,
+        );
+
+        // Copy the original file to the new file location.
+        const newFile = await this.copy(originalFile);
+
+        // Store the new upload in the repository and return it
+        return this.repository.create(newFile, meta);
+    }
+
+    // /**
+    //  * Delete an uploaded file from the disk.
+    //  *
+    //  * TODO Actually delete off the disk once @carimus/node-disks implements the operation 😅
+    //  *
+    //  * @param upload
+    //  * @param onlyFile
+    //  */
+    // public async delete(upload: Upload, onlyFile: boolean = false): Promise<void> {
+    //     // Ask the repository for info on where and how the upload file is stored.
+    //     const file = await this.repository.getUploadedFileInfo(upload);
+    //
+    //     // Resolve the disk for the file
+    //     const disk = this.disks.getDisk(file.disk);
+    //
+    //     // Delete the upload in the repository before deleting it on the disk
+    //     if (!onlyFile) {
+    //         await this.repository.delete(upload);
+    //     }
+    //
+    //     // Delete the file on the disk (See jsDoc)
+    //     await disk.delete(file.path);
+    // }
+}

src/lib/Uploads.ts

@@ -0,0 +1,36 @@
+import {
+    defaultSanitizeFilename,
+    defaultGeneratePathForInstant,
+} from './defaults';
+
+test("defaultSanitizeFilename doesn't modify string with only basic allowed characters", () => {
+    expect(defaultSanitizeFilename('foo_123.txt')).toBe('foo_123.txt');
+});
+
+test('defaultSanitizeFilename handles whitespace on the ends', () => {
+    expect(defaultSanitizeFilename(' foo_123.txt  ')).toBe('foo_123.txt');
+});
+
+test('defaultSanitizeFilename replaces invalid characters', () => {
+    expect(defaultSanitizeFilename('foo_1   2-3.txt')).toBe(
+        'foo_1______2-3.txt',
+    );
+    expect(defaultSanitizeFilename('test*&^%$)¶§∞¶•∆µ.abc••')).toBe(
+        'test__________________________.abc____',
+    );
+});
+
+test('defaultGeneratePathForInstant generates paths in the format YYYY/MM/DD/HHmmssuuu-name.ext', () => {
+    expect(
+        defaultGeneratePathForInstant(
+            new Date('2019-04-04T23:52:26.473Z'),
+            'test.png',
+        ),
+    ).toBe('2019/04/04/235226473-test.png');
+    expect(
+        defaultGeneratePathForInstant(
+            new Date('1992-12-04T05:02:00.000Z'),
+            'test.png',
+        ),
+    ).toBe('1992/12/04/050200000-test.png');
+});