feat(core): support withTempFile to get local temp files from uploads

Author: bericp1

.eslintrc.json

@@ -33,6 +33,7 @@
         "node/no-unsupported-features/es-syntax": "off",
         "@typescript-eslint/explicit-function-return-type": "error",
         "@typescript-eslint/no-explicit-any": "off",
+        "@typescript-eslint/no-unused-vars": "error",
         "curly": ["error", "all"],
         "max-len": ["error", { "code": 140, "ignoreUrls": true }],
         "no-undefined": "error",

package.json

@@ -39,6 +39,7 @@
         "@commitlint/cli": "^7.5.2",
         "@commitlint/config-conventional": "^7.5.0",
         "@types/jest": "^24.0.11",
+        "@types/tmp": "^0.1.0",
         "@typescript-eslint/eslint-plugin": "^1.5.0",
         "@typescript-eslint/parser": "^1.5.0",
         "commitizen": "^3.0.7",
@@ -92,6 +93,7 @@
         ]
     },
     "dependencies": {
-        "@carimus/node-disks": "^1.2.0"
+        "@carimus/node-disks": "^1.8.0",
+        "tmp": "^0.1.0"
     }
 }

src/lib/Uploads.test.ts

@@ -1,8 +1,13 @@
+import * as fs from 'fs';
+import { promisify } from 'util';
 import { DiskDriver, DiskManager } from '@carimus/node-disks';
 import { MemoryRepository } from '../support';
 import { Uploads } from './Uploads';
 import { UploadMeta } from '../types';
 
+const readFileFromLocalFilesystem = promisify(fs.readFile);
+const deleteFromLocalFilesystem = promisify(fs.unlink);
+
 const disks = {
     default: 'memory',
     memory: {
@@ -46,7 +51,18 @@ const files: {
     weirdName: {
         uploadedAs: '.~my~cool~data~&^%$*(¶•ª•.csv',
         data: Buffer.from('a,b,c\nfoo,bar,baz\n1,2,3\n', 'utf8'),
-        meta: { context: 'test', isFoo: false, isImage: true },
+        meta: { context: 'test', isFoo: false, isImage: false },
+    },
+    longName: {
+        uploadedAs:
+            '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~.csv',
+        data: Buffer.from('a,b,c\nfoo,bar,baz\n1,2,3\n', 'utf8'),
+        meta: {
+            context: 'test',
+            isFoo: false,
+            isImage: false,
+            isSuperLong: true,
+        },
     },
 };
 
@@ -258,3 +274,44 @@ test('Uploads service can delete only the file', async () => {
         diskManager.getDisk(fileInfo.disk).read(fileInfo.path),
     ).rejects.toBeTruthy();
 });
+
+test('Uploads service can create temp files for local manipulation from uploads', async () => {
+    const { diskManager, repository, uploads } = setup();
+
+    // Upload a file
+    const upload = await uploads.upload(
+        files.longName.data,
+        files.longName.uploadedAs,
+        files.longName.meta,
+    );
+    const fileInfo = await repository.getUploadedFileInfo(upload);
+    const uploadedFileData = await diskManager
+        .getDisk(fileInfo.disk)
+        .read(fileInfo.path);
+
+    // Get the temp file for it and check to make sure their contents match
+    const tempPath = await uploads.withTempFile(upload, async (path) => {
+        const tempFileData = await readFileFromLocalFilesystem(path);
+        expect(tempFileData.toString('base64')).toBe(
+            uploadedFileData.toString('base64'),
+        );
+    });
+
+    // Ensure that once the callback is completed, the file doesn't exist since we didn't tell it not to cleanup
+    expect(tempPath).toBeTruthy();
+    await expect(readFileFromLocalFilesystem(tempPath)).rejects.toBeTruthy();
+
+    // Do the same stuff again but using the bypass cleanup approach to take cleanup into our own hands
+    const persistentTempPath = await uploads.withTempFile(upload);
+    expect(persistentTempPath).toBeTruthy();
+    const persistentTempFileData = await readFileFromLocalFilesystem(
+        persistentTempPath,
+    );
+    expect(persistentTempFileData.toString('base64')).toBe(
+        uploadedFileData.toString('base64'),
+    );
+    // Note that we use `.resolves.toBeUndefined()` to verify the file is deleted (unlink resolves with void/undefined)
+    expect(
+        deleteFromLocalFilesystem(persistentTempPath),
+    ).resolves.toBeUndefined();
+});

src/lib/Uploads.ts

@@ -1,5 +1,6 @@
 import { Readable } from 'stream';
-import { DiskManager } from '@carimus/node-disks';
+import * as fs from 'fs';
+import { DiskManager, pipeStreams } from '@carimus/node-disks';
 import { InvalidConfigError, PathNotUniqueError } from '../errors';
 import {
     Upload,
@@ -8,7 +9,7 @@ import {
     UploadRepository,
     UploadsConfig,
 } from '../types';
-import { trimPath } from './utils';
+import { trimPath, withTempFile } from './utils';
 import { defaultSanitizeFilename, defaultGeneratePath } from './defaults';
 
 /**
@@ -17,7 +18,6 @@ import { defaultSanitizeFilename, defaultGeneratePath } from './defaults';
  * TODO Do URL generation for publicly available disks.
  * TODO Support temporary URLs (e.g. presigned URLs for S3 buckets) for disks that support it
  * TODO Support transfer logic for transferring single uploads from one disk to another and in bulk.
- * TODO Support `getTemporaryFile` to copy an upload file to the local filesystem tmp directory for direct manipulation
  */
 export class Uploads {
     private config: UploadsConfig;
@@ -248,4 +248,53 @@ export class Uploads {
         // Delete the file on the disk
         await disk.delete(file.path);
     }
+
+    /**
+     * Download the file to the local disk as a temporary file for operations that require local data manipuation
+     * and which can't handle Buffers, i.e. operations expected to be performed on large files where it's easier to
+     * deal with the data in chunks off of the disk or something instead of keeping them in a Buffer in memory in their
+     * entirety.
+     *
+     * This methods streams the data directly to the local filesystem so large files shouldn't cause any memory issues.
+     *
+     * If an `execute` callback is not provided, the cleanup step will be skipped and the path that this resolves to
+     * will exist and can be manipulated directly. IMPORTANT: in such a scenario, the caller is responsible for
+     * deleting the file when they're done with it.
+     *
+     * @param upload
+     * @param execute
+     */
+    public async withTempFile(
+        upload: Upload,
+        execute: ((path: string) => Promise<void> | void) | null = null,
+    ): Promise<string> {
+        // Ask the repository for info on where and how the upload file is stored.
+        const uploadedFile = await this.repository.getUploadedFileInfo(upload);
+        // Resolve the disk for the file.
+        const disk = this.disks.getDisk(uploadedFile.disk);
+        // Generate a descriptive postfix for the temp file that isn't too long.
+        const postfix = `-${uploadedFile.uploadedAs}`.slice(-50);
+        // Create a temp file, write the upload's file data to it, and pass its path to
+        return withTempFile(
+            async (path: string) => {
+                // Create a write stream to the temp file that will auto close once the stream is fully piped.
+                const tempFileWriteStream = fs.createWriteStream(path, {
+                    autoClose: true,
+                });
+                // Create a read stream for the file on the disk.
+                const diskFileReadStream = await disk.createReadStream(
+                    uploadedFile.path,
+                );
+                // Pipe the disk read stream to the temp file write stream.
+                await pipeStreams(diskFileReadStream, tempFileWriteStream);
+                // Run the caller callback if it was provided.
+                if (execute) {
+                    await execute(path);
+                }
+            },
+            // Skip clean up if no execute callback is provided.
+            !execute,
+            { postfix },
+        );
+    }
 }

src/lib/utils.ts

@@ -1,4 +1,4 @@
-import { Readable, Stream, Writable } from 'stream';
+import tmp = require('tmp');
 
 /**
  * Remove all whitespace and slashes from the beginning and end of a string.
@@ -8,3 +8,44 @@ import { Readable, Stream, Writable } from 'stream';
 export function trimPath(path: string): string {
     return `${path}`.replace(/(^(\s|\/)+|(\s|\/)+$)/g, '');
 }
+
+/**
+ * Create a temp file and do something with it.
+ *
+ * @param execute An optionally async function that will receive the temp file's name (path)
+ * @param skipCleanup If true, don't delete the file until process end.
+ * @param extraOptions Additional options to pass into `tmp.file`
+ * @return The temporary's file path which won't exist after this resolves unless `skipCleanup` was `true`
+ */
+export async function withTempFile(
+    execute: (name: string) => Promise<void> | void,
+    skipCleanup: boolean = false,
+    extraOptions: import('tmp').FileOptions = {},
+): Promise<string> {
+    // Receive the temp file's name (path) and cleanup function from `tmp`, throwing if it rejects.
+    const {
+        name,
+        cleanupCallback,
+    }: { name: string; cleanupCallback: () => void } = await new Promise(
+        (resolve, reject) => {
+            tmp.file(
+                { discardDescriptor: true, ...extraOptions },
+                (err, name, fd, cleanupCallback) => {
+                    if (err) {
+                        reject(err);
+                    } else {
+                        resolve({ name, cleanupCallback });
+                    }
+                },
+            );
+        },
+    );
+    // Run the execute callback with the name (path)
+    await execute(name);
+    // Don't delete the file if requested.
+    if (!skipCleanup) {
+        await cleanupCallback();
+    }
+    // Return the temporary file's name (path)
+    return name;
+}

yarn.lock

@@ -138,10 +138,10 @@
     lodash "^4.17.11"
     to-fast-properties "^2.0.0"
 
-"@carimus/node-disks@^1.2.0":
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/@carimus/node-disks/-/node-disks-1.7.0.tgz#fcf9c1f9ca275935fc84a4286766266bec00a6f7"
-  integrity sha512-S7eBw5G7ykToLMO3FINDSrI/wbJniau/AGvkWgNWiq0c5nz+QaSqAQ9YB+ZxTpRoeYtCBypNXC7OjREd4uQ4TQ==
+"@carimus/node-disks@^1.8.0":
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/@carimus/node-disks/-/node-disks-1.8.0.tgz#d5d12300dfaf244889570f5309e354b8df18ef3b"
+  integrity sha512-zdi7euqzn76Z2hyMBN5hUsr9beSDRB8fCktRz4xuUTXNAqmL15kCKruSV3WHB7tz4nUDn+Hr2yCYM3U+S3jgyw==
   dependencies:
     aws-sdk "^2.431.0"
     fs-extra "^7.0.1"
@@ -644,6 +644,11 @@
   resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-1.0.1.tgz#0a851d3bd96498fa25c33ab7278ed3bd65f06c3e"
   integrity sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw==
 
+"@types/tmp@^0.1.0":
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/@types/tmp/-/tmp-0.1.0.tgz#19cf73a7bcf641965485119726397a096f0049bd"
+  integrity sha512-6IwZ9HzWbCq6XoQWhxLpDjuADodH/MKXRUIDFudvgjcVdjFknvmR+DNsoUeer4XPrEnrZs04Jj+kfV9pFsrhmA==
+
 "@types/yargs@^12.0.2", "@types/yargs@^12.0.9":
   version "12.0.10"
   resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-12.0.10.tgz#17a8ec65cd8e88f51b418ceb271af18d3137df67"
@@ -6601,7 +6606,7 @@ right-pad@^1.0.1:
   resolved "https://registry.yarnpkg.com/right-pad/-/right-pad-1.0.1.tgz#8ca08c2cbb5b55e74dafa96bf7fd1a27d568c8d0"
   integrity sha1-jKCMLLtbVedNr6lr9/0aJ9VoyNA=
 
-rimraf@2, rimraf@2.6.3, rimraf@^2.2.8, rimraf@^2.5.2, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@~2.6.2:
+rimraf@2, rimraf@2.6.3, rimraf@^2.2.8, rimraf@^2.5.2, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3, rimraf@~2.6.2:
   version "2.6.3"
   resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab"
   integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==
@@ -7387,6 +7392,13 @@ tmp@^0.0.33:
   dependencies:
     os-tmpdir "~1.0.2"
 
+tmp@^0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.1.0.tgz#ee434a4e22543082e294ba6201dcc6eafefa2877"
+  integrity sha512-J7Z2K08jbGcdA1kkQpJSqLF6T0tdQqpR2pnSUXsIchbPdTI9v3e85cLW0d6WDhwuAleOV71j2xWs8qMPfK7nKw==
+  dependencies:
+    rimraf "^2.6.3"
+
 tmpl@1.0.x:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1"