feat: add option to handle file streams (#666)

* feat: add option to store files or not
* feat: add new option fileWriteStreamHandler

Author: leonardovillela

README.md

@@ -64,7 +64,8 @@ _**Note: v2 is coming soon!**_
 ## Highlights
 
 - [Fast (~900-2500 mb/sec)](#benchmarks) & streaming multipart parser
-- Automatically writing file uploads to disk (soon optionally)
+- Automatically writing file uploads to disk (optional, see
+  [`options.fileWriteStreamHandler`](#options))
 - [Plugins API](#useplugin-plugin) - allowing custom parsers and plugins
 - Low memory footprint
 - Graceful error handling
@@ -310,8 +311,8 @@ const form = new Formidable(options);
 
 ### Options
 
-See it's defaults in [src/Formidable.js DEFAULT_OPTIONS](./src/Formidable.js) (the
-`DEFAULT_OPTIONS` constant).
+See it's defaults in [src/Formidable.js DEFAULT_OPTIONS](./src/Formidable.js)
+(the `DEFAULT_OPTIONS` constant).
 
 - `options.encoding` **{string}** - default `'utf-8'`; sets encoding for
   incoming form fields,
@@ -334,6 +335,16 @@ See it's defaults in [src/Formidable.js DEFAULT_OPTIONS](./src/Formidable.js) (t
   for incoming files, set this to some hash algorithm, see
   [crypto.createHash](https://nodejs.org/api/crypto.html#crypto_crypto_createhash_algorithm_options)
   for available algorithms
+- `options.fileWriteStreamHandler` **{function}** - default `null`, which by
+  default writes to host machine file system every file parsed; The function
+  should return an instance of a
+  [Writable stream](https://nodejs.org/api/stream.html#stream_class_stream_writable)
+  that will receive the uploaded file data. With this option, you can have any
+  custom behavior regarding where the uploaded file data will be streamed for.
+  If you are looking to write the file uploaded in other types of cloud storages
+  (AWS S3, Azure blob storage, Google cloud storage) or private file storage,
+  this is the option you're looking for. When this option is defined the default
+  behavior of writing the file in the host machine file system is lost.
 - `options.multiples` **{boolean}** - default `false`; when you call the
   `.parse` method, the `files` argument (of the callback) will contain arrays of
   files for inputs which submit multiple files using the HTML5 `multiple`
@@ -636,8 +647,8 @@ form.on('end', () => {});
 
 If the documentation is unclear or has a typo, please click on the page's `Edit`
 button (pencil icon) and suggest a correction. If you would like to help us fix
-a bug or add a new feature, please check our
-[Contributing Guide][contributing-url]. Pull requests are welcome!
+a bug or add a new feature, please check our [Contributing
+Guide][contributing-url]. Pull requests are welcome!
 
 Thanks goes to these wonderful people
 ([emoji key](https://allcontributors.org/docs/en/emoji-key)):

examples/log-file-content-to-console.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const http = require('http');
+const { Writable } = require('stream');
+const formidable = require('../src/index');
+
+const server = http.createServer((req, res) => {
+  if (req.url === '/api/upload' && req.method.toLowerCase() === 'post') {
+    // parse a file upload
+    const form = formidable({
+      fileWriteStreamHandler: () => {
+        const writable = Writable();
+        // eslint-disable-next-line no-underscore-dangle
+        writable._write = (chunk, enc, next) => {
+          console.log(chunk.toString());
+          next();
+        };
+        return writable;
+      },
+    });
+
+    form.parse(req, () => {
+      res.writeHead(200);
+      res.end();
+    });
+
+    return;
+  }
+
+  // show a file upload form
+  res.writeHead(200, { 'Content-Type': 'text/html' });
+  res.end(`
+    <h2>With Node.js <code>"http"</code> module</h2>
+    <form action="/api/upload" enctype="multipart/form-data" method="post">
+      <div>Text field title: <input type="text" name="title" /></div>
+      <div>File: <input type="file" name="file" /></div>
+      <input type="submit" value="Upload" />
+    </form>
+  `);
+});
+
+server.listen(3000, () => {
+  console.log('Server listening on http://localhost:3000 ...');
+});

examples/store-files-on-s3.js

@@ -0,0 +1,63 @@
+// To test this example you have to install aws-sdk nodejs package and create a bucket named "demo-bucket"
+
+'use strict';
+
+const http = require('http');
+const { PassThrough } = require('stream');
+// eslint-disable-next-line import/no-unresolved
+const AWS = require('aws-sdk');
+const formidable = require('../src/index');
+
+const s3Client = new AWS.S3({
+  credentials: {
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+    secretAccessKey: process.env.AWS_SECRET_KEY,
+  },
+});
+
+const uploadStream = (filename) => {
+  const pass = PassThrough();
+  s3Client.upload(
+    {
+      Bucket: 'demo-bucket',
+      Key: filename,
+      Body: pass,
+    },
+    (err, data) => {
+      console.log(err, data);
+    },
+  );
+
+  return pass;
+};
+
+const server = http.createServer((req, res) => {
+  if (req.url === '/api/upload' && req.method.toLowerCase() === 'post') {
+    // parse a file upload
+    const form = formidable({
+      fileWriteStreamHandler: uploadStream,
+    });
+
+    form.parse(req, () => {
+      res.writeHead(200);
+      res.end();
+    });
+
+    return;
+  }
+
+  // show a file upload form
+  res.writeHead(200, { 'Content-Type': 'text/html' });
+  res.end(`
+    <h2>With Node.js <code>"http"</code> module</h2>
+    <form action="/api/upload" enctype="multipart/form-data" method="post">
+      <div>Text field title: <input type="text" name="title" /></div>
+      <div>File: <input type="file" name="file"/></div>
+      <input type="submit" value="Upload" />
+    </form>
+  `);
+});
+
+server.listen(3000, () => {
+  console.log('Server listening on http://localhost:3000 ...');
+});

src/Formidable.js

@@ -26,9 +26,11 @@ const DEFAULT_OPTIONS = {
   uploadDir: os.tmpdir(),
   multiples: false,
   enabledPlugins: ['octetstream', 'querystring', 'multipart', 'json'],
+  fileWriteStreamHandler: null,
 };
 
-const File = require('./File');
+const PersistentFile = require('./PersistentFile');
+const VolatileFile = require('./VolatileFile');
 const DummyParser = require('./parsers/Dummy');
 const MultipartParser = require('./parsers/Multipart');
 
@@ -138,11 +140,11 @@ class IncomingForm extends EventEmitter {
       const fields = {};
       let mockFields = '';
       const files = {};
-      
+
       this.on('field', (name, value) => {
         if (this.options.multiples) {
-          let mObj = { [name] : value };
-          mockFields = mockFields + '&' + qs.stringify(mObj);
+          const mObj = { [name]: value };
+          mockFields = `${mockFields}&${qs.stringify(mObj)}`;
         } else {
           fields[name] = value;
         }
@@ -295,11 +297,10 @@ class IncomingForm extends EventEmitter {
 
     this._flushing += 1;
 
-    const file = new File({
+    const file = this._newFile({
       path: this._rename(part),
-      name: part.filename,
-      type: part.mime,
-      hash: this.options.hash,
+      filename: part.filename,
+      mime: part.mime,
     });
     file.on('error', (err) => {
       this._error(err);
@@ -420,7 +421,7 @@ class IncomingForm extends EventEmitter {
 
     if (Array.isArray(this.openedFiles)) {
       this.openedFiles.forEach((file) => {
-        file._writeStream.destroy();
+        file.destroy();
         setTimeout(fs.unlink, 0, file.path, () => {});
       });
     }
@@ -443,6 +444,22 @@ class IncomingForm extends EventEmitter {
     return new MultipartParser(this.options);
   }
 
+  _newFile({ path: filePath, filename: name, mime: type }) {
+    return this.options.fileWriteStreamHandler
+      ? new VolatileFile({
+          name,
+          type,
+          createFileWriteStream: this.options.fileWriteStreamHandler,
+          hash: this.options.hash,
+        })
+      : new PersistentFile({
+          path: filePath,
+          name,
+          type,
+          hash: this.options.hash,
+        });
+  }
+
   _getFileName(headerValue) {
     // matches either a quoted-string or a token (RFC 2616 section 19.5.1)
     const m = headerValue.match(

src/File.js renamed to src/PersistentFile.js

@@ -6,7 +6,7 @@ const fs = require('fs');
 const crypto = require('crypto');
 const { EventEmitter } = require('events');
 
-class File extends EventEmitter {
+class PersistentFile extends EventEmitter {
   constructor(properties) {
     super();
 
@@ -56,7 +56,7 @@ class File extends EventEmitter {
   }
 
   toString() {
-    return `File: ${this.name}, Path: ${this.path}`;
+    return `PersistentFile: ${this.name}, Path: ${this.path}`;
   }
 
   write(buffer, cb) {
@@ -86,6 +86,10 @@ class File extends EventEmitter {
       cb();
     });
   }
+
+  destroy() {
+    this._writeStream.destroy();
+  }
 }
 
-module.exports = File;
+module.exports = PersistentFile;

src/VolatileFile.js

@@ -0,0 +1,89 @@
+/* eslint-disable no-underscore-dangle */
+
+'use strict';
+
+const crypto = require('crypto');
+const { EventEmitter } = require('events');
+
+class VolatileFile extends EventEmitter {
+  constructor(properties) {
+    super();
+
+    this.size = 0;
+    this.name = null;
+    this.type = null;
+    this.hash = null;
+
+    this._writeStream = null;
+
+    // eslint-disable-next-line guard-for-in, no-restricted-syntax
+    for (const key in properties) {
+      this[key] = properties[key];
+    }
+
+    if (typeof this.hash === 'string') {
+      this.hash = crypto.createHash(properties.hash);
+    } else {
+      this.hash = null;
+    }
+  }
+
+  open() {
+    this._writeStream = this.createFileWriteStream(this.name);
+    this._writeStream.on('error', (err) => {
+      this.emit('error', err);
+    });
+  }
+
+  destroy() {
+    this._writeStream.destroy();
+  }
+
+  toJSON() {
+    const json = {
+      size: this.size,
+      name: this.name,
+      type: this.type,
+      length: this.length,
+      filename: this.filename,
+      mime: this.mime,
+    };
+    if (this.hash && this.hash !== '') {
+      json.hash = this.hash;
+    }
+    return json;
+  }
+
+  toString() {
+    return `VolatileFile: ${this.name}`;
+  }
+
+  write(buffer, cb) {
+    if (this.hash) {
+      this.hash.update(buffer);
+    }
+
+    if (this._writeStream.closed || this._writeStream.destroyed) {
+      cb();
+      return;
+    }
+
+    this._writeStream.write(buffer, () => {
+      this.size += buffer.length;
+      this.emit('progress', this.size);
+      cb();
+    });
+  }
+
+  end(cb) {
+    if (this.hash) {
+      this.hash = this.hash.digest('hex');
+    }
+    this._writeStream.end(() => {
+      this.emit('end');
+      cb();
+    });
+  }
+}
+
+module.exports = VolatileFile;

src/index.js

@@ -1,6 +1,7 @@
 'use strict';
 
-const File = require('./File');
+const PersistentFile = require('./PersistentFile');
+const VolatileFile = require('./VolatileFile');
 const Formidable = require('./Formidable');
 
 const plugins = require('./plugins/index');
@@ -11,7 +12,9 @@ const parsers = require('./parsers/index');
 const formidable = (...args) => new Formidable(...args);
 
 module.exports = Object.assign(formidable, {
-  File,
+  File: PersistentFile,
+  PersistentFile,
+  VolatileFile,
   Formidable,
   formidable,
 

src/plugins/octetstream.js

@@ -2,7 +2,6 @@
 
 'use strict';
 
-const File = require('../File');
 const OctetStreamParser = require('../parsers/OctetStream');
 
 // the `options` is also available through the `options` / `formidable.options`
@@ -28,11 +27,10 @@ function init(_self, _opts) {
   const filename = this.headers['x-file-name'];
   const mime = this.headers['content-type'];
 
-  const file = new File({
+  const file = this._newFile({
     path: this._uploadPath(filename),
-    name: filename,
-    type: mime,
-    hash: this.options.hash,
+    filename,
+    mime,
   });
 
   this.emit('fileBegin', filename, file);