feat: Provide file mode to handle multipart request (#19)

see RFC to get detail: eggjs/egg#3052 closes eggjs/egg#3052
eggjs · Sep 29, 2018 · 75c0733 · 75c0733
1 parent 0b4e118
commit 75c0733
Show file tree

Hide file tree

Showing 22 changed files with 873 additions and 70 deletions.
diff --git a/README.md b/README.md
@@ -20,7 +20,8 @@
 [download-image]: https://img.shields.io/npm/dm/egg-multipart.svg?style=flat-square
 [download-url]: https://npmjs.org/package/egg-multipart
 
-Use [co-busboy](https://github.com/cojs/busboy) to upload file by streaming and process it without save to disk.
+Use [co-busboy](https://github.com/cojs/busboy) to upload file by streaming and
+process it without save to disk(using the `stream` mode).
 
 Just use `ctx.multipart()` to got file stream, then pass to image processing liberary such as `gm` or upload to cloud storage such as `oss`.
 
@@ -107,9 +108,135 @@ exports.multipart = {
 
 ## Examples
 
-[More Examples](https://github.com/eggjs/examples/tree/master/multipart)
+More examples please follow:
 
-### Upload File
+- [Handle multipart request in `stream` mode](https://github.com/eggjs/examples/tree/master/multipart)
+- [Handle multipart request in `file` mode](https://github.com/eggjs/examples/tree/master/multipart-file-mode)
+
+## `file` mode: the easy way
+
+If you don't know the [Node.js Stream](https://nodejs.org/dist/latest-v10.x/docs/api/stream.html) work, maybe you should use the `file` mode to get started.
+
+The usage very similar to [bodyParser](https://eggjs.org/en/basics/controller.html#body).
+
+- `ctx.request.body`: Get all the multipart fields and values, except `file`.
+- `ctx.request.files`: Contains all `file` from the multipart request, it's an Array object.
+
+**WARNING: you should remove the temporary upload file after you use it**
+
+### Enable `file` mode on config
+
+You need to set `config.multipart.mode = 'file'` to enable `file` mode:
+
+```js
+// config/config.default.js
+exports.multipart = {
+  mode: 'file',
+};
+```
+
+After `file` mode enable, egg will remove the old temporary files(don't include today's files) on `04:30 AM` every day by default.
+
+```js
+config.multipart = {
+  mode: 'file',
+  tmpdir: path.join(os.tmpdir(), 'egg-multipart-tmp', appInfo.name),
+  cleanSchedule: {
+    // run tmpdir clean job on every day 04:30 am
+    // cron style see https://github.com/eggjs/egg-schedule#cron-style-scheduling
+    cron: '0 30 4 * * *',
+  },
+};
+```
+
+### Upload One File
+
+```html
+<form method="POST" action="/upload?_csrf={{ ctx.csrf | safe }}" enctype="multipart/form-data">
+  title: <input name="title" />
+  file: <input name="file" type="file" />
+  <button type="submit">Upload</button>
+</form>
+```
+
+Controller which hanlder `POST /upload`:
+
+```js
+// app/controller/upload.js
+const Controller = require('egg').Controller;
+const fs = require('mz/fs');
+
+module.exports = class extends Controller {
+  async upload() {
+    const { ctx } = this;
+    const file = ctx.request.files[0];
+    const name = 'egg-multipart-test/' + path.basename(file.filename);
+    let result;
+    try {
+      // process file or upload to cloud storage
+      result = await ctx.oss.put(name, file.filepath);
+    } finally {
+      // need to remove the tmp file
+      await fs.unlink(file.filepath);
+    }
+
+    ctx.body = {
+      url: result.url,
+      // get all field values
+      requestBody: ctx.request.body,
+    };
+  }
+};
+```
+
+### Upload Multiple Files
+
+```html
+<form method="POST" action="/upload?_csrf={{ ctx.csrf | safe }}" enctype="multipart/form-data">
+  title: <input name="title" />
+  file1: <input name="file1" type="file" />
+  file2: <input name="file2" type="file" />
+  <button type="submit">Upload</button>
+</form>
+```
+
+Controller which hanlder `POST /upload`:
+
+```js
+// app/controller/upload.js
+const Controller = require('egg').Controller;
+const fs = require('mz/fs');
+
+module.exports = class extends Controller {
+  async upload() {
+    const { ctx } = this;
+    console.log(ctx.request.body);
+    console.log('got %d files', ctx.request.files.length);
+    for (const file of ctx.request.files) {
+      console.log('field: ' + file.fieldname);
+      console.log('filename: ' + file.filename);
+      console.log('encoding: ' + file.encoding);
+      console.log('mime: ' + file.mime);
+      console.log('tmp filepath: ' + file.filepath);
+      let result;
+      try {
+        // process file or upload to cloud storage
+        result = await ctx.oss.put('egg-multipart-test/' + file.filename, file.filepath);
+      } finally {
+        // need to remove the tmp file
+        await fs.unlink(file.filepath);
+      }
+      console.log(result);
+    }
+  }
+};
+```
+
+## `stream` mode: the hard way
+
+If you're well-known about know the Node.js Stream work, you should use the `stream` mode.
+
+### Upload One File
 
 You can got upload stream by `ctx.getFileStream*()`.
 
@@ -129,9 +256,9 @@ const path = require('path');
 const sendToWormhole = require('stream-wormhole');
 const Controller = require('egg').Controller;
 
-module.exports = Class UploadController extends Controller {
+module.exports = class extends Controller {
   async upload() {
-    const ctx = this.ctx;
+    const { ctx } = this;
     // file not exists will response 400 error
     const stream = await ctx.getFileStream();
     const name = 'egg-multipart-test/' + path.basename(stream.filename);
@@ -146,7 +273,7 @@ module.exports = Class UploadController extends Controller {
   }
 
   async uploadNotRequiredFile() {
-    const ctx = this.ctx;
+    const { ctx } = this;
     // file not required
     const stream = await ctx.getFileStream({ requireFile: false });
     let result;
@@ -173,7 +300,8 @@ module.exports = Class UploadController extends Controller {
 ```html
 <form method="POST" action="/upload?_csrf={{ ctx.csrf | safe }}" enctype="multipart/form-data">
   title: <input name="title" />
-  file: <input name="file" type="file" />
+  file1: <input name="file1" type="file" />
+  file2: <input name="file2" type="file" />
   <button type="submit">Upload</button>
 </form>
 ```
@@ -184,9 +312,9 @@ Controller which hanlder `POST /upload`:
 // app/controller/upload.js
 const Controller = require('egg').Controller;
 
-module.exports = Class UploadController extends Controller {
+module.exports = class extends Controller {
   async upload() {
-    const ctx = this.ctx;
+    const { ctx } = this;
     const parts = ctx.multipart();
     let part;
     while ((part = await parts()) != null) {
@@ -201,7 +329,7 @@ module.exports = Class UploadController extends Controller {
           // user click `upload` before choose a file,
           // `part` will be file stream, but `part.filename` is empty
           // must handler this, such as log error.
-          return;
+          continue;
         }
         // otherwise, it's a stream
         console.log('field: ' + part.fieldname);
@@ -216,3 +344,7 @@ module.exports = Class UploadController extends Controller {
   }
 };
 ```
+
+## License
+
+[MIT](LICENSE)
diff --git a/app.js b/app.js
@@ -79,4 +79,17 @@ module.exports = app => {
       }
     },
   };
+
+  options.mode = options.mode || 'stream';
+  if (![ 'stream', 'file' ].includes(options.mode)) {
+    throw new TypeError(`Expect mode to be 'stream' or 'file', but got '${options.mode}'`);
+  }
+
+  app.coreLogger.info('[egg-multipart] %s mode enable', options.mode);
+  if (options.mode === 'file') {
+    app.coreLogger.info('[egg-multipart] will save temporary files to %j, cleanup job cron: %j',
+      options.tmpdir, options.cleanSchedule.cron);
+    // enable multipart middleware
+    app.config.coreMiddleware.push('multipart');
+  }
 };
diff --git a/app/extend/context.js b/app/extend/context.js
@@ -9,6 +9,8 @@ class EmptyStream extends Readable {
   }
 }
 
+const HAS_CONSUMED = Symbol('Context#multipartHasConsumed');
+
 module.exports = {
   /**
    * create multipart.parts instance, to get separated files.
@@ -21,6 +23,9 @@ module.exports = {
     if (!this.is('multipart')) {
       this.throw(400, 'Content-Type must be multipart/*');
     }
+    if (this[HAS_CONSUMED]) throw new TypeError('the multipart request can\'t be consumed twice');
+
+    this[HAS_CONSUMED] = true;
     const parseOptions = {};
     Object.assign(parseOptions, this.app.config.multipartParseOptions, options);
     return parse(this, parseOptions);

diff --git a/app/middleware/multipart.js b/app/middleware/multipart.js
@@ -0,0 +1,108 @@
+'use strict';
+
+const path = require('path');
+const fs = require('mz/fs');
+const uuid = require('uuid');
+const mkdirp = require('mz-modules/mkdirp');
+const pump = require('mz-modules/pump');
+const sendToWormhole = require('stream-wormhole');
+const moment = require('moment');
+
+module.exports = options => {
+  async function cleanup(requestFiles) {
+    for (const file of requestFiles) {
+      try {
+        await fs.unlink(file.filepath);
+      } catch (_) {
+        // do nothing
+      }
+    }
+  }
+
+  async function limit(requestFiles, code, message) {
+    // cleanup requestFiles
+    await cleanup(requestFiles);
+
+    // throw 413 error
+    const err = new Error(message);
+    err.code = code;
+    err.status = 413;
+    throw err;
+  }
+
+  return async function multipart(ctx, next) {
+    if (!ctx.is('multipart')) return next();
+
+    let storedir;
+
+    const requestBody = {};
+    const requestFiles = [];
+
+    const parts = ctx.multipart({ autoFields: false });
+    let part;
+    do {
+      try {
+        part = await parts();
+      } catch (err) {
+        await cleanup(requestFiles);
+        throw err;
+      }
+
+      if (!part) break;
+
+      if (part.length) {
+        ctx.coreLogger.debug('[egg-multipart:storeMultipart] handle value part: %j', part);
+        const fieldnameTruncated = part[2];
+        const valueTruncated = part[3];
+        if (valueTruncated) {
+          return await limit(requestFiles, 'Request_fieldSize_limit', 'Reach fieldSize limit');
+        }
+        if (fieldnameTruncated) {
+          return await limit(requestFiles, 'Request_fieldNameSize_limit', 'Reach fieldNameSize limit');
+        }
+
+        // arrays are busboy fields
+        requestBody[part[0]] = part[1];
+        continue;
+      }
+
+      // otherwise, it's a stream
+      const meta = {
+        field: part.fieldname,
+        filename: part.filename,
+        encoding: part.encoding,
+        mime: part.mime,
+      };
+      ctx.coreLogger.debug('[egg-multipart:storeMultipart] handle stream part: %j', meta);
+      // empty part, ignore it
+      if (!part.filename) {
+        await sendToWormhole(part);
+        continue;
+      }
+
+      if (!storedir) {
+        // ${tmpdir}/YYYY/MM/DD/HH
+        storedir = path.join(options.tmpdir, moment().format('YYYY/MM/DD/HH'));
+        const exists = await fs.exists(storedir);
+        if (!exists) {
+          await mkdirp(storedir);
+        }
+      }
+      const filepath = path.join(storedir, uuid.v4() + path.extname(meta.filename));
+      const target = fs.createWriteStream(filepath);
+      await pump(part, target);
+      // https://github.com/mscdex/busboy/blob/master/lib/types/multipart.js#L221
+      meta.filepath = filepath;
+      requestFiles.push(meta);
+
+      // https://github.com/mscdex/busboy/blob/master/lib/types/multipart.js#L221
+      if (part.truncated) {
+        return await limit(requestFiles, 'Request_fileSize_limit', 'Reach fileSize limit');
+      }
+    } while (part != null);
+
+    ctx.request.body = requestBody;
+    ctx.request.files = requestFiles;
+    return next();
+  };
+};