fix: upload support auto clean and whitelist (#1484)

doc/zh-cn/component/upload.md

@@ -65,15 +65,18 @@ export class HomeController {
 ## 配置
 ```ts
 // src/config/config.default.ts
+import { uploadWhiteList } from '@midwayjs/upload';
 export const upload = {
   // mode: UploadMode, 默认为file，即上传到服务器临时目录，可以配置为 stream
   mode: 'file',
   // fileSize: string, 最大上传文件大小，默认为 10mb
   fileSize: '10mb',
   // whitelist: string[]，文件扩展名白名单
-  whitelist: null,
+  whitelist: uploadWhiteList.filter(ext => ext !== '.pdf'),
   // tmpdir: string，上传的文件临时存储路径
   tmpdir: join(tmpdir(), 'midway-upload-files'),
+  // cleanTimeout: number，上传的文件在临时目录中多久之后自动删除，默认为 5 分钟
+  cleanTimeout: 5 * 60 * 1000,
 }
 ```
 
@@ -135,6 +138,17 @@ export const upload = {
 '.avi',
 ```
 
+可以通过 `@midwayjs/upload` 包中导出的 `uploadWhiteList` 获取到默认的后缀名白名单。
+
+### 临时文件与清理
+
+
+如果你使用了 `file` 模式来获取上传的文件，那么上传的文件会存放在您于 `config` 文件中设置的 `upload` 组件配置中的 `tmpdir` 选项指向的文件夹内。
+
+你可以通过在配置中使用 `cleanTimeout` 来控制自动的临时文件清理时间，默认值为 `5 * 60 * 1000`，即上传的文件于 `5 分钟` 后自动清理，设置为 `0` 则视为不开启自动清理功能。
+
+你也可以在代码中通过调用 `await ctx.cleanupRequestFiles()` 来主动清理当前请求上传的临时文件。
+
 ## 前端如何将文件上传到服务器？
 
 ### 1. html form 的形式

packages/upload/README.md

@@ -68,5 +68,7 @@ export const upload = {
   whitelist: null,
   // tmpdir: string，上传的文件临时存储路径
   tmpdir: join(tmpdir(), 'midway-upload-files'),
+  // cleanTimeout: number，上传的文件在临时目录中多久之后自动删除，默认为 5 分钟
+  cleanTimeout: 5 * 60 * 1000,
 }
 ```

packages/upload/index.d.ts

@@ -1,8 +1,39 @@
-import { UploadOptions } from './dist/index';
+import { UploadFileInfo, UploadOptions } from './dist/index';
 export * from './dist/index';
 
 declare module '@midwayjs/core/dist/interface' {
   interface MidwayConfig {
     upload: Partial<UploadOptions>;
   }
 }
+declare module '@midwayjs/koa/dist/interface' {
+  interface Context {
+    files?: UploadFileInfo<any>[];
+    fields?: { [fieldName: string]: any };
+    cleanupRequestFiles?: () => Promise<Array<boolean>>;
+  }
+}
+
+declare module '@midwayjs/web/dist/interface' {
+  interface Context {
+    files?: UploadFileInfo<any>[];
+    fields?: { [fieldName: string]: any };
+    cleanupRequestFiles?: () => Promise<Array<boolean>>;
+  }
+}
+
+declare module '@midwayjs/faas/dist/interface' {
+  interface Context {
+    files?: UploadFileInfo<any>[];
+    fields?: { [fieldName: string]: any };
+    cleanupRequestFiles?: () => Promise<Array<boolean>>;
+  }
+}
+
+declare module '@midwayjs/express/dist/interface' {
+  interface Context {
+    files?: UploadFileInfo<any>[];
+    fields?: { [fieldName: string]: any };
+    cleanupRequestFiles?: () => Promise<Array<boolean>>;
+  }
+}

packages/upload/package.json

@@ -21,7 +21,6 @@
   },
   "license": "MIT",
   "dependencies": {
-    "fs-extra": "^8.0.1",
     "raw-body": "^2.4.1"
   },
   "devDependencies": {

packages/upload/src/config/config.default.ts

@@ -1,41 +1,12 @@
 import { UploadOptions } from '../interface';
 import { join } from 'path';
 import { tmpdir } from 'os';
+import { uploadWhiteList } from '../constants';
 
 export const upload: UploadOptions = {
   mode: 'file',
   fileSize: '10mb',
-  whitelist: [
-    // images
-    '.jpg',
-    '.jpeg', // image/jpeg
-    '.png', // image/png, image/x-png
-    '.gif', // image/gif
-    '.bmp', // image/bmp
-    '.wbmp', // image/vnd.wap.wbmp
-    '.webp',
-    '.tif',
-    '.psd',
-    // text
-    '.svg',
-    '.js',
-    '.jsx',
-    '.json',
-    '.css',
-    '.less',
-    '.html',
-    '.htm',
-    '.xml',
-    '.pdf',
-    // tar
-    '.zip',
-    '.gz',
-    '.tgz',
-    '.gzip',
-    // video
-    '.mp3',
-    '.mp4',
-    '.avi',
-  ],
+  whitelist: uploadWhiteList,
   tmpdir: join(tmpdir(), 'midway-upload-files'),
+  cleanTimeout: 5 * 60 * 1000,
 };

packages/upload/src/configuration.ts

@@ -2,7 +2,12 @@ import { Config, Configuration, Inject } from '@midwayjs/decorator';
 import * as DefaultConfig from './config/config.default';
 import { MidwayApplicationManager } from '@midwayjs/core';
 import { UploadMiddleware } from './middleware';
-import { ensureDir } from 'fs-extra';
+import {
+  autoRemoveUploadTmpFile,
+  ensureDir,
+  stopAutoRemoveUploadTmpFile,
+} from './utils';
+import { UploadOptions } from './interface';
 @Configuration({
   namespace: 'upload',
   importConfigs: [
@@ -16,17 +21,24 @@ export class UploadConfiguration {
   applicationManager: MidwayApplicationManager;
 
   @Config('upload')
-  uploadConfig;
+  uploadConfig: UploadOptions;
 
   async onReady() {
-    const { tmpdir } = this.uploadConfig;
+    const { tmpdir, cleanTimeout } = this.uploadConfig;
     if (tmpdir) {
       await ensureDir(tmpdir);
+      if (cleanTimeout) {
+        autoRemoveUploadTmpFile(tmpdir, cleanTimeout);
+      }
     }
     this.applicationManager
       .getApplications(['koa', 'faas', 'express', 'egg'])
       .forEach(app => {
         app.useMiddleware(UploadMiddleware);
       });
   }
+
+  async onStop() {
+    await stopAutoRemoveUploadTmpFile();
+  }
 }

packages/upload/src/constants.ts

@@ -0,0 +1,32 @@
+export const uploadWhiteList = [
+  // images
+  '.jpg',
+  '.jpeg', // image/jpeg
+  '.png', // image/png, image/x-png
+  '.gif', // image/gif
+  '.bmp', // image/bmp
+  '.wbmp', // image/vnd.wap.wbmp
+  '.webp',
+  '.tif',
+  '.psd',
+  // text
+  '.svg',
+  '.js',
+  '.jsx',
+  '.json',
+  '.css',
+  '.less',
+  '.html',
+  '.htm',
+  '.xml',
+  '.pdf',
+  // tar
+  '.zip',
+  '.gz',
+  '.tgz',
+  '.gzip',
+  // video
+  '.mp3',
+  '.mp4',
+  '.avi',
+];

packages/upload/src/index.ts

@@ -2,3 +2,4 @@ export { UploadConfiguration as Configuration } from './configuration';
 export * from './interface';
 export * from './middleware';
 export * from './error';
+export * from './constants';

packages/upload/src/interface.ts

@@ -7,13 +7,7 @@ export interface UploadOptions {
   fileSize?: string;   // Max file size (in bytes), default is `10mb`
   whitelist?: string[]; // The white ext file names, default is `null`
   tmpdir?: string; // 临时文件目录
-  // autoFields: false, // Auto set fields to parts, default is `false`. Only work on `stream` mode.
-  // fieldNameSize: number; // Max field name size (in bytes), default is `100`
-  // fieldSize: string;  // Max field value size (in bytes), default is `100kb`
-  // fields: number;     // Max number of non-file fields, default is `10`
-  // files: number;      // Max number of file fields, default is `10`
-  // fileExtensions: [],
-  // allowArrayField: false,
+  cleanTimeout?: number; // 临时文件自动清理时间
 }
 
 export interface UploadFileInfo<T> {

packages/upload/src/middleware.ts

@@ -6,11 +6,16 @@ import {
 } from '@midwayjs/decorator';
 import { IMiddleware, IMidwayLogger } from '@midwayjs/core';
 import { resolve, extname } from 'path';
-import { writeFileSync } from 'fs';
+import { promises } from 'fs';
 import { Readable, Stream } from 'stream';
-import { MultipartInvalidFilenameError, UploadOptions } from '.';
-import { parseFromReadableStream, parseMultipart } from './upload';
+import {
+  MultipartInvalidFilenameError,
+  UploadFileInfo,
+  UploadOptions,
+} from '.';
+import { parseFromReadableStream, parseMultipart } from './parse';
 import * as getRawBody from 'raw-body';
+const { unlink, writeFile } = promises;
 
 @Middleware()
 export class UploadMiddleware implements IMiddleware<any, any> {
@@ -40,6 +45,25 @@ export class UploadMiddleware implements IMiddleware<any, any> {
       return next();
     }
     ctx.fields = {};
+    ctx.files = [];
+    ctx.cleanupRequestFiles = async (): Promise<Array<boolean>> => {
+      if (!ctx.files?.length) {
+        return [];
+      }
+      return Promise.all(
+        ctx.files.map(async (fileInfo: UploadFileInfo<any>) => {
+          if (typeof fileInfo.data !== 'string') {
+            return false;
+          }
+          try {
+            await unlink(fileInfo.data);
+            return true;
+          } catch {
+            return false;
+          }
+        })
+      );
+    };
 
     let body;
     if (this.isReadableStream(req, isExpress)) {
@@ -80,23 +104,25 @@ export class UploadMiddleware implements IMiddleware<any, any> {
     if (notCheckFile) {
       throw new MultipartInvalidFilenameError(notCheckFile.filename);
     }
-    ctx.files = files.map((file, index) => {
-      const { data, filename } = file;
-      if (mode === 'file') {
-        const ext = extname(filename);
-        const tmpFileName = resolve(tmpdir, `${requireId}.${index}${ext}`);
-        writeFileSync(tmpFileName, data, 'binary');
-        file.data = tmpFileName;
-      } else if (mode === 'stream') {
-        file.data = new Readable({
-          read() {
-            this.push(data);
-            this.push(null);
-          },
-        });
-      }
-      return file;
-    });
+    ctx.files = await Promise.all(
+      files.map(async (file, index) => {
+        const { data, filename } = file;
+        if (mode === 'file') {
+          const ext = extname(filename);
+          const tmpFileName = resolve(tmpdir, `${requireId}.${index}${ext}`);
+          await writeFile(tmpFileName, data, 'binary');
+          file.data = tmpFileName;
+        } else if (mode === 'stream') {
+          file.data = new Readable({
+            read() {
+              this.push(data);
+              this.push(null);
+            },
+          });
+        }
+        return file;
+      })
+    );
 
     return next();
   }

packages/upload/src/utils.ts

@@ -0,0 +1,73 @@
+import { promises, constants } from 'fs';
+import { join } from 'path';
+const { readdir, access, stat, unlink, mkdir } = promises;
+let autoRemoveUploadTmpFileTimeoutHandler;
+let autoRemoveUploadTmpFilePromise;
+export const autoRemoveUploadTmpFile = async (
+  tmpDir: string,
+  cleanTimeout: number
+) => {
+  clearTimeout(autoRemoveUploadTmpFileTimeoutHandler);
+  let waitTime = cleanTimeout / 3;
+  if (waitTime < 1000) {
+    waitTime = 1000;
+  }
+
+  if (autoRemoveUploadTmpFilePromise) {
+    const exists = await checkExists(tmpDir);
+    if (exists) {
+      const paths = await readdir(tmpDir);
+      const now = Date.now();
+      await Promise.all(
+        paths.map(async path => {
+          const filePath = join(tmpDir, path);
+          try {
+            const statInfo = await stat(filePath);
+            if (statInfo.isFile() && now - statInfo.ctimeMs > cleanTimeout) {
+              await unlink(filePath);
+            }
+          } catch {
+            return false;
+          }
+        })
+      );
+    }
+  }
+
+  autoRemoveUploadTmpFileTimeoutHandler = setTimeout(() => {
+    autoRemoveUploadTmpFilePromise = autoRemoveUploadTmpFile(
+      tmpDir,
+      cleanTimeout
+    );
+  }, waitTime);
+};
+
+export const stopAutoRemoveUploadTmpFile = async () => {
+  if (autoRemoveUploadTmpFilePromise) {
+    await autoRemoveUploadTmpFilePromise;
+    autoRemoveUploadTmpFilePromise = null;
+  }
+  clearTimeout(autoRemoveUploadTmpFileTimeoutHandler);
+};
+
+export const checkExists = async (path: string): Promise<boolean> => {
+  try {
+    await access(path, constants.W_OK | constants.R_OK);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+export const ensureDir = async (dirPath: string): Promise<boolean> => {
+  const isExists = await checkExists(dirPath);
+  if (isExists) {
+    return true;
+  }
+  try {
+    await mkdir(dirPath, { recursive: true });
+    return true;
+  } catch {
+    return false;
+  }
+};