fix(uplaod): 文件合并后损坏的问题

Author: tuhongwei

file-upload/public/generate-hash.js

@@ -1,12 +1,11 @@
 self.importScripts('/spark-md5.min.js');
 
 self.onmessage = e => {
-  const { fileChunkList, fileSize } = e.data;
+  const { fileChunkList, fileSize, chunkSize } = e.data;
   const spark = new self.SparkMD5.ArrayBuffer();
   let count = 0;
   let percentage = 0;
   let fileReader = new FileReader();
-  let size = fileChunkList[0].file.size;
   fileReader.onload = e => {
     spark.append(e.target.result); // Append array buffer
     count++;
@@ -22,9 +21,9 @@ self.onmessage = e => {
   };
 
   fileReader.onprogress = e => {
-    // console.log('计算文件hash进度', (e.loaded + size * count) / fileSize * 100);
+    // console.log('计算文件hash进度', (e.loaded + chunkSize * count) / fileSize * 100);
     self.postMessage({
-      percentage: (e.loaded + size * count) / fileSize * 100
+      percentage: (e.loaded + chunkSize * count) / fileSize * 100
     });
   };
 

file-upload/server/controller.js

@@ -17,28 +17,34 @@ const resolvePost = req =>
     });
   });
 
-const pipStream = (path, writeStream) => 
-  new Promise(resolve => {
-    const readStream = fse.createReadStream(path);
-    writeStream.on('finish', () => {
-    	writeStream.end();
-      fse.unlinkSync(path);
+const pipeStream = (path, writeStream, chunkSize) => 
+  new Promise((resolve, reject) => {
+    const readStream = fse.createReadStream(path, {
+      highWaterMark: chunkSize
+    });
+    readStream.on('end', () => {
+      // fse.unlinkSync(path);
+      readStream.unpipe();
       resolve();
     });
-    readStream.pipe(writeStream);
+    readStream.pipe(writeStream, { end: false });
+    readStream.on('error', reject);
   });
 
-const mergeFileChunk = async (fileHash, fileName, size) => {
+const mergeFileChunk = async (fileHash, fileName, chunkSize) => {
   const filePath = path.resolve(UPLOAD_DIR, `${fileHash}${extractExt(fileName)}`);
   const chunkDir = path.resolve(UPLOAD_DIR, fileHash);
   const chunkPaths = await fse.readdir(chunkDir);
   chunkPaths.sort((a, b) => a.slice(a.lastIndexOf('-') + 1) - b.slice(b.lastIndexOf('-') + 1));
-  await Promise.all(chunkPaths.map((chunkPath, index) => 
-    pipStream(path.resolve(chunkDir, chunkPath), fse.createWriteStream(filePath, {
-      start: index * size
-    }))
-  ));
-  fse.rmdirSync(chunkDir); //  合并完成后删除切片目录
+  const writeStream = fse.createWriteStream(filePath);
+  await Promise.all(chunkPaths.map((chunkPath, index) =>
+    pipeStream(path.resolve(chunkDir, chunkPath), writeStream, chunkSize)
+  )).then(() => {
+    // close the stream to prevent memory leaks
+    writeStream.close();
+    return Promise.resolve(filePath);
+  })
+  // fse.rmdirSync(chunkDir); //  合并完成后删除切片目录
 };
 
 module.exports = {
@@ -47,8 +53,8 @@ module.exports = {
   },
   async handleMerge (req, res) {
     const data = await resolvePost(req);
-    const { fileHash, fileName, size } = data;
-    await mergeFileChunk(fileHash, fileName, size);
+    const { fileHash, fileName, chunkSize } = data;
+    await mergeFileChunk(fileHash, fileName, chunkSize);
     res.end(JSON.stringify({
       code: 0,
       msg: 'file merged success'

file-upload/src/views/Home.vue

@@ -15,8 +15,8 @@
 <script>
 import { Button } from 'element-ui';
 
-// const SIZE = 10 * 1024 * 1024; // 切片大小
-const SIZE = 10 * 1024;
+// const CHUNK_SIZE = 10 * 1024 * 1024; // 切片大小
+const CHUNK_SIZE = 10 * 1024;
 
 export default {
   data: () => ({
@@ -55,9 +55,9 @@ export default {
       let cur = 0;
       while (cur < file.size) {
         fileChunkList.push({
-          file: file.slice(cur, cur + SIZE),
+          file: file.slice(cur, cur + CHUNK_SIZE),
         });
-        cur += SIZE;
+        cur += CHUNK_SIZE;
       }
       return fileChunkList;
     },
@@ -95,7 +95,7 @@ export default {
           'Content-Type': 'application/json'
         },
         data: JSON.stringify({
-          size: SIZE,
+          chunkSize: CHUNK_SIZE,
           fileHash: this.container.hash,
           fileName: this.container.file.name
         })
@@ -105,7 +105,7 @@ export default {
     calculateHash (fileChunkList) {
       return new Promise (resolve => {
         const worker = new Worker('/generate-hash.js');
-        worker.postMessage({ fileChunkList, fileSize: this.container.file.size });
+        worker.postMessage({ fileChunkList, fileSize: this.container.file.size, chunkSize: CHUNK_SIZE });
         worker.onmessage = e => {
           const { percentage, hash } = e.data;
           this.hashPercentage = percentage;