Fix bugs in #127 and remove etag in upgrade

package.json

@@ -1,6 +1,6 @@
 {
   "name": "kodo-browser",
-  "version": "1.0.19-alpha",
+  "version": "1.0.19-alpha3",
   "license": "Apache-2.0",
   "author": {
     "name": "Rong Zhou",
@@ -96,6 +96,7 @@
     "bootstrap-table": "^1.12.1",
     "clipboard": "^2.0.4",
     "codemirror": "^5.41.0",
+    "crc32-stream": "^4.0.2",
     "csv-stringify": "^5.3.6",
     "downloads-folder": "1.0.3",
     "electron": "18.3.3",
@@ -105,7 +106,7 @@
     "jquery.qrcode": "^1.0.3",
     "js-base64": "^3.4.5",
     "js-md5": "^0.7.3",
-    "kodo-s3-adapter-sdk": "0.2.31",
+    "kodo-s3-adapter-sdk": "0.2.32",
     "lodash": "^4.17.21",
     "mime": "^2.3.1",
     "moment": "^2.22.2",

src/common/models/job/crc32.test.ts

@@ -0,0 +1,32 @@
+import mockFs from "mock-fs";
+
+import crc32Async from "./crc32";
+import ByteSize from "@common/const/byte-size";
+
+describe("test crc32", () => {
+    beforeAll(() => {
+        mockFs({
+            "/path/to": {
+                "file": "hello kodo browser!\n",
+                "320KB-a.data": Buffer.alloc(5 * 64 * ByteSize.KB).fill(0x61), // 0x61 === 'a'
+            },
+        });
+    });
+    afterAll(() => {
+        mockFs.restore();
+    });
+
+    it("test crc32Async", async () => {
+        const actual = await crc32Async("/path/to/file");
+
+        expect(actual).toBe("F9629E23");
+    });
+
+    it("test crc32Async read at least 3 times", async () => {
+        // Because crc32-stream is a transform stream, it doesn't consume readable stream.
+        // The default highWaterMark of file readable stream is 64KB, so we mock a 5 * 64KB file.
+        const actual = await crc32Async("/path/to/320KB-a.data");
+
+        expect(actual).toBe("61399770");
+    });
+});

src/common/models/job/crc32.ts

@@ -0,0 +1,24 @@
+import fs from "fs";
+import stream from "stream";
+
+// @ts-ignore
+import {CRC32Stream} from "crc32-stream";
+
+export default function (filePath: string): Promise<string> {
+    const fileStream = fs.createReadStream(filePath);
+    const checksum = new CRC32Stream();
+    const emptyConsumer = new stream.Writable({
+        write(_chunk: Buffer, _encoding: BufferEncoding, callback: (error?: (Error | null)) => void) {
+            callback();
+        }
+    });
+
+    return new Promise<string>(resolve => {
+        fileStream
+            .pipe(checksum)
+            .pipe(emptyConsumer)
+            .on("finish", () => {
+                resolve(checksum.hex());
+            });
+    });
+}

src/common/models/job/transfer-job.ts

@@ -54,7 +54,7 @@ export default abstract class TransferJob {
     private lastTimestamp = 0 // ms
     private lastLoaded = 0 // Bytes
     private speed: number = 0 // Bytes/s
-    private estimatedTime: number = 0 // seconds
+    private estimatedTime: number = 0 // timestamp
     // message
     message: string
 
@@ -96,6 +96,7 @@ export default abstract class TransferJob {
             progress: this.prog,
             speed: this.speed,
             estimatedTime: this.estimatedTime,
+            estimatedDuration: this.estimatedTime - Date.now(),
         }
     }
 
@@ -140,15 +141,14 @@ export default abstract class TransferJob {
         }
 
         const nowTimestamp = Date.now();
-        const currentSpeed = (this.prog.loaded - this.lastLoaded) /
-            ((nowTimestamp - this.lastTimestamp) / Duration.Second);
+        const currentSpeedByMs = (this.prog.loaded - this.lastLoaded) / (nowTimestamp - this.lastTimestamp);
 
-        this.speed = Math.round(currentSpeed);
+        this.speed = Math.round(currentSpeedByMs * Duration.Second);
         if (speedLimit > 0) {
             this.speed = Math.min(this.speed, speedLimit);
         }
-        this.estimatedTime = Math.max(
-            Math.round((this.prog.total - this.prog.loaded) / this.speed) * Duration.Second,
+        this.estimatedTime = Date.now() + Math.max(
+            Math.round(this.prog.total - this.prog.loaded) / currentSpeedByMs,
             0,
         );
 

src/common/models/job/upload-job.ts

@@ -13,6 +13,7 @@ import ByteSize from "@common/const/byte-size";
 
 import {LocalPath, RemotePath, Status, UploadedPart} from "./types";
 import TransferJob from "./transfer-job";
+import crc32Async from "@common/models/job/crc32";
 
 // if change options, remember to check `get persistInfo()`
 interface RequiredOptions {
@@ -255,6 +256,7 @@ export default class UploadJob extends TransferJob {
         // upload
         this.uploader = new Uploader(client);
         const fileHandle = await fsPromises.open(this.options.from.path, "r");
+        const fileCrc32Hex = await crc32Async(this.options.from.path);
         await this.uploader.putObjectFromFile(
             this.options.region,
             {
@@ -266,8 +268,9 @@ export default class UploadJob extends TransferJob {
             this.options.from.size,
             this.options.from.name,
             {
+                crc32: parseInt(fileCrc32Hex, 16).toString(),
                 header: {
-                    contentType: mime.getType(this.options.from.path)
+                    contentType: mime.getType(this.options.from.path),
                 },
                 recovered: this.uploadedId && this.uploadedParts
                     ? {

src/common/qiniu/index.ts

@@ -1,3 +1,2 @@
 export * from "./types";
 export {default as createQiniuClient} from "./create-client";
-export * from "./etag";

src/main/index.ts

@@ -412,29 +412,35 @@ function setMenu() {
         {
           label: "Undo",
           accelerator: "CmdOrCtrl+Z",
+          role: "undo"
         },
         {
           label: "Redo",
           accelerator: "Shift+CmdOrCtrl+Z",
+          role: "redo",
         },
         {
           type: "separator"
         },
         {
           label: "Cut",
           accelerator: "CmdOrCtrl+X",
+          role: "cut",
         },
         {
           label: "Copy",
           accelerator: "CmdOrCtrl+C",
+          role: "copy",
         },
         {
           label: "Paste",
           accelerator: "CmdOrCtrl+V",
+          role: "paste",
         },
         {
           label: "Select All",
           accelerator: "CmdOrCtrl+A",
+          role: "selectAll",
         }
       ]
     },

src/renderer/components/services/auto-upgrade.js

@@ -4,7 +4,6 @@ import path from 'path'
 import request from 'request'
 import downloadsFolder from 'downloads-folder'
 
-import {getEtag} from '@common/qiniu'
 import webModule from '@/app-module/web'
 
 import { upgrade } from '@/customize'
@@ -80,15 +79,9 @@ webModule.factory(AUTO_UPGRADE_SVS_FACTORY_NAME, [
         this._changeStatus("finished");
       };
       this.check = function (expected, callback) {
-        //crc
-        console.log("etag check");
-        return getEtag(to + ".download", function(actual) {
-          if (expected !== `"${actual}"`) {
-            callback(new Error(`Etag check failed, expected: ${expected}, actual: "${actual}"`));
-          } else {
-            callback();
-          }
-        });
+        // skip check, because etag is unreliable.
+        // TODO: check x-qiniu-hash-crc64ecma header when crc64-ecma-182 online
+        callback();
       };
 
       this.precheck = function () {

src/renderer/main/files/transfer/downloads.html

@@ -99,7 +99,7 @@
             <span
               ng-if="item.progress.loaded!=item.progress.total">{{item.progress.loaded|sizeFormat}}/</span>{{item.progress.total|sizeFormat}}
           </small>
-          <small ng-show="item.estimatedTime">, {{item.estimatedTime|leftTimeFormat}}</small>
+          <small ng-show="item.estimatedTime">, {{item.estimatedDuration|leftTimeFormat}}</small>
         </div>
 
         <div class="col-sm-2 text-right">

src/renderer/main/files/transfer/frame.js

@@ -411,7 +411,7 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [
           secretKey: AuthInfo.get().secret,
           ucUrl: ngConfig.load().ucUrl || "",
           regions: ngConfig.load().regionId || [],
-          backendMode: bucketInfo.qiniuBackendMode,
+          backendMode: $scope.selectedDomain.domain.qiniuBackendMode(),
         },
       });
     }

src/renderer/main/files/transfer/uploads.html

@@ -100,7 +100,7 @@
             <span
               ng-if="item.progress.loaded!=item.progress.total">{{item.progress.loaded|sizeFormat}}/</span>{{item.progress.total|sizeFormat}}
           </small>
-          <small ng-show="item.estimatedTime">, {{item.estimatedTime|leftTimeFormat}}</small>
+          <small ng-show="item.estimatedTime">, {{item.estimatedDuration|leftTimeFormat}}</small>
           <small ng-if="item.status == 'duplicated'" class="text-danger">
             <i class="glyphicon glyphicon-alert"></i>
             {{'upload.duplicated'|translate}}