From e9208f26f61f2e9ff36e5c977ab36b3dfec9a866 Mon Sep 17 00:00:00 2001 From: LiHS Date: Tue, 2 Aug 2022 13:17:58 +0800 Subject: [PATCH 1/6] change version to v1.0.19-alpha2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1eef1eb3..1a8ff16e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kodo-browser", - "version": "1.0.19-alpha", + "version": "1.0.19-alpha2", "license": "Apache-2.0", "author": { "name": "Rong Zhou", From aca7638d62f142177931b14cd6363c00ca9e098a Mon Sep 17 00:00:00 2001 From: LiHS Date: Mon, 1 Aug 2022 15:01:40 +0800 Subject: [PATCH 2/6] fix menu edit shortcut(015a4a0) --- src/main/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/index.ts b/src/main/index.ts index fee7e908..72b828c4 100755 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -412,10 +412,12 @@ function setMenu() { { label: "Undo", accelerator: "CmdOrCtrl+Z", + role: "undo" }, { label: "Redo", accelerator: "Shift+CmdOrCtrl+Z", + role: "redo", }, { type: "separator" @@ -423,18 +425,22 @@ function setMenu() { { label: "Cut", accelerator: "CmdOrCtrl+X", + role: "cut", }, { label: "Copy", accelerator: "CmdOrCtrl+C", + role: "copy", }, { label: "Paste", accelerator: "CmdOrCtrl+V", + role: "paste", }, { label: "Select All", accelerator: "CmdOrCtrl+A", + role: "selectAll", } ] }, From d68315e79e0da6ac915c3eaa49e79e6e422dcb62 Mon Sep 17 00:00:00 2001 From: LiHS Date: Mon, 1 Aug 2022 15:07:45 +0800 Subject: [PATCH 3/6] remove etag and skip auto upgrade check --- src/common/qiniu/etag.test.ts | 45 ---------- src/common/qiniu/etag.ts | 83 ------------------- src/common/qiniu/index.ts | 1 - .../components/services/auto-upgrade.js | 13 +-- 4 files changed, 3 insertions(+), 139 deletions(-) delete mode 100644 src/common/qiniu/etag.test.ts delete mode 100644 src/common/qiniu/etag.ts diff --git a/src/common/qiniu/etag.test.ts b/src/common/qiniu/etag.test.ts deleted file mode 100644 index 3b11c508..00000000 --- a/src/common/qiniu/etag.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import mockFs from "mock-fs"; -import {Readable as ReadableStream} from "stream"; - -import {getEtag} from "./etag"; - -describe("test getEtag", () => { - it("by Buffer", async () => { - const etag = await new Promise(resolve => { - getEtag(Buffer.from("Hello kodo browser!"), resolve); - }); - expect(etag).toBe("FvV1cV1RzovZlzHlGApMIXL2RRHW"); - }); - it("by path(string)", async () => { - mockFs({ - "/path/to/get/etag": "Hello kodo browser!", - }); - const etag = await new Promise(resolve => { - getEtag("/path/to/get/etag", resolve); - }); - expect(etag).toBe("FvV1cV1RzovZlzHlGApMIXL2RRHW"); - mockFs.restore(); - }); - it("by ReadableStream", async () => { - class MyStream extends ReadableStream { - data = "Hello kodo browser!" - - _read(size: number) { - if (this.data.length) { - let chunk: string; - [chunk, this.data] = [this.data.slice(0, size), this.data.slice(size)]; - this.push(chunk); - } else { - this.push(null); - } - } - } - - const readableStream = new MyStream(); - - const etag = await new Promise(resolve => { - getEtag(readableStream, resolve); - }); - expect(etag).toBe("FvV1cV1RzovZlzHlGApMIXL2RRHW"); - }); -}); diff --git a/src/common/qiniu/etag.ts b/src/common/qiniu/etag.ts deleted file mode 100644 index 8c17c0fc..00000000 --- a/src/common/qiniu/etag.ts +++ /dev/null @@ -1,83 +0,0 @@ -import crypto from "crypto"; -import { Readable as ReadableStream } from "stream"; -import fs from "fs"; - -// get etag -function sha1(content: Buffer): Buffer { - return crypto.createHash("sha1") - .update(content) - .digest(); -} -function calcEtag(sha1StringList: Buffer[], blockCount: number): string { - if (!sha1StringList.length) { - return 'Fto5o-5ea0sNMlW_75VgGJCv2AcJ'; - } - let sha1Buffer = Buffer.concat(sha1StringList, blockCount * 20); - - let prefix = 0x16; - // 如果大于4M，则对各个块的sha1结果再次sha1，why? - if (blockCount > 1){ - prefix = 0x96; - sha1Buffer = sha1(sha1Buffer); - } - - sha1Buffer = Buffer.concat([Buffer.from([prefix]), sha1Buffer], sha1Buffer.length + 1); - - return sha1Buffer.toString('base64') - .replace(/\//g,'_').replace(/\+/g,'-'); -} -function getEtagByBuffer(content: Buffer, callback: (etag: string) => void): void { - // 以4M为单位分割，why? - const blockSize = 4*1024*1024; - const blockCount = Math.ceil(content.length / blockSize); - const sha1String = chunk(content, blockSize).map(sha1); - - process.nextTick(function(){ - callback(calcEtag(sha1String, blockCount)); - }); - - function chunk(buffer: Buffer, size: number): Buffer[] { - const result = Array(Math.ceil(buffer.length / size)); - let index = 0; - let resIndex = 0; - - while (index < buffer.length) { - result[resIndex++] = buffer.slice(index, (index += size)); - } - return result; - } -} -function getEtagByStream(dataStream: ReadableStream, callback: (etag: string) => void): void { - // 以 4M 为单位分割，why? - // 对外暴露的方法仅有自动下载新版本，猜测时用于验证是否下载完整，后续 crc 上线后需要修改这里 - // 4M 是上传新版本到七牛空间时采用的分片 - const blockSize = 4*1024*1024; - const sha1String: Buffer[] = []; - let blockCount = 0; - - dataStream.on('readable', function() { - let chunk; - while (chunk = dataStream.read(blockSize)) { - sha1String.push(sha1(chunk)); - blockCount++; - } - }); - dataStream.on('end',function(){ - callback(calcEtag(sha1String, blockCount)); - }); -} -function getEtagByPath(path: string, callback: (etag: string) => void): void { - const dataStream = fs.createReadStream(path); - getEtagByStream(dataStream, callback) -} -export function getEtag(data: Buffer | string | ReadableStream, callback: (etag: string) => void): void { - if (typeof data === 'string') { - getEtagByPath(data, callback); - return - } - if (data instanceof ReadableStream){ - getEtagByStream(data, callback); - return; - } - getEtagByBuffer(data, callback); -} diff --git a/src/common/qiniu/index.ts b/src/common/qiniu/index.ts index 5163e954..3f7f8afc 100644 --- a/src/common/qiniu/index.ts +++ b/src/common/qiniu/index.ts @@ -1,3 +1,2 @@ export * from "./types"; export {default as createQiniuClient} from "./create-client"; -export * from "./etag"; diff --git a/src/renderer/components/services/auto-upgrade.js b/src/renderer/components/services/auto-upgrade.js index 80c021ae..789f262c 100755 --- a/src/renderer/components/services/auto-upgrade.js +++ b/src/renderer/components/services/auto-upgrade.js @@ -4,7 +4,6 @@ import path from 'path' import request from 'request' import downloadsFolder from 'downloads-folder' -import {getEtag} from '@common/qiniu' import webModule from '@/app-module/web' import { upgrade } from '@/customize' @@ -80,15 +79,9 @@ webModule.factory(AUTO_UPGRADE_SVS_FACTORY_NAME, [ this._changeStatus("finished"); }; this.check = function (expected, callback) { - //crc - console.log("etag check"); - return getEtag(to + ".download", function(actual) { - if (expected !== `"${actual}"`) { - callback(new Error(`Etag check failed, expected: ${expected}, actual: "${actual}"`)); - } else { - callback(); - } - }); + // skip check, because etag is unreliable. + // TODO: check x-qiniu-hash-crc64ecma header when crc64-ecma-182 online + callback(); }; this.precheck = function () { From e368b8ca44889dfb8a5825680b4383130f921222 Mon Sep 17 00:00:00 2001 From: LiHS Date: Tue, 2 Aug 2022 12:42:28 +0800 Subject: [PATCH 4/6] fix can't download without own domain(015a4a0) --- src/renderer/main/files/transfer/frame.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/renderer/main/files/transfer/frame.js b/src/renderer/main/files/transfer/frame.js index 8f8c6498..8617f0eb 100755 --- a/src/renderer/main/files/transfer/frame.js +++ b/src/renderer/main/files/transfer/frame.js @@ -411,7 +411,7 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ secretKey: AuthInfo.get().secret, ucUrl: ngConfig.load().ucUrl || "", regions: ngConfig.load().regionId || [], - backendMode: bucketInfo.qiniuBackendMode, + backendMode: $scope.selectedDomain.domain.qiniuBackendMode(), }, }); } From 523270c5c165dd7210e3bd6989ada0d7181d1c4e Mon Sep 17 00:00:00 2001 From: LiHS Date: Tue, 2 Aug 2022 15:25:41 +0800 Subject: [PATCH 5/6] fix transfer ETA not refresh for a long time --- src/common/models/job/transfer-job.ts | 12 ++++++------ src/renderer/main/files/transfer/downloads.html | 2 +- src/renderer/main/files/transfer/uploads.html | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/common/models/job/transfer-job.ts b/src/common/models/job/transfer-job.ts index 204e265e..818fea80 100644 --- a/src/common/models/job/transfer-job.ts +++ b/src/common/models/job/transfer-job.ts @@ -54,7 +54,7 @@ export default abstract class TransferJob { private lastTimestamp = 0 // ms private lastLoaded = 0 // Bytes private speed: number = 0 // Bytes/s - private estimatedTime: number = 0 // seconds + private estimatedTime: number = 0 // timestamp // message message: string @@ -96,6 +96,7 @@ export default abstract class TransferJob { progress: this.prog, speed: this.speed, estimatedTime: this.estimatedTime, + estimatedDuration: this.estimatedTime - Date.now(), } } @@ -140,15 +141,14 @@ export default abstract class TransferJob { } const nowTimestamp = Date.now(); - const currentSpeed = (this.prog.loaded - this.lastLoaded) / - ((nowTimestamp - this.lastTimestamp) / Duration.Second); + const currentSpeedByMs = (this.prog.loaded - this.lastLoaded) / (nowTimestamp - this.lastTimestamp); - this.speed = Math.round(currentSpeed); + this.speed = Math.round(currentSpeedByMs * Duration.Second); if (speedLimit > 0) { this.speed = Math.min(this.speed, speedLimit); } - this.estimatedTime = Math.max( - Math.round((this.prog.total - this.prog.loaded) / this.speed) * Duration.Second, + this.estimatedTime = Date.now() + Math.max( + Math.round(this.prog.total - this.prog.loaded) / currentSpeedByMs, 0, ); diff --git a/src/renderer/main/files/transfer/downloads.html b/src/renderer/main/files/transfer/downloads.html index e8b7c1d0..b5ad59ce 100755 --- a/src/renderer/main/files/transfer/downloads.html +++ b/src/renderer/main/files/transfer/downloads.html @@ -99,7 +99,7 @@ {{item.progress.loaded|sizeFormat}}/{{item.progress.total|sizeFormat}} - , {{item.estimatedTime|leftTimeFormat}} + , {{item.estimatedDuration|leftTimeFormat}}
diff --git a/src/renderer/main/files/transfer/uploads.html b/src/renderer/main/files/transfer/uploads.html index 501b1011..c11a8dc9 100755 --- a/src/renderer/main/files/transfer/uploads.html +++ b/src/renderer/main/files/transfer/uploads.html @@ -100,7 +100,7 @@ {{item.progress.loaded|sizeFormat}}/{{item.progress.total|sizeFormat}} - , {{item.estimatedTime|leftTimeFormat}} + , {{item.estimatedDuration|leftTimeFormat}} {{'upload.duplicated'|translate}} From 4a01c7842159f021fb8271c5c79f2defc3fcf18b Mon Sep 17 00:00:00 2001 From: LiHS Date: Thu, 4 Aug 2022 00:20:40 +0800 Subject: [PATCH 6/6] bump kodo-s3-adapter-sdk to v0.2.32 and fix bugs - add async crc32 to prevent block upload worker; - fix split upload callback even abort; - fix split upload block upload worker by remove useless auth; --- package.json | 5 +++-- src/common/models/job/crc32.test.ts | 32 +++++++++++++++++++++++++++++ src/common/models/job/crc32.ts | 24 ++++++++++++++++++++++ src/common/models/job/upload-job.ts | 5 ++++- yarn.lock | 21 +++++++++++++++---- 5 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 src/common/models/job/crc32.test.ts create mode 100644 src/common/models/job/crc32.ts diff --git a/package.json b/package.json index 1a8ff16e..3f6b1a92 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kodo-browser", - "version": "1.0.19-alpha2", + "version": "1.0.19-alpha3", "license": "Apache-2.0", "author": { "name": "Rong Zhou", @@ -96,6 +96,7 @@ "bootstrap-table": "^1.12.1", "clipboard": "^2.0.4", "codemirror": "^5.41.0", + "crc32-stream": "^4.0.2", "csv-stringify": "^5.3.6", "downloads-folder": "1.0.3", "electron": "18.3.3", @@ -105,7 +106,7 @@ "jquery.qrcode": "^1.0.3", "js-base64": "^3.4.5", "js-md5": "^0.7.3", - "kodo-s3-adapter-sdk": "0.2.31", + "kodo-s3-adapter-sdk": "0.2.32", "lodash": "^4.17.21", "mime": "^2.3.1", "moment": "^2.22.2", diff --git a/src/common/models/job/crc32.test.ts b/src/common/models/job/crc32.test.ts new file mode 100644 index 00000000..662b5051 --- /dev/null +++ b/src/common/models/job/crc32.test.ts @@ -0,0 +1,32 @@ +import mockFs from "mock-fs"; + +import crc32Async from "./crc32"; +import ByteSize from "@common/const/byte-size"; + +describe("test crc32", () => { + beforeAll(() => { + mockFs({ + "/path/to": { + "file": "hello kodo browser!\n", + "320KB-a.data": Buffer.alloc(5 * 64 * ByteSize.KB).fill(0x61), // 0x61 === 'a' + }, + }); + }); + afterAll(() => { + mockFs.restore(); + }); + + it("test crc32Async", async () => { + const actual = await crc32Async("/path/to/file"); + + expect(actual).toBe("F9629E23"); + }); + + it("test crc32Async read at least 3 times", async () => { + // Because crc32-stream is a transform stream, it doesn't consume readable stream. + // The default highWaterMark of file readable stream is 64KB, so we mock a 5 * 64KB file. + const actual = await crc32Async("/path/to/320KB-a.data"); + + expect(actual).toBe("61399770"); + }); +}); diff --git a/src/common/models/job/crc32.ts b/src/common/models/job/crc32.ts new file mode 100644 index 00000000..f0ff56ff --- /dev/null +++ b/src/common/models/job/crc32.ts @@ -0,0 +1,24 @@ +import fs from "fs"; +import stream from "stream"; + +// @ts-ignore +import {CRC32Stream} from "crc32-stream"; + +export default function (filePath: string): Promise { + const fileStream = fs.createReadStream(filePath); + const checksum = new CRC32Stream(); + const emptyConsumer = new stream.Writable({ + write(_chunk: Buffer, _encoding: BufferEncoding, callback: (error?: (Error | null)) => void) { + callback(); + } + }); + + return new Promise(resolve => { + fileStream + .pipe(checksum) + .pipe(emptyConsumer) + .on("finish", () => { + resolve(checksum.hex()); + }); + }); +} diff --git a/src/common/models/job/upload-job.ts b/src/common/models/job/upload-job.ts index 848c49c9..e27f9566 100644 --- a/src/common/models/job/upload-job.ts +++ b/src/common/models/job/upload-job.ts @@ -13,6 +13,7 @@ import ByteSize from "@common/const/byte-size"; import {LocalPath, RemotePath, Status, UploadedPart} from "./types"; import TransferJob from "./transfer-job"; +import crc32Async from "@common/models/job/crc32"; // if change options, remember to check `get persistInfo()` interface RequiredOptions { @@ -255,6 +256,7 @@ export default class UploadJob extends TransferJob { // upload this.uploader = new Uploader(client); const fileHandle = await fsPromises.open(this.options.from.path, "r"); + const fileCrc32Hex = await crc32Async(this.options.from.path); await this.uploader.putObjectFromFile( this.options.region, { @@ -266,8 +268,9 @@ export default class UploadJob extends TransferJob { this.options.from.size, this.options.from.name, { + crc32: parseInt(fileCrc32Hex, 16).toString(), header: { - contentType: mime.getType(this.options.from.path) + contentType: mime.getType(this.options.from.path), }, recovered: this.uploadedId && this.uploadedParts ? { diff --git a/yarn.lock b/yarn.lock index 0739c3a3..d6f3867a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3360,6 +3360,11 @@ core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== +crc-32@^1.2.0: + version "1.2.2" + resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.2.tgz#3cad35a934b8bf71f25ca524b6da51fb7eace2ff" + integrity sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ== + crc32-stream@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/crc32-stream/-/crc32-stream-3.0.1.tgz#cae6eeed003b0e44d739d279de5ae63b171b4e85" @@ -3368,6 +3373,14 @@ crc32-stream@^3.0.1: crc "^3.4.4" readable-stream "^3.4.0" +crc32-stream@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-4.0.2.tgz#c922ad22b38395abe9d3870f02fa8134ed709007" + integrity sha512-DxFZ/Hk473b/muq1VJ///PMNLj0ZMnzye9thBpmjpJKCc5eMgB95aK8zCGrGfQ90cWo561Te6HK9D+j4KPdM6w== + dependencies: + crc-32 "^1.2.0" + readable-stream "^3.4.0" + crc@^3.4.4: version "3.8.0" resolved "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz#ad60269c2c856f8c299e2c4cc0de4556914056c6" @@ -6906,10 +6919,10 @@ kleur@^3.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== -kodo-s3-adapter-sdk@0.2.31: - version "0.2.31" - resolved "https://registry.yarnpkg.com/kodo-s3-adapter-sdk/-/kodo-s3-adapter-sdk-0.2.31.tgz#bafdcc178e9e7984c53a44bd3d2e1f5957fc84fb" - integrity sha512-aHvFFoFJ4memT2MuktlkiE7fHm8UQXfcwSWTf0sUTqFtFzCUJyi8FIxH20fTjSSjEFirydOrQxcZZMS0qjtf6g== +kodo-s3-adapter-sdk@0.2.32: + version "0.2.32" + resolved "https://registry.yarnpkg.com/kodo-s3-adapter-sdk/-/kodo-s3-adapter-sdk-0.2.32.tgz#80932d034a6626d12708026fa426218e2e173bf3" + integrity sha512-5A72s6kbw3LRkkS45so9K1jMjuE+SHo5+mRfw9sFqvuumW1x99Kp6/nSRdQoLUPBV+RpnO/J6LDGDCCHz3IodA== dependencies: async-lock "^1.2.4" aws-sdk "^2.800.0"