From 7aed3bcab6088db3902e9cfb9b36bc382dc12140 Mon Sep 17 00:00:00 2001 From: LiHS Date: Thu, 16 Jun 2022 11:34:58 +0800 Subject: [PATCH 1/8] migrate electron to v18.3.3 --- gulpfile.js | 2 +- package.json | 3 +- src/main/index.js | 9 +- src/renderer/components/services/dialog.s.js | 46 ++-- yarn.lock | 258 ++++--------------- 5 files changed, 86 insertions(+), 232 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index f6d794fb..6f0df38f 100755 --- a/gulpfile.js +++ b/gulpfile.js @@ -11,7 +11,7 @@ const gulp = require("gulp"), const NAME = 'Kodo Browser'; const KICK_NAME = 'kodo-browser'; const VERSION = pkg.version; -const ELECTRON_VERSION = "4.2.7"; +const ELECTRON_VERSION = "18.3.3"; const ROOT = __dirname; const ICONS = `${ROOT}/src/renderer/icons`; const DIST = `${ROOT}/dist`; diff --git a/package.json b/package.json index 5cb662a9..e993a4c6 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "webpack-cli": "^4.9.0" }, "dependencies": { + "@electron/remote": "^2.0.8", "@root/walk": "^1.1.0", "@uirouter/angularjs": "^1.0.20", "angular": "1.7.9", @@ -97,7 +98,7 @@ "codemirror": "^5.41.0", "csv-stringify": "^5.3.6", "downloads-folder": "1.0.3", - "electron": "^4.0", + "electron": "18.3.3", "events": "^2.0.0", "form-data": "^4.0.0", "jquery": "^3.5.1", diff --git a/src/main/index.js b/src/main/index.js index 4cc9b185..be48d1a6 100755 --- a/src/main/index.js +++ b/src/main/index.js @@ -1,7 +1,7 @@ -const electron = require("electron"); const fs = require("fs"); const os = require("os"); const path = require("path"); +const electron = require("electron"); const { app, globalShortcut, @@ -10,6 +10,8 @@ const { ipcMain, BrowserWindow } = electron; +const electronRemote = require('@electron/remote/main'); +electronRemote.initialize(); const { fork } = require("child_process"); @@ -69,6 +71,10 @@ let createWindow = () => { minHeight: 600, title: "Kodo Browser", icon: path.join(iconRoot, "icon.ico"), + webPreferences: { + nodeIntegration: true, + contextIsolation: false, + }, }; let confirmForWorkers = (e) => { @@ -165,6 +171,7 @@ let createWindow = () => { win = new BrowserWindow(opt); win.setTitle(opt.title); win.setMenuBarVisibility(false); + electronRemote.enable(win.webContents); // Emitted before window reload win.on("beforeunload", confirmForWorkers); diff --git a/src/renderer/components/services/dialog.s.js b/src/renderer/components/services/dialog.s.js index 65e78b77..f0a7b846 100755 --- a/src/renderer/components/services/dialog.s.js +++ b/src/renderer/components/services/dialog.s.js @@ -1,4 +1,4 @@ -import { remote as electronRemote } from 'electron' +import { dialog as electronDialog } from '@electron/remote' import webModule from '../../app-module/web' @@ -12,8 +12,6 @@ webModule .factory(DIALOG_FACTORY_NAME, [ "$uibModal", function ($modal) { - var dialog = electronRemote.dialog; - return { alert: alert, confirm: confirm, @@ -25,27 +23,33 @@ webModule function showUploadDialog(fn) { var isMac = navigator.userAgent.indexOf("Macintosh") != -1; - dialog.showOpenDialog({ - title: "Upload", - properties: isMac ? - ["openFile", "openDirectory", "multiSelections"] : - ["openFile", "multiSelections"] - }, - function (filePaths) { - if (typeof fn == "function") fn(filePaths); - } - ); + electronDialog.showOpenDialog( + { + title: "Upload", + properties: isMac ? + ["openFile", "openDirectory", "multiSelections"] : + ["openFile", "multiSelections"] + }, + ) + .then(({canceled, filePaths}) => { + if (!canceled && typeof fn == "function") { + fn(filePaths); + } + }); } function showDownloadDialog(fn) { - dialog.showOpenDialog({ - title: "Download", - properties: ["openDirectory"] - }, - function (filePaths) { - if (typeof fn == "function") fn(filePaths); - } - ); + electronDialog.showOpenDialog( + { + title: "Download", + properties: ["openDirectory"] + }, + ) + .then(({canceled, filePaths}) => { + if (!canceled && typeof fn == "function") { + fn(filePaths); + } + }); } /** diff --git a/yarn.lock b/yarn.lock index 73739207..c19ed0c6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1068,6 +1068,22 @@ resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.5.tgz#9283c9ce5b289a3c4f61c12757469e59377f81f3" integrity sha512-6nFkfkmSeV/rqSaS4oWHgmpnYw194f6hmWF5is6b0J1naJZoiD0NTc9AiUwPHvWsowkjuHErCZT1wa0jg+BLIA== +"@electron/get@^1.13.0": + version "1.14.1" + resolved "https://registry.yarnpkg.com/@electron/get/-/get-1.14.1.tgz#16ba75f02dffb74c23965e72d617adc721d27f40" + integrity sha512-BrZYyL/6m0ZXz/lDxy/nlVhQz+WF+iPS6qXolEU8atw7h6v1aYkjwJZ63m+bJMBTxDE66X+r2tPS4a/8C82sZw== + dependencies: + debug "^4.1.1" + env-paths "^2.2.0" + fs-extra "^8.1.0" + got "^9.6.0" + progress "^2.0.3" + semver "^6.2.0" + sumchecker "^3.0.1" + optionalDependencies: + global-agent "^3.0.0" + global-tunnel-ng "^2.7.1" + "@electron/get@^1.6.0": version "1.12.2" resolved "https://registry.npmjs.org/@electron/get/-/get-1.12.2.tgz#6442066afb99be08cefb9a281e4b4692b33764f3" @@ -1084,6 +1100,11 @@ global-agent "^2.0.2" global-tunnel-ng "^2.7.1" +"@electron/remote@^2.0.8": + version "2.0.8" + resolved "https://registry.yarnpkg.com/@electron/remote/-/remote-2.0.8.tgz#85ff321f0490222993207106e2f720273bb1a5c3" + integrity sha512-P10v3+iFCIvEPeYzTWWGwwHmqWnjoh8RYnbtZAb3RlQefy4guagzIwcWtfftABIfm6JJTNQf4WPSKWZOpLmHXw== + "@iarna/cli@^1.2.0": version "1.2.0" resolved "https://registry.npmjs.org/@iarna/cli/-/cli-1.2.0.tgz#0f7af5e851afe895104583c4ca07377a8094d641" @@ -1475,10 +1496,10 @@ resolved "https://registry.npmjs.org/@types/node/-/node-14.14.19.tgz#5135176a8330b88ece4e9ab1fdcfc0a545b4bab4" integrity sha512-4nhBPStMK04rruRVtVc6cDqhu7S9GZai0fpXgPXrFpcPX6Xul8xnrjSdGB4KPBVYG/R5+fXWdCM8qBoiULWGPQ== -"@types/node@^10.12.18": - version "10.17.50" - resolved "https://registry.npmjs.org/@types/node/-/node-10.17.50.tgz#7a20902af591282aa9176baefc37d4372131c32d" - integrity sha512-vwX+/ija9xKc/z9VqMCdbf4WYcMTGsI0I/L/6shIF3qXURxZOhPQlPRHtjTpiNhAwn0paMJzlOQqw6mAGEQnTA== +"@types/node@^16.11.26": + version "16.11.39" + resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.39.tgz#07223cd2bc332ad9d92135e3a522eebdee3b060e" + integrity sha512-K0MsdV42vPwm9L6UwhIxMAOmcvH/1OoVkZyCgEtVu4Wx7sElGloy/W7kMBNe/oJ7V/jW9BVt1F6RahH6e7tPXw== "@types/prettier@^2.1.5": version "2.4.1" @@ -2075,11 +2096,6 @@ array-each@^1.0.0, array-each@^1.0.1: resolved "https://registry.yarnpkg.com/array-each/-/array-each-1.0.1.tgz#a794af0c05ab1752846ee753a1f211a05ba0c44f" integrity sha1-p5SvDAWrF1KEbudTofIRoFugxE8= -array-find-index@^1.0.1: - version "1.0.2" - resolved "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz#df010aa1287e164bbda6f9723b0a96a1ec4187a1" - integrity sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E= - array-initial@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/array-initial/-/array-initial-1.1.0.tgz#2fa74b26739371c3947bd7a7adc73be334b3d795" @@ -2758,19 +2774,6 @@ camel-case@^4.1.1: pascal-case "^3.1.2" tslib "^2.0.3" -camelcase-keys@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz#308beeaffdf28119051efa1d932213c91b8f92e7" - integrity sha1-MIvur/3ygRkFHvodkyITyRuPkuc= - dependencies: - camelcase "^2.0.0" - map-obj "^1.0.0" - -camelcase@^2.0.0: - version "2.1.1" - resolved "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz#7c1d16d679a1bbe59ca02cacecfb011e201f5a1f" - integrity sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8= - camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" @@ -3491,13 +3494,6 @@ cuint@^0.2.2: resolved "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz#408086d409550c2631155619e9fa7bcadc3b991b" integrity sha1-QICG1AlVDCYxFVYZ6fp7ytw7mRs= -currently-unhandled@^0.4.1: - version "0.4.1" - resolved "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea" - integrity sha1-mI3zP+qxke95mmE2nddsF635V+o= - dependencies: - array-find-index "^1.0.1" - cyclist@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" @@ -3542,7 +3538,7 @@ dateformat@^2.0.0: resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-2.2.0.tgz#4065e2013cf9fb916ddfd82efb506ad4c6769062" integrity sha1-QGXiATz5+5Ft39gu+1Bq1MZ2kGI= -debug@2, debug@2.6.9, debug@^2.1.3, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9: +debug@2, debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9: version "2.6.9" resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== @@ -3563,7 +3559,7 @@ debug@4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: dependencies: ms "2.1.2" -debug@^3.0.0, debug@^3.1.0: +debug@^3.1.0: version "3.2.7" resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== @@ -3582,7 +3578,7 @@ debuglog@^1.0.1: resolved "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" integrity sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI= -decamelize@^1.1.1, decamelize@^1.1.2, decamelize@^1.2.0: +decamelize@^1.1.1, decamelize@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= @@ -3982,21 +3978,6 @@ electron-builder@^20.34.0: update-notifier "^3.0.0" yargs "^13.2.4" -electron-download@^4.1.0: - version "4.1.1" - resolved "https://registry.npmjs.org/electron-download/-/electron-download-4.1.1.tgz#02e69556705cc456e520f9e035556ed5a015ebe8" - integrity sha512-FjEWG9Jb/ppK/2zToP+U5dds114fM1ZOJqMAR4aXXL5CvyPE9fiqBK/9YcwC9poIFQTEJk/EM/zyRwziziRZrg== - dependencies: - debug "^3.0.0" - env-paths "^1.0.0" - fs-extra "^4.0.1" - minimist "^1.2.0" - nugget "^2.0.1" - path-exists "^3.0.0" - rc "^1.2.1" - semver "^5.4.1" - sumchecker "^2.0.2" - electron-installer-dmg@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/electron-installer-dmg/-/electron-installer-dmg-3.0.0.tgz#08935b602f3120981c8d5fd24d3f6525f8b0198a" @@ -4086,13 +4067,13 @@ electron-to-chromium@^1.3.878: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.886.tgz#ac039c4001b665b1dd0f0ed9c2e4da90ff3c9267" integrity sha512-+vYdeBosI63VkCtNWnEVFjgNd/IZwvnsWkKyPtWAvrhA+XfByKoBJcbsMgudVU/bUcGAF9Xp3aXn96voWlc3oQ== -electron@^4.0: - version "4.2.12" - resolved "https://registry.npmjs.org/electron/-/electron-4.2.12.tgz#8e8926a6a6654cde5eb0612952fed98a56941875" - integrity sha512-EES8eMztoW8gEP5E4GQLP8slrfS2jqTYtHbu36mlu3k1xYAaNPyQQr6mCILkYxqj4l3la4CT2Vcs89CUG62vcQ== +electron@18.3.3: + version "18.3.3" + resolved "https://registry.yarnpkg.com/electron/-/electron-18.3.3.tgz#1c48273c1ad1522b8c18f19575e862c7ccd9f409" + integrity sha512-LYxf3uCDc/r0klu7LL0eZLxkseoGIY/vrCfS0Qj4YTU3M7LLjOaIqrajI7icKwaI2dgxiuJJH3n4eqALFpJAFg== dependencies: - "@types/node" "^10.12.18" - electron-download "^4.1.0" + "@electron/get" "^1.13.0" + "@types/node" "^16.11.26" extract-zip "^1.0.3" emittery@^0.8.1: @@ -4157,11 +4138,6 @@ entities@^2.0.0: resolved "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5" integrity sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w== -env-paths@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/env-paths/-/env-paths-1.0.0.tgz#4168133b42bb05c38a35b1ae4397c8298ab369e0" - integrity sha1-QWgTO0K7BcOKNbGuQ5fIKYqzaeA= - env-paths@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/env-paths/-/env-paths-2.2.0.tgz#cdca557dc009152917d6166e2febe1f039685e43" @@ -4856,7 +4832,7 @@ fs-extra-p@^8.0.2: bluebird-lst "^1.0.9" fs-extra "^8.1.0" -fs-extra@^4.0.0, fs-extra@^4.0.1: +fs-extra@^4.0.0: version "4.0.3" resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.3.tgz#0d852122e5bc5beb453fb028e9c0c9bf36340c94" integrity sha512-q6rbdDd1o2mAnQreO7YADIxf/Whx4AHBiRf6d+/cVT8h44ss+lHgxf1FemcqDnQt9X3ct4McHr+JMGlYSsK7Cg== @@ -5058,11 +5034,6 @@ get-package-type@^0.1.0: resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== -get-stdin@^4.0.1: - version "4.0.1" - resolved "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz#b968c6b0a04384324902e8bf1a5df32579a450fe" - integrity sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4= - get-stream@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" @@ -5204,6 +5175,18 @@ global-agent@^2.0.2: semver "^7.3.2" serialize-error "^7.0.1" +global-agent@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/global-agent/-/global-agent-3.0.0.tgz#ae7cd31bd3583b93c5a16437a1afe27cc33a1ab6" + integrity sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q== + dependencies: + boolean "^3.0.1" + es6-error "^4.1.1" + matcher "^3.0.0" + roarr "^2.15.3" + semver "^7.3.2" + serialize-error "^7.0.1" + global-dirs@^0.1.0: version "0.1.1" resolved "https://registry.npmjs.org/global-dirs/-/global-dirs-0.1.1.tgz#b319c0dd4607f353f3be9cca4c72fc148c49f445" @@ -5782,13 +5765,6 @@ imurmurhash@^0.1.4: resolved "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= -indent-string@^2.1.0: - version "2.1.0" - resolved "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz#8e2d48348742121b4a8218b7a137e9a52049dc80" - integrity sha1-ji1INIdCEhtKghi3oTfppSBJ3IA= - dependencies: - repeating "^2.0.0" - infer-owner@^1.0.3, infer-owner@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz#c4cefcaa8e51051c2a40ba2ce8a3d27295af9467" @@ -5989,11 +5965,6 @@ is-extglob@^2.1.0, is-extglob@^2.1.1: resolved "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI= -is-finite@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/is-finite/-/is-finite-1.1.0.tgz#904135c77fb42c0641d6aa1bcdbc4daa8da082f3" - integrity sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w== - is-fullwidth-code-point@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" @@ -7449,14 +7420,6 @@ lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== -loud-rejection@^1.0.0: - version "1.6.0" - resolved "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz#5b46f80147edee578870f086d04821cf998e551f" - integrity sha1-W0b4AUft7leIcPCG0Eghz5mOVR8= - dependencies: - currently-unhandled "^0.4.1" - signal-exit "^3.0.0" - lower-case@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-2.0.2.tgz#6fa237c63dbdc4a82ca0fd882e4722dc5e634e28" @@ -7558,11 +7521,6 @@ map-cache@^0.2.0, map-cache@^0.2.2: resolved "https://registry.yarnpkg.com/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf" integrity sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8= -map-obj@^1.0.0, map-obj@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d" - integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0= - map-stream@^0.0.7: version "0.0.7" resolved "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz#8a1f07896d82b10926bd3744a2420009f88974a8" @@ -7609,22 +7567,6 @@ memorystream@^0.3.1: resolved "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz#86d7090b30ce455d63fbae12dda51a47ddcaf9b2" integrity sha1-htcJCzDORV1j+64S3aUaR93K+bI= -meow@^3.1.0: - version "3.7.0" - resolved "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz#72cb668b425228290abbfa856892587308a801fb" - integrity sha1-cstmi0JSKCkKu/qFaJJYcwioAfs= - dependencies: - camelcase-keys "^2.0.0" - decamelize "^1.1.2" - loud-rejection "^1.0.0" - map-obj "^1.0.1" - minimist "^1.1.3" - normalize-package-data "^2.3.4" - object-assign "^4.0.1" - read-pkg-up "^1.0.1" - redent "^1.0.0" - trim-newlines "^1.0.0" - merge-stream@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" @@ -7982,7 +7924,7 @@ nopt@^4.0.1, nopt@^4.0.3: abbrev "1" osenv "^0.1.4" -normalize-package-data@^2.0.0, normalize-package-data@^2.3.2, normalize-package-data@^2.3.4, normalize-package-data@^2.4.0, normalize-package-data@^2.5.0: +normalize-package-data@^2.0.0, normalize-package-data@^2.3.2, normalize-package-data@^2.4.0, normalize-package-data@^2.5.0: version "2.5.0" resolved "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA== @@ -8297,19 +8239,6 @@ nth-check@^2.0.0: dependencies: boolbase "^1.0.0" -nugget@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/nugget/-/nugget-2.0.1.tgz#201095a487e1ad36081b3432fa3cada4f8d071b0" - integrity sha1-IBCVpIfhrTYIGzQy+jytpPjQcbA= - dependencies: - debug "^2.1.3" - minimist "^1.1.0" - pretty-bytes "^1.0.2" - progress-stream "^1.1.0" - request "^2.45.0" - single-line-log "^1.1.2" - throttleit "0.0.2" - number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" @@ -8359,11 +8288,6 @@ object-keys@^1.0.12, object-keys@^1.1.1: resolved "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== -object-keys@~0.4.0: - version "0.4.0" - resolved "https://registry.npmjs.org/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" - integrity sha1-KKaq50KN0sOpLz2V8hM13SBOAzY= - object-visit@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/object-visit/-/object-visit-1.0.1.tgz#f79c4493af0c5377b59fe39d395e41042dd045bb" @@ -8987,14 +8911,6 @@ prepend-http@^2.0.0: resolved "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897" integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc= -pretty-bytes@^1.0.2: - version "1.0.4" - resolved "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-1.0.4.tgz#0a22e8210609ad35542f8c8d5d2159aff0751c84" - integrity sha1-CiLoIQYJrTVUL4yNXSFZr/B1HIQ= - dependencies: - get-stdin "^4.0.1" - meow "^3.1.0" - pretty-error@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/pretty-error/-/pretty-error-3.0.4.tgz#94b1d54f76c1ed95b9c604b9de2194838e5b574e" @@ -9023,14 +8939,6 @@ process-nextick-args@^2.0.0, process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== -progress-stream@^1.1.0: - version "1.2.0" - resolved "https://registry.npmjs.org/progress-stream/-/progress-stream-1.2.0.tgz#2cd3cfea33ba3a89c9c121ec3347abe9ab125f77" - integrity sha1-LNPP6jO6OonJwSHsM0er6asSX3c= - dependencies: - speedometer "~0.1.2" - through2 "~0.2.3" - progress@^2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" @@ -9231,7 +9139,7 @@ raw-body@~1.1.0: bytes "1" string_decoder "0.10" -rc@^1.0.1, rc@^1.1.6, rc@^1.2.1, rc@^1.2.8: +rc@^1.0.1, rc@^1.1.6, rc@^1.2.8: version "1.2.8" resolved "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== @@ -9431,14 +9339,6 @@ rechoir@^0.7.0: dependencies: resolve "^1.9.0" -redent@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz#cf916ab1fd5f1f16dfb20822dd6ec7f730c2afde" - integrity sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94= - dependencies: - indent-string "^2.1.0" - strip-indent "^1.0.1" - regenerate-unicode-properties@^8.2.0: version "8.2.0" resolved "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-8.2.0.tgz#e5de7111d655e7ba60c057dbe9ff37c87e65cdec" @@ -9572,13 +9472,6 @@ repeat-string@^1.5.4, repeat-string@^1.6.1: resolved "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= -repeating@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz#5214c53a926d3552707527fbab415dbc08d06dda" - integrity sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo= - dependencies: - is-finite "^1.0.0" - replace-ext@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-0.0.1.tgz#29bbd92078a739f0bcce2b4ee41e837953522924" @@ -9598,7 +9491,7 @@ replace-homedir@^1.0.0: is-absolute "^1.0.0" remove-trailing-separator "^1.1.0" -request@^2.45.0, request@^2.88.0: +request@^2.88.0: version "2.88.2" resolved "https://registry.npmjs.org/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -10047,13 +9940,6 @@ signal-exit@^3.0.3: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.5.tgz#9e3e8cc0c75a99472b44321033a7702e7738252f" integrity sha512-KWcOiKeQj6ZyXx7zq4YxSMgHRlod4czeBQZrPb8OKcohcqAXShm7E20kEMle9WBt26hFcAf0qLOcp5zmY7kOqQ== -single-line-log@^1.1.2: - version "1.1.2" - resolved "https://registry.npmjs.org/single-line-log/-/single-line-log-1.1.2.tgz#c2f83f273a3e1a16edb0995661da0ed5ef033364" - integrity sha1-wvg/Jzo+GhbtsJlWYdoO1e8DM2Q= - dependencies: - string-width "^1.0.1" - sisteransi@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" @@ -10230,11 +10116,6 @@ spdx-license-ids@^3.0.0: resolved "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.7.tgz#e9c18a410e5ed7e12442a549fbd8afa767038d65" integrity sha512-U+MTEOO0AiDzxwFvoa4JVnMV6mZlJKk2sBLt90s7G0Gd0Mlknc7kxEn3nuDPNZRta7O2uy8oLcZLVT+4sqNZHQ== -speedometer@~0.1.2: - version "0.1.4" - resolved "https://registry.npmjs.org/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" - integrity sha1-mHbb0qFp0xFUAtSObqYynIgWpQ0= - split-on-first@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz#f610afeee3b12bce1d0c30425e76398b78249a5f" @@ -10515,13 +10396,6 @@ strip-final-newline@^2.0.0: resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== -strip-indent@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz#0c7962a6adefa7bbd4ac366460a638552ae1a0a2" - integrity sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI= - dependencies: - get-stdin "^4.0.1" - strip-json-comments@1.0.x: version "1.0.4" resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-1.0.4.tgz#1e15fbcac97d3ee99bf2d73b4c656b082bbafb91" @@ -10532,13 +10406,6 @@ strip-json-comments@~2.0.1: resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= -sumchecker@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/sumchecker/-/sumchecker-2.0.2.tgz#0f42c10e5d05da5d42eea3e56c3399a37d6c5b3e" - integrity sha1-D0LBDl0F2l1C7qPlbDOZo31sWz4= - dependencies: - debug "^2.2.0" - sumchecker@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/sumchecker/-/sumchecker-3.0.1.tgz#6377e996795abb0b6d348e9b3e1dfb24345a8e42" @@ -10708,11 +10575,6 @@ throat@^6.0.1: resolved "https://registry.yarnpkg.com/throat/-/throat-6.0.1.tgz#d514fedad95740c12c2d7fc70ea863eb51ade375" integrity sha512-8hmiGIJMDlwjg7dlJ4yKGLK8EsYqKgPWbG3b4wjJddKNwc7N7Dpn08Df4szr/sZdMVeOstrdYSsqzX6BYbcB+w== -throttleit@0.0.2: - version "0.0.2" - resolved "https://registry.npmjs.org/throttleit/-/throttleit-0.0.2.tgz#cfedf88e60c00dd9697b61fdd2a8343a9b680eaf" - integrity sha1-z+34jmDADdlpe2H90qg0OptoDq8= - through2-filter@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/through2-filter/-/through2-filter-3.0.0.tgz#700e786df2367c2c88cd8aa5be4cf9c1e7831254" @@ -10729,14 +10591,6 @@ through2@^2.0.0, through2@^2.0.3, through2@~2.0.0: readable-stream "~2.3.6" xtend "~4.0.1" -through2@~0.2.3: - version "0.2.3" - resolved "https://registry.npmjs.org/through2/-/through2-0.2.3.tgz#eb3284da4ea311b6cc8ace3653748a52abf25a3f" - integrity sha1-6zKE2k6jEbbMis42U3SKUqvyWj8= - dependencies: - readable-stream "~1.1.9" - xtend "~2.1.1" - "through@>=2.2.7 <3", through@~2.3: version "2.3.8" resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" @@ -10897,11 +10751,6 @@ tr46@^2.1.0: dependencies: punycode "^2.1.1" -trim-newlines@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613" - integrity sha1-WIeWa7WCpFA6QetST301ARgVphM= - truncate-utf8-bytes@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz#405923909592d56f78a5818434b0b78489ca5f2b" @@ -11760,13 +11609,6 @@ xtend@^4.0.0, xtend@~4.0.0, xtend@~4.0.1: resolved "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== -xtend@~2.1.1: - version "2.1.2" - resolved "https://registry.npmjs.org/xtend/-/xtend-2.1.2.tgz#6efecc2a4dad8e6962c4901b337ce7ba87b5d28b" - integrity sha1-bv7MKk2tjmlixJAbM3znuoe10os= - dependencies: - object-keys "~0.4.0" - y18n@^3.2.1: version "3.2.1" resolved "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" From 7f4d37494e593a124f1974939ad64f74d7b1ee21 Mon Sep 17 00:00:00 2001 From: LiHS Date: Mon, 25 Jul 2022 14:38:18 +0800 Subject: [PATCH 2/8] blink after upgrade electron --- package.json | 2 +- src/renderer/main/files/transfer/uploads.html | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index e993a4c6..84b1f3ba 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kodo-browser", - "version": "1.0.18", + "version": "1.0.19-alpha", "license": "Apache-2.0", "author": { "name": "Rong Zhou", diff --git a/src/renderer/main/files/transfer/uploads.html b/src/renderer/main/files/transfer/uploads.html index 655a0914..501b1011 100755 --- a/src/renderer/main/files/transfer/uploads.html +++ b/src/renderer/main/files/transfer/uploads.html @@ -69,7 +69,7 @@
-
    +
    • @@ -110,23 +110,25 @@
      + tooltip-placement="left" tooltip-append-to-body="true"> + tooltip-placement="left" tooltip-append-to-body="true"> + class="btn btn-success btn-xs" uib-tooltip="{{'upload.job.retry'|translate}}" + tooltip-placement="left" tooltip-append-to-body="true"> + class="btn btn-warning btn-xs" uib-tooltip="{{'upload.job.retry.overwrite'|translate}}" + tooltip-placement="left" tooltip-append-to-body="true">
      From 789b74b2578f72c819b6a4651aaa248b4e4b0186 Mon Sep 17 00:00:00 2001 From: LiHS Date: Mon, 25 Jul 2022 14:44:59 +0800 Subject: [PATCH 3/8] more comments and fix some code styles about upload --- src/common/ipc-actions/upload.ts | 16 +++--- src/common/models/job/upload-job.ts | 50 +++++++++++-------- src/main/uploader/index.ts | 2 +- src/main/util/createClient.ts | 8 +-- .../components/services/ipc-upload-manager.ts | 3 +- src/renderer/main/files/transfer/frame.js | 19 +++++-- src/renderer/main/files/transfer/uploads.js | 5 +- 7 files changed, 59 insertions(+), 44 deletions(-) diff --git a/src/common/ipc-actions/upload.ts b/src/common/ipc-actions/upload.ts index e2292dac..ef2e8c04 100644 --- a/src/common/ipc-actions/upload.ts +++ b/src/common/ipc-actions/upload.ts @@ -200,56 +200,56 @@ export class UploadActionFns { ) { } - updateConfig(data: UpdateConfigMessage['data']) { + updateConfig(data: UpdateConfigMessage["data"]) { this.ipc.send(this.channel, { action: UploadAction.UpdateConfig, data, }); } - loadPersistJobs(data: LoadPersistJobsMessage['data']) { + loadPersistJobs(data: LoadPersistJobsMessage["data"]) { this.ipc.send(this.channel, { action: UploadAction.LoadPersistJobs, data, }); } - addJobs(data: AddJobsMessage['data']) { + addJobs(data: AddJobsMessage["data"]) { this.ipc.send(this.channel, { action: UploadAction.AddJobs, data, }); } - updateUiData(data: UpdateUiDataMessage['data']) { + updateUiData(data: UpdateUiDataMessage["data"]) { this.ipc.send(this.channel, { action: UploadAction.UpdateUiData, data, }); } - waitJob(data: WaitJobMessage['data']) { + waitJob(data: WaitJobMessage["data"]) { this.ipc.send(this.channel, { action: UploadAction.WaitJob, data, }); } - startJob(data: StartJobMessage['data']) { + startJob(data: StartJobMessage["data"]) { this.ipc.send(this.channel, { action: UploadAction.StartJob, data, }); } - stopJob(data: StopJobMessage['data']) { + stopJob(data: StopJobMessage["data"]) { this.ipc.send(this.channel, { action: UploadAction.StopJob, data, }); } - removeJob(data: RemoveJobMessage['data']) { + removeJob(data: RemoveJobMessage["data"]) { this.ipc.send(this.channel, { action: UploadAction.RemoveJob, data, diff --git a/src/common/models/job/upload-job.ts b/src/common/models/job/upload-job.ts index 4f9e1319..18494ba0 100644 --- a/src/common/models/job/upload-job.ts +++ b/src/common/models/job/upload-job.ts @@ -1,4 +1,4 @@ -import {promises as fsPromises} from 'fs'; +import {promises as fsPromises} from "fs"; // @ts-ignore import mime from "mime"; @@ -65,7 +65,7 @@ interface OptionalOptions extends UploadOptions{ export type Options = RequiredOptions & Partial const DEFAULT_OPTIONS: OptionalOptions = { - id: '', + id: "", multipartUploadThreshold: 10 * ByteSize.MB, multipartUploadSize: 4 * ByteSize.MB, @@ -87,24 +87,27 @@ const DEFAULT_OPTIONS: OptionalOptions = { }; type PersistInfo = { - from: RequiredOptions['from'], - storageClasses: RequiredOptions['storageClasses'], - region: RequiredOptions['region'], - to: RequiredOptions['to'], - overwrite: RequiredOptions['overwrite'], - storageClassName: RequiredOptions['storageClassName'], - // if we can remove backendMode? - // because we always use the client current backendMode. - backendMode: RequiredOptions['clientOptions']['backendMode'], - prog: OptionalOptions['prog'], - status: OptionalOptions['status'], - message: OptionalOptions['message'], - uploadedId: OptionalOptions['uploadedId'], - // ugly. if can do some break changes, make it be - // `uploadedParts: OptionalOptions['uploadedParts'],` + from: RequiredOptions["from"], + storageClasses: RequiredOptions["storageClasses"], + region: RequiredOptions["region"], + to: RequiredOptions["to"], + overwrite: RequiredOptions["overwrite"], + storageClassName: RequiredOptions["storageClassName"], + // Q: if we can remove backendMode? + // Be client backendMode. + // A: It seems no problems, because backendMode is invariant in business logic. + // But It's also a risk, if business logic changes. Because we may get errors + // when restart job from break point with different backendMode. + backendMode: RequiredOptions["clientOptions"]["backendMode"], + prog: OptionalOptions["prog"], + status: OptionalOptions["status"], + message: OptionalOptions["message"], + uploadedId: OptionalOptions["uploadedId"], + // ugly. if we can do some break changes, make it be + // `uploadedParts: OptionalOptions["uploadedParts"],` uploadedParts: { - PartNumber: UploadedPart['partNumber'], - ETag: UploadedPart['etag'], + PartNumber: UploadedPart["partNumber"], + ETag: UploadedPart["etag"], }[], multipartUploadThreshold: OptionalOptions["multipartUploadThreshold"], multipartUploadSize: OptionalOptions["multipartUploadSize"], @@ -114,7 +117,7 @@ export default class UploadJob extends Base { static fromPersistInfo( id: string, persistInfo: PersistInfo, - clientOptions: RequiredOptions['clientOptions'], + clientOptions: RequiredOptions["clientOptions"], uploadOptions: { uploadSpeedLimit: number, isDebug: boolean, @@ -292,7 +295,7 @@ export default class UploadJob extends Base { // upload this.startSpeedCounter(); await qiniuClient.enter( - 'uploadFile', + "uploadFile", this.startUpload, { targetBucket: this.options.to.bucket, @@ -310,6 +313,8 @@ export default class UploadJob extends Base { private async startUpload(client: Adapter) { client.storageClasses = this.options.storageClasses; + + // check overwrite const isOverwrite = this.isForceOverwrite || this.options.overwrite; if (!isOverwrite) { const isExists = await client.isExists( @@ -325,8 +330,9 @@ export default class UploadJob extends Base { } } + // upload this.uploader = new Uploader(client); - const fileHandle = await fsPromises.open(this.options.from.path, 'r'); + const fileHandle = await fsPromises.open(this.options.from.path, "r"); await this.uploader.putObjectFromFile( this.options.region, { diff --git a/src/main/uploader/index.ts b/src/main/uploader/index.ts index 7c9ea218..e7a4e9f5 100644 --- a/src/main/uploader/index.ts +++ b/src/main/uploader/index.ts @@ -144,7 +144,7 @@ process.on("exit", () => { handleExit() }); -process.on('SIGTERM', () => { +process.on("SIGTERM", () => { handleExit() .then(() => { process.exit(0); diff --git a/src/main/util/createClient.ts b/src/main/util/createClient.ts index c10c23a3..9535a7a2 100644 --- a/src/main/util/createClient.ts +++ b/src/main/util/createClient.ts @@ -22,7 +22,7 @@ export default function createQiniuClient( clientOptions.regions, ); const modeOptions: ModeOptions = { - appName: 'kodo-browser/ioutil', + appName: "kodo-browser/ioutil", appVersion: AppConfig.app.version, appNatureLanguage: options.userNatureLanguage, // disable uplog when use customize cloud @@ -48,7 +48,7 @@ function debugRequest(mode: BackendMode) { method = request.method; headers = request.headers; } - console.info('>>', mode, 'REQ_URL:', url, 'REQ_METHOD:', method, 'REQ_HEADERS:', headers); + console.info(">>", mode, "REQ_URL:", url, "REQ_METHOD:", method, "REQ_HEADERS:", headers); }; } @@ -68,7 +68,7 @@ function debugResponse(mode: BackendMode) { requestHeaders = response.request.headers; } } - console.info('<<', mode, 'REQ_URL:', requestUrl, 'REQ_METHOD:', requestMethod, 'REQ_HEADERS: ', requestHeaders, - 'RESP_STATUS:', responseStatusCode, 'RESP_HEADERS:', responseHeaders, 'RESP_INTERVAL:', responseInterval, 'ms RESP_DATA:', responseData, 'RESP_ERROR:', responseError); + console.info("<<", mode, "REQ_URL:", requestUrl, "REQ_METHOD:", requestMethod, "REQ_HEADERS: ", requestHeaders, + "RESP_STATUS:", responseStatusCode, "RESP_HEADERS:", responseHeaders, "RESP_INTERVAL:", responseInterval, "ms RESP_DATA:", responseData, "RESP_ERROR:", responseError); }; } diff --git a/src/renderer/components/services/ipc-upload-manager.ts b/src/renderer/components/services/ipc-upload-manager.ts index 90a96172..d66cb473 100644 --- a/src/renderer/components/services/ipc-upload-manager.ts +++ b/src/renderer/components/services/ipc-upload-manager.ts @@ -1,6 +1,7 @@ -import {UploadActionFns} from "@common/ipc-actions/upload"; import {ipcRenderer} from "electron"; +import {UploadActionFns} from "@common/ipc-actions/upload"; + const ipcUploadManager = new UploadActionFns(ipcRenderer, "UploaderManager"); export default ipcUploadManager; diff --git a/src/renderer/main/files/transfer/frame.js b/src/renderer/main/files/transfer/frame.js index c9912b0c..d8c83667 100755 --- a/src/renderer/main/files/transfer/frame.js +++ b/src/renderer/main/files/transfer/frame.js @@ -73,6 +73,8 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ totalStat: { running: 0, total: 0, + + // upload up: 0, upRunning: 0, upDone: 0, @@ -162,7 +164,9 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ maxConcurrency: Settings.maxUploadConcurrency, multipartUploadSize: Settings.multipartUploadSize * ByteSize.MB, multipartUploadThreshold: Settings.multipartUploadThreshold * ByteSize.MB, - uploadSpeedLimit: Settings.uploadSpeedLimitEnabled * ByteSize.KB, + uploadSpeedLimit: Settings.uploadSpeedLimitEnabled === 0 + ? 0 + : Settings.uploadSpeedLimitKBperSec * ByteSize.KB, isDebug: Settings.isDebug !== 0, isSkipEmptyDirectory: $scope.emptyFolderUploading.enabled, persistPath: getProgFilePath(), @@ -211,9 +215,16 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ /** * upload - * @param filePaths [] {array}, iter for folder - * @param bucketInfo {object} {bucket, region, key} - * @param uploadOptions {object} {isOverwrite, storageClassName}, storageClassName is fetched from server + * @param {string[]} filePaths iter for folder + * @param {object} bucketInfo + * @param {string} bucketInfo.bucketName + * @param {string} bucketInfo.regionId + * @param {string} bucketInfo.key + * @param {string} bucketInfo.qiniuBackendMode + * @param {object[]} bucketInfo.availableStorageClasses + * @param {object} uploadOptions + * @param {string} uploadOptions.isOverwrite + * @param {string} uploadOptions.storageClassName storageClassName is fetched from server */ function uploadFilesHandler(filePaths, bucketInfo,uploadOptions) { Toast.info(T("upload.addtolist.on")); diff --git a/src/renderer/main/files/transfer/uploads.js b/src/renderer/main/files/transfer/uploads.js index 031cc7a2..d5857f8c 100755 --- a/src/renderer/main/files/transfer/uploads.js +++ b/src/renderer/main/files/transfer/uploads.js @@ -3,7 +3,6 @@ import angular from "angular" import webModule from '@/app-module/web' import ipcUploadManager from "@/components/services/ipc-upload-manager" import jobUtil from '@/components/services/job-util' -import DelayDone from '@/components/services/delay-done' import { TOAST_FACTORY_NAME as Toast } from '@/components/directives/toast-list' import { EMPTY_FOLDER_UPLOADING, @@ -17,7 +16,6 @@ webModule.controller(TRANSFER_UPLOAD_CONTROLLER_NAME, [ "$timeout", "$translate", jobUtil, - DelayDone, Toast, Dialog, function ( @@ -25,7 +23,6 @@ webModule.controller(TRANSFER_UPLOAD_CONTROLLER_NAME, [ $timeout, $translate, jobUtil, - DelayDone, Toast, Dialog ) { @@ -131,7 +128,7 @@ webModule.controller(TRANSFER_UPLOAD_CONTROLLER_NAME, [ Toast.info(T("pause.on")); //'正在暂停...' $scope.allActionBtnDisabled = true; - ipcUploadManager.stopAllJobs() + ipcUploadManager.stopAllJobs(); Toast.info(T("pause.success")); From 015a4a09422d02d0f6c77eb8a22a59c09963b361 Mon Sep 17 00:00:00 2001 From: LiHS Date: Mon, 25 Jul 2022 15:22:48 +0800 Subject: [PATCH 4/8] refactoring transfer module --- src/common/const/qiniu.ts | 4 - src/common/ipc-actions/common.ts | 0 src/common/ipc-actions/download.ts | 290 +++++++ src/common/ipc-actions/upload.ts | 48 +- src/common/models/job/_mock-helpers_/auth.ts | 4 + src/common/models/job/_mock-helpers_/data.ts | 116 --- src/common/models/job/base.test.ts | 29 - src/common/models/job/base.ts | 32 - src/common/models/job/download-job.test.ts | 244 +++--- src/common/models/job/download-job.ts | 518 ++++++------ src/common/models/job/transfer-job.ts | 166 ++++ src/common/models/job/types.ts | 89 -- src/common/models/job/upload-job.test.ts | 125 +-- src/common/models/job/upload-job.ts | 233 ++---- src/common/models/job/utils.test.ts | 32 +- src/common/models/job/utils.ts | 8 +- src/common/qiniu-store/lib/consts.js | 5 - src/common/qiniu-store/lib/ioutil.js | 419 ---------- .../qiniu/create-client.ts} | 3 +- src/common/qiniu/index.ts | 2 + src/common/qiniu/types.ts | 16 + src/main/download-worker.js | 95 --- src/main/download-worker.ts | 164 ++++ src/main/index.js | 763 ------------------ src/main/index.ts | 461 +++++++++++ .../boundary-const.ts | 0 .../transfer-managers/download-manager.ts | 291 +++++++ src/main/transfer-managers/remote-walker.ts | 19 + .../transfer-managers/transfer-manager.ts | 285 +++++++ .../upload-manager.ts | 321 +------- .../{uploader/index.ts => upload-worker.ts} | 16 +- .../components/services/delay-done.js | 63 -- .../components/services/download-manager.js | 472 ----------- src/renderer/components/services/index.js | 2 - .../services/ipc-download-manager.ts | 7 + src/renderer/components/services/settings.ts | 42 +- src/renderer/main/files/_/file-list.html | 2 +- src/renderer/main/files/files.js | 4 +- .../main/files/transfer/downloads.html | 37 +- src/renderer/main/files/transfer/downloads.js | 167 +--- src/renderer/main/files/transfer/frame.html | 4 +- src/renderer/main/files/transfer/frame.js | 224 +++-- webpack/paths.js | 6 +- webpack/webpack-main.config.js | 4 +- 44 files changed, 2594 insertions(+), 3238 deletions(-) delete mode 100644 src/common/const/qiniu.ts create mode 100644 src/common/ipc-actions/common.ts create mode 100644 src/common/ipc-actions/download.ts create mode 100644 src/common/models/job/_mock-helpers_/auth.ts delete mode 100644 src/common/models/job/_mock-helpers_/data.ts delete mode 100644 src/common/models/job/base.test.ts delete mode 100644 src/common/models/job/base.ts create mode 100644 src/common/models/job/transfer-job.ts delete mode 100644 src/common/qiniu-store/lib/consts.js delete mode 100644 src/common/qiniu-store/lib/ioutil.js rename src/{main/util/createClient.ts => common/qiniu/create-client.ts} (96%) create mode 100644 src/common/qiniu/index.ts create mode 100644 src/common/qiniu/types.ts delete mode 100644 src/main/download-worker.js create mode 100644 src/main/download-worker.ts delete mode 100755 src/main/index.js create mode 100755 src/main/index.ts rename src/main/{uploader => transfer-managers}/boundary-const.ts (100%) create mode 100644 src/main/transfer-managers/download-manager.ts create mode 100644 src/main/transfer-managers/remote-walker.ts create mode 100644 src/main/transfer-managers/transfer-manager.ts rename src/main/{uploader => transfer-managers}/upload-manager.ts (53%) rename src/main/{uploader/index.ts => upload-worker.ts} (96%) delete mode 100755 src/renderer/components/services/delay-done.js delete mode 100755 src/renderer/components/services/download-manager.js create mode 100644 src/renderer/components/services/ipc-download-manager.ts diff --git a/src/common/const/qiniu.ts b/src/common/const/qiniu.ts deleted file mode 100644 index 86d713a1..00000000 --- a/src/common/const/qiniu.ts +++ /dev/null @@ -1,4 +0,0 @@ -export enum BackendMode { - Kodo = "kodo", - S3 = "s3", -} diff --git a/src/common/ipc-actions/common.ts b/src/common/ipc-actions/common.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/common/ipc-actions/download.ts b/src/common/ipc-actions/download.ts new file mode 100644 index 00000000..cbb898a6 --- /dev/null +++ b/src/common/ipc-actions/download.ts @@ -0,0 +1,290 @@ +import {IpcRenderer} from "electron"; +import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; +import {Domain} from "kodo-s3-adapter-sdk/dist/adapter"; + +import {ClientOptions} from "@common/qiniu"; +import {Status} from "@common/models/job/types"; +import DownloadJob from "@common/models/job/download-job"; +import StorageClass from "@common/models/storage-class"; + +export interface RemoteObject { + name: string, + region: string, + bucket: string, + key: string, + size: number, + mtime: number, + isDirectory: boolean, + isFile: boolean, +} + +// +// export interface FromInfo { +// regionId: string, +// bucketName: string, +// objectList: RemoteObject[], +// } + +export interface DownloadOptions { + region: string, + bucket: string, + + domain?: Domain, + + isOverwrite: boolean, + storageClasses: StorageClass[], + userNatureLanguage: NatureLanguage +} + +export enum DownloadAction { + UpdateConfig = "UpdateConfig", + LoadPersistJobs = "LoadPersistJobs", + AddJobs = "AddJobs", + StopJob = "StopJob", + WaitJob = "WaitJob", + StartJob = "StartJob", + RemoveJob = "RemoveJob", + CleanupJobs = "CleanupJobs", + StartAllJobs = "StartAllJobs", + StopAllJobs = "StopAllJobs", + RemoveAllJobs = "RemoveAllJobs", + + // common + UpdateUiData = "UpdateUiData", + + // reply only + AddedJobs = "AddedJobs", + JobCompleted = "JobCompleted", +} + +export interface UpdateConfigMessage { + action: DownloadAction.UpdateConfig, + data: Partial<{ + resumable: boolean, + maxConcurrency: number, + multipartSize: number, + multipartThreshold: number, + speedLimit: number, + isDebug: boolean, + persistPath: string, + + isOverwrite: boolean, + }> +} + +export interface LoadPersistJobsMessage { + action: DownloadAction.LoadPersistJobs, + data: { + clientOptions: Pick + downloadOptions: Pick, + } +} + +export interface AddJobsMessage { + action: DownloadAction.AddJobs, + data: { + remoteObjects: RemoteObject[], + destPath: string, + downloadOptions: DownloadOptions, + clientOptions: ClientOptions, + } +} + +export interface AddedJobsReplyMessage { + action: DownloadAction.AddedJobs, + data: { + remoteObjects: RemoteObject[], + destPath: string, + }, +} + +export interface UpdateUiDataMessage { + action: DownloadAction.UpdateUiData, + data: { + pageNum: number, + count: number, + query?: { + status?: Status, + name?: string, + }, + }, +} + +export interface UpdateUiDataReplyMessage { + action: DownloadAction.UpdateUiData, + data: { + list: (DownloadJob["uiData"] | undefined)[], + total: number, + finished: number, + running: number, + failed: number, + stopped: number, + }, +} + +export interface StopJobMessage { + action: DownloadAction.StopJob, + data: { + jobId: string, + }, +} + +export interface WaitJobMessage { + action: DownloadAction.WaitJob, + data: { + jobId: string, + }, +} + +export interface StartJobMessage { + action: DownloadAction.StartJob, + data: { + jobId: string, + forceOverwrite?: boolean, + }, +} + +export interface RemoveJobMessage { + action: DownloadAction.RemoveJob, + data: { + jobId: string, + }, +} + +export interface CleanupJobMessage { + action: DownloadAction.CleanupJobs, + data?: {}, +} + + +export interface StartAllJobsMessage { + action: DownloadAction.StartAllJobs, + data?: {}, +} + +export interface StopAllJobsMessage { + action: DownloadAction.StopAllJobs, + data?: {}, +} + +export interface RemoveAllJobsMessage { + action: DownloadAction.RemoveAllJobs, + data?: {}, +} + +export interface JobCompletedReplyMessage { + action: DownloadAction.JobCompleted, + data: { + jobsId: string, + jobUiData: DownloadJob["uiData"], + } +} + +export type DownloadMessage = UpdateConfigMessage + | LoadPersistJobsMessage + | AddJobsMessage + | UpdateUiDataMessage + | StopJobMessage + | WaitJobMessage + | StartJobMessage + | RemoveJobMessage + | CleanupJobMessage + | StartAllJobsMessage + | StopAllJobsMessage + | RemoveAllJobsMessage + +export type DownloadReplyMessage = UpdateUiDataReplyMessage + | AddedJobsReplyMessage + | JobCompletedReplyMessage + +export class DownloadActionFns { + constructor( + private readonly ipc: IpcRenderer, + private readonly channel: string, + ) { + } + + updateConfig(data: UpdateConfigMessage["data"]) { + this.ipc.send(this.channel, { + action: DownloadAction.UpdateConfig, + data: data, + }); + } + + loadPersistJobs(data: LoadPersistJobsMessage["data"]) { + this.ipc.send(this.channel, { + action: DownloadAction.LoadPersistJobs, + data, + }) + } + + addJobs(data: AddJobsMessage["data"]) { + this.ipc.send(this.channel, { + action: DownloadAction.AddJobs, + data, + }); + } + + updateUiData(data: UpdateUiDataMessage["data"]) { + this.ipc.send(this.channel, { + action: DownloadAction.UpdateUiData, + data, + }); + } + + waitJob(data: WaitJobMessage["data"]) { + this.ipc.send(this.channel, { + action: DownloadAction.WaitJob, + data, + }); + } + + startJob(data: StartJobMessage["data"]) { + this.ipc.send(this.channel, { + action: DownloadAction.StartJob, + data, + }); + } + + stopJob(data: StopJobMessage["data"]) { + this.ipc.send(this.channel, { + action: DownloadAction.StopJob, + data, + }); + } + + removeJob(data: RemoveJobMessage["data"]) { + this.ipc.send(this.channel, { + action: DownloadAction.RemoveJob, + data, + }); + } + + cleanUpJobs() { + this.ipc.send(this.channel, { + action: DownloadAction.CleanupJobs, + data: {}, + }); + } + + startAllJobs() { + this.ipc.send(this.channel, { + action: DownloadAction.StartAllJobs, + data: {}, + }); + } + + stopAllJobs() { + this.ipc.send(this.channel, { + action: DownloadAction.StopAllJobs, + data: {}, + }); + } + + removeAllJobs() { + this.ipc.send(this.channel, { + action: DownloadAction.RemoveAllJobs, + data: {}, + }); + } +} diff --git a/src/common/ipc-actions/upload.ts b/src/common/ipc-actions/upload.ts index ef2e8c04..65c239b8 100644 --- a/src/common/ipc-actions/upload.ts +++ b/src/common/ipc-actions/upload.ts @@ -1,10 +1,10 @@ import {IpcRenderer} from "electron"; -import {Region} from "kodo-s3-adapter-sdk"; import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; -import {BackendMode} from "@common/const/qiniu"; -import {Status} from "@common/models/job/types"; + +import {ClientOptions} from "@common/qiniu"; import StorageClass from "@common/models/storage-class"; import {UploadJob} from "@common/models/job"; +import {Status} from "@common/models/job/types"; // some types maybe should in models export interface DestInfo { @@ -20,15 +20,6 @@ export interface UploadOptions { userNatureLanguage: NatureLanguage, } -// TODO: merge with `RequiredOptions['clientOptions']` in upload-job.ts -export interface ClientOptions { - accessKey: string, - secretKey: string, - ucUrl: string, - regions: Region[], - backendMode: BackendMode, -} - // action names export enum UploadAction { UpdateConfig = "UpdateConfig", @@ -56,11 +47,11 @@ export enum UploadAction { export interface UpdateConfigMessage { action: UploadAction.UpdateConfig, data: Partial<{ - resumeUpload: boolean, + resumable: boolean, maxConcurrency: number, - multipartUploadSize: number, // Bytes - multipartUploadThreshold: number, // Bytes - uploadSpeedLimit: number, // Bytes/s + multipartSize: number, // Bytes + multipartThreshold: number, // Bytes + speedLimit: number, // Bytes/s isDebug: boolean, isSkipEmptyDirectory: boolean, persistPath: string, @@ -85,6 +76,14 @@ export interface AddJobsMessage { }, } +export interface AddedJobsReplyMessage { + action: UploadAction.AddedJobs, + data: { + filePathnameList: string[], + destInfo: DestInfo, + }, +} + export interface UpdateUiDataMessage { action: UploadAction.UpdateUiData, data: { @@ -124,7 +123,9 @@ export interface StartJobMessage { action: UploadAction.StartJob, data: { jobId: string, - forceOverwrite?: boolean, + options?: { + forceOverwrite: boolean, + }, }, } @@ -155,14 +156,6 @@ export interface RemoveAllJobsMessage { data?: {}, } -export interface AddedJobsReplyMessage { - action: UploadAction.AddedJobs, - data: { - filePathnameList: string[], - destInfo: DestInfo, - }, -} - export interface JobCompletedReplyMessage { action: UploadAction.JobCompleted, data: { @@ -192,6 +185,11 @@ export type UploadMessage = UpdateConfigMessage | StopAllJobsMessage | RemoveAllJobsMessage +export type UploadReplyMessage = UpdateUiDataReplyMessage + | AddedJobsReplyMessage + | JobCompletedReplyMessage + | CreatedDirectoryReplyMessage + // send actions functions export class UploadActionFns { constructor( diff --git a/src/common/models/job/_mock-helpers_/auth.ts b/src/common/models/job/_mock-helpers_/auth.ts new file mode 100644 index 00000000..d0528b56 --- /dev/null +++ b/src/common/models/job/_mock-helpers_/auth.ts @@ -0,0 +1,4 @@ +// TODO: merge with src/renderer/components/services/qiniu-client/_mock-helpers_/auth.ts +export const QINIU_ACCESS_KEY = "NgKd0BmebvsFERFEBfKVVZGeGn7VsZQe_H_AunOC"; +export const QINIU_SECRET_KEY = "lp4Zv3Gi_7CHtxNTcJx2Pum5hUJB3gHROcg4vp0i"; +export const QINIU_UC = "http://fake.qiniu.com"; diff --git a/src/common/models/job/_mock-helpers_/data.ts b/src/common/models/job/_mock-helpers_/data.ts deleted file mode 100644 index e3ffcd54..00000000 --- a/src/common/models/job/_mock-helpers_/data.ts +++ /dev/null @@ -1,116 +0,0 @@ -export const uploadOptionsFromNewJob = { - "clientOptions": { - "accessKey": "NgKd0BmebvsFERFEBfKVVZGeGn7VsZQe_H_AunOC", - "secretKey": "lp4Zv3Gi_7CHtxNTcJx2Pum5hUJB3gHROcg4vp0i", - ucUrl: "", - "regions": [], - backendMode: "kodo" as const, - }, - // ↑ manual add ↓ from file - storageClasses: [], // break change from 1.0.16, older task will be uploaded with standard storage class - "region": "cn-east-1", - "to": { - "bucket": "kodo-browser-dev", - "key": "remote/path/to/out.gif" - }, - "from": { - "name": "out.gif", - "path": "/local/path/to/out.gif", - "size": 1024, - "mtime": 1651042948862, - }, - "overwrite": false, - storageClassName: "Standard" as const, - "resumeUpload": false, - "multipartUploadSize": 16, - "multipartUploadThreshold": 100, - uploadSpeedLimit: 0, // break change from 1.0.16, 0 means no limit. remove persist from 1.0.18 - "isDebug": false, - - userNatureLanguage: 'zh-CN' as const, // break change from 1.0.16 -} - -export const uploadOptionsFromResumeJob = { - "clientOptions": { - "accessKey": "NgKd0BmebvsFERFEBfKVVZGeGn7VsZQe_H_AunOC", - "secretKey": "lp4Zv3Gi_7CHtxNTcJx2Pum5hUJB3gHROcg4vp0i", - ucUrl: "", // break change from 1.0.16, "" means public - "regions": [], - backendMode: "kodo" as const, - }, - // ↑ manual add ↓ from file - storageClasses: [], // break change from 1.0.16, older task will be uploaded with standard storage class - "region": "cn-east-1", - "to": { - "bucket": "kodo-browser-dev", - "key": "remote/path/to/out.mp4" - }, - "from": { - "name": "out.mp4", - "path": "/local/path/to/out.mp4", - "size": 135515599, - "mtime": 1607594980239.0781 - }, - "prog": { - "total": 135515599, - "loaded": 17825792, - "resumable": false - }, - "message": "", - "uploadedId": "61a5f55fda9ed605fd263bc2region02z0", - "uploadedParts": [ - { - "partNumber": 1, - "etag": "lhBVs6yZLGUrM4XMCVts4yylKd-d" - }, - { - "partNumber": 2, - "etag": "lkYHX1O5MuEHazAy7TWuFTdriBP4" - }, - { - "partNumber": 3, - "etag": "lmsVRxCUz954oLhp_X3pQh8up9J1" - } - ], - "overwrite": false, - "storageClassName": "Standard", - "backendMode": "kodo", - "resumeUpload": false, - "multipartUploadSize": 16, - "multipartUploadThreshold": 100, - "uploadSpeedLimit": 0, - "isDebug": false, - - userNatureLanguage: 'zh-CN' as const, // break change from 1.0.16 -} - -export const downloadOptionsFromResumeJob = { - "clientOptions": { - "accessKey": "NgKd0BmebvsFERFEBfKVVZGeGn7VsZQe_H_AunOC", - "secretKey": "lp4Zv3Gi_7CHtxNTcJx2Pum5hUJB3gHROcg4vp0i", - ucUrl: "", // break change from 1.0.16, "" means public - "regions": [] - }, - // ↑ manual add ↓ from file - storageClasses: [], // break change from 1.0.16, older task will be toasted error but work - "region": "cn-east-1", - "to": { - "name": "out.mp4", - "path": "/local/path/to/out.mp4" - }, - "from": { - "bucket": "kodo-browser-dev", - "key": "remote/path/to/out.mp4", - "size": 135515599, - mtime: 1635403263903, - }, - "prog": { - loaded: 25177695, - "total": 135515599, - "resumable": true - }, - "backendMode": "s3" as const, // break change from 1.0.16, must as const - "message": "", - - userNatureLanguage: 'zh-CN' as const, // break change from 1.0.16 -} diff --git a/src/common/models/job/base.test.ts b/src/common/models/job/base.test.ts deleted file mode 100644 index e060f322..00000000 --- a/src/common/models/job/base.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import Base from "./base"; - -describe("test models/job/base.ts", () => { - it("on and emit", () => { - const mockedCallback = jest.fn(); - const base = new Base(); - base.on("boom", mockedCallback); - base.emit("boom"); - expect(mockedCallback).toBeCalled(); - }); - it("on and emit with payload", () => { - const mockedCallback = jest.fn(); - const base = new Base(); - base.on("boom", mockedCallback); - base.emit("boom", { name: "kodo-browser" }); - expect(mockedCallback).toBeCalledWith({ name: "kodo-browser" }); - }); - it("off", () => { - const mockedCallbackToOff = jest.fn(); - const mockedCallback = jest.fn(); - const base = new Base(); - base.on("boom", mockedCallbackToOff); - base.on("boom", mockedCallback); - base.off("boom", mockedCallbackToOff); - base.emit("boom", { name: "kodo-browser" }); - expect(mockedCallbackToOff).not.toBeCalled(); - expect(mockedCallback).toBeCalledWith({ name: "kodo-browser" }); - }); -}); diff --git a/src/common/models/job/base.ts b/src/common/models/job/base.ts deleted file mode 100644 index aaf27bfd..00000000 --- a/src/common/models/job/base.ts +++ /dev/null @@ -1,32 +0,0 @@ -export default class Base { - private _eventStack: Record boolean)[]> = {}; - - on(eventName: string, callbackFn: (...args: any[]) => boolean): this { - if (!this._eventStack[eventName]) { - this._eventStack[eventName] = []; - } - this._eventStack[eventName].push(callbackFn); - return this; - } - - off(eventName: string, callbackFn: (...args: any[]) => boolean): this { - if (this._eventStack[eventName]) { - this._eventStack[eventName] = this._eventStack[eventName].filter( - fn => fn !== callbackFn - ); - } - return this; - } - - emit(eventName: string, ...args: any[]) { - if (!this._eventStack[eventName]) { - return this; - } - for (const callbackFn of this._eventStack[eventName]) { - if (!callbackFn.apply(this, args)) { - break; - } - } - return this; - } -} diff --git a/src/common/models/job/download-job.test.ts b/src/common/models/job/download-job.test.ts index 653eabd5..d7315b7a 100644 --- a/src/common/models/job/download-job.test.ts +++ b/src/common/models/job/download-job.test.ts @@ -1,154 +1,128 @@ -jest.mock("electron", () => ({ - __esModule: true, - ipcRenderer: { - on: jest.fn(), - send: jest.fn(), - removeListener: jest.fn(), +import * as MockAuth from "./_mock-helpers_/auth"; +import { mocked } from "ts-jest/utils"; +import mockFs from "mock-fs"; +import fs from "fs"; + +jest.mock("kodo-s3-adapter-sdk", () => { + const mockedDownloader = jest.fn(); + mockedDownloader.constructor = mockedDownloader; + mockedDownloader.prototype.getObjectToFile = function (_region: string, _obj: Object, tempFilepath: string) { + return new Promise(resolve => { + fs.copyFileSync("/path/to/dir/to-be-downloaded.txt", tempFilepath); // mock downloaded + setTimeout(resolve, 300) + }); + }; + mockedDownloader.prototype.getObjectToFile = + jest.fn(mockedDownloader.prototype.getObjectToFile); + mockedDownloader.prototype.abort = jest.fn(); + + return { + __esModule: true, + ...jest.requireActual("kodo-s3-adapter-sdk"), + Downloader: mockedDownloader, } -})); +}); -import { ipcRenderer } from "electron"; -import * as AppConfig from "@common/const/app-config"; +import {Downloader} from "kodo-s3-adapter-sdk"; -import { EventKey, IpcJobEvent, Status } from "./types"; -import { downloadOptionsFromResumeJob } from "./_mock-helpers_/data"; +import {BackendMode} from "@common/qiniu"; +import {Status} from "@common/models/job/types"; import DownloadJob from "./download-job"; describe("test models/job/download-job.ts", () => { - describe("test stop", () => { - it("stop", () => { - const downloadJob = new DownloadJob(downloadOptionsFromResumeJob); - const spiedEmit = jest.spyOn(downloadJob, "emit"); - spiedEmit.mockImplementation((_eventName: string, ..._args: any[]) => downloadJob); - expect(downloadJob.stop()).toBe(downloadJob); - expect(downloadJob.speed).toBe(0); - expect(downloadJob.predictLeftTime).toBe(0); - expect(downloadJob.status).toBe(Status.Stopped); - expect(downloadJob.emit).toBeCalledWith("stop"); - expect(ipcRenderer.send).toBeCalledWith( - "asynchronous-job", - { - job: downloadJob.id, - key: IpcJobEvent.Stop, - }, - ); - expect(ipcRenderer.removeListener).toBeCalledWith( - downloadJob.id, - downloadJob.startDownload, - ); - }); - }); - - describe("test start", () => { - it("start()", () => { - const downloadJob = new DownloadJob(downloadOptionsFromResumeJob); - const spiedEmit = jest.spyOn(downloadJob, "emit"); - spiedEmit.mockImplementation((_eventName: string, ..._args: any[]) => downloadJob); - expect(downloadJob.start()).toBe(downloadJob); - expect(downloadJob.message).toBe(""); - - // private status flow - expect(downloadJob.emit).toBeCalledWith("statuschange", Status.Running); - expect(downloadJob.status).toBe(Status.Running); + const MockedDownloader = mocked(Downloader, true); - // ipcRenderer flow - expect(ipcRenderer.on).toBeCalledWith(downloadJob.id, downloadJob.startDownload); - expect(ipcRenderer.send).toBeCalledWith( - "asynchronous-job", - { - clientOptions: { - accessKey: "NgKd0BmebvsFERFEBfKVVZGeGn7VsZQe_H_AunOC", - backendMode: "s3", - regions: [], - secretKey: "lp4Zv3Gi_7CHtxNTcJx2Pum5hUJB3gHROcg4vp0i", - ucUrl: undefined, - userNatureLanguage: "zh-CN", - }, - job: downloadJob.id, - key: "job-download", - options: { - kodoBrowserVersion: AppConfig.app.version, - maxConcurrency: 10, - multipartDownloadSize: 8388608, - multipartDownloadThreshold: 104857600, - resumeDownload: false, - downloadSpeedLimit: 0, - }, - params: { - bucket: "kodo-browser-dev", - key: "remote/path/to/out.mp4", - localFile: "/local/path/to/out.mp4.download", - region: "cn-east-1", - isDebug: false, - }, + describe("test DownloadJob methods", () => { + beforeEach(() => { + mockFs({ + "/path/to/dir": { + "to-be-downloaded.txt": "some content", }, - ); - - // startSpeedCounter flow - expect(downloadJob.speed).toBe(0); - expect(downloadJob.predictLeftTime).toBe(0); - downloadJob.stop(); + }); }); - }); - - describe("test resume download job", () => { - it("getInfoForSave()", () => { - const downloadJob = new DownloadJob(downloadOptionsFromResumeJob); - - // stat - const fakeProgressTotal = 1024; - const fakeProgressResumable = true; - downloadJob.startDownload(null, { - key: EventKey.Stat, - data: { - progressTotal: fakeProgressTotal, - progressResumable: fakeProgressResumable, + afterEach(() => { + mockFs.restore(); + }); + it("test DownloadJob start", async () => { + const downloadJob = new DownloadJob({ + clientOptions: { + accessKey: MockAuth.QINIU_ACCESS_KEY, + secretKey: MockAuth.QINIU_SECRET_KEY, + ucUrl: MockAuth.QINIU_UC, + regions: [], + backendMode: BackendMode.Kodo, }, + from: { + bucket: "fake-bucket", + key: "path/to/fake-filename", + size: 1024, + mtime: 1658815528906, + }, + to: { + name: "fake-filename", + path: "/path/to/dir/fake-filename", + }, + overwrite: false, + region: "fake-region", + storageClasses: [], }); - expect(downloadJob.prog.total).toBe(fakeProgressTotal); - expect(downloadJob.prog.resumable).toBe(fakeProgressResumable); - // progress - const fakeProgressLoaded = 512; - downloadJob.startDownload(null, { - key: EventKey.Progress, - data: { - progressLoaded: fakeProgressLoaded, - progressResumable: fakeProgressResumable, - }, + await downloadJob.start(); + expect(MockedDownloader.prototype.getObjectToFile).toBeCalledTimes(1); + const [ + regionName, + objectInfo, + localFile, + ] = MockedDownloader.prototype.getObjectToFile.mock.calls[0]; + expect(regionName).toBe("fake-region"); + expect(objectInfo).toStrictEqual({ + bucket: "fake-bucket", + key: "path/to/fake-filename", }); - expect(downloadJob.prog.loaded).toBe(fakeProgressLoaded); - expect(downloadJob.prog.resumable).toBe(fakeProgressResumable); + expect(localFile).toBe("/path/to/dir/fake-filename.download"); + expect(downloadJob.message).toBe(""); + expect(downloadJob.status).toBe(Status.Finished); + }); + }); - // part downloaded - const lastDownloadedSize = downloadJob.prog.loaded; - const fakeDownloadedSize = 512; - downloadJob.startDownload(null, { - key: EventKey.PartDownloaded, - data: { - size: fakeDownloadedSize, + describe("test DownloadJob.getTempFilePath", () => { + beforeAll(() => { + mockFs({ + "/path/to": { + "exists-file": "some contents", + "exists-file.txt": "some contents", + "downloading-file.txt.download": "some contents", }, }); - expect(downloadJob.prog.loaded).toBe(lastDownloadedSize + fakeDownloadedSize); - - // info should in disk - expect(downloadJob.getInfoForSave()) - .toEqual({ - storageClasses: downloadOptionsFromResumeJob.storageClasses, - region: downloadOptionsFromResumeJob.region, - to: downloadOptionsFromResumeJob.to, - from: downloadOptionsFromResumeJob.from, - backendMode: downloadOptionsFromResumeJob.backendMode, - - prog: { - loaded: downloadJob.prog.loaded, - total: fakeProgressTotal, - resumable: fakeProgressResumable, - }, - status: Status.Waiting, - message: "", - }); }); - }) + afterAll(() => { + mockFs.restore(); + }); + it("file exists", async () => { + const filePath = "/path/to/exists-file.txt"; + const actual = await DownloadJob.getTempFilePath(filePath); + expect(actual).toBe("/path/to/exists-file.1.txt.download"); + }); + it("no file exists", async () => { + const filePath = "/path/to/non-exists-file.txt"; + const actual = await DownloadJob.getTempFilePath(filePath); + expect(actual).toBe("/path/to/non-exists-file.txt.download"); + }); + it("downloading file exists", async () => { + const filePath = "/path/to/downloading-file.txt"; + const actual = await DownloadJob.getTempFilePath(filePath); + expect(actual).toBe("/path/to/downloading-file.1.txt.download"); + }); + it("no file exists without ext", async () => { + const filePath = "/path/to/non-exists-file"; + const actual = await DownloadJob.getTempFilePath(filePath); + expect(actual).toBe("/path/to/non-exists-file.download"); + }); + it("file exists without ext", async () => { + const filePath = "/path/to/exists-file"; + const actual = await DownloadJob.getTempFilePath(filePath); + expect(actual).toBe("/path/to/exists-file.1.download"); + }); + }); }); diff --git a/src/common/models/job/download-job.ts b/src/common/models/job/download-job.ts index 14db6ecc..10df908e 100644 --- a/src/common/models/job/download-job.ts +++ b/src/common/models/job/download-job.ts @@ -1,60 +1,61 @@ -import fs from "fs"; -import { ipcRenderer } from "electron"; -import { Region } from "kodo-s3-adapter-sdk"; -import { StorageClass } from "kodo-s3-adapter-sdk/dist/adapter"; -import { NatureLanguage } from "kodo-s3-adapter-sdk/dist/uplog"; +import path from "path"; +import fs, {promises as fsPromises, constants as fsConstants} from "fs"; -import Duration from "@common/const/duration"; -import * as AppConfig from "@common/const/app-config"; +import lodash from "lodash"; +import {Downloader} from "kodo-s3-adapter-sdk"; +import {Adapter, Domain, ObjectHeader, StorageClass} from "kodo-s3-adapter-sdk/dist/adapter"; +import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; -import { BackendMode, EventKey, IpcDownloadJob, IpcJobEvent, Status } from "./types"; -import Base from "./base"; +import {ClientOptions, createQiniuClient} from "@common/qiniu"; + +import {Status} from "./types"; import * as Utils from "./utils"; +import TransferJob from "./transfer-job"; interface RequiredOptions { - clientOptions: { - accessKey: string, - secretKey: string, - ucUrl: string, - regions: Region[], - }, + clientOptions: ClientOptions, - from: Utils.RemotePath, + from: Required, to: Utils.LocalPath region: string, - backendMode: BackendMode, + overwrite: boolean, storageClasses: StorageClass[], +} + +interface DownloadOptions { + multipartDownloadThreshold: number, // Bytes + multipartDownloadSize: number, // Bytes + downloadSpeedLimit: number, // Bytes/s + + isDebug: boolean, + // could be removed if there is a better uplog userNatureLanguage: NatureLanguage, } -interface OptionalOptions { - domain?: string, - - maxConcurrency: number, - resumeDownload: boolean, - multipartDownloadThreshold: number, - multipartDownloadSize: number, - downloadSpeedLimit: number, +interface OptionalOptions extends DownloadOptions { + id: string, + domain?: Domain, status: Status, + message: string, prog: { total: number, loaded: number, - resumable: boolean, // what's difference from resumeDownload? + resumable?: boolean, // what's difference from resumeDownload? }, - message: string, - isDebug: boolean, + onStatusChange?: (status: Status) => void, + onProgress?: (prog: DownloadJob["prog"]) => void, } type Options = RequiredOptions & Partial const DEFAULT_OPTIONS: OptionalOptions = { - maxConcurrency: 10, - resumeDownload: false, + id: "", + multipartDownloadThreshold: 100, multipartDownloadSize: 8, downloadSpeedLimit: 0, // 0 means no limit @@ -64,164 +65,258 @@ const DEFAULT_OPTIONS: OptionalOptions = { prog: { total: 0, loaded: 0, - // synced: 0, resumable: false, }, message: "", isDebug: false, + + userNatureLanguage: "zh-CN", }; -export default class DownloadJob extends Base { - // - create options - - private readonly options: RequiredOptions & OptionalOptions +type PersistInfo = { + storageClasses: RequiredOptions["storageClasses"], + region: RequiredOptions["region"], + to: RequiredOptions["to"], + from: RequiredOptions["from"], + backendMode: RequiredOptions["clientOptions"]["backendMode"], + domain: OptionalOptions["domain"], + prog: OptionalOptions["prog"], + status: OptionalOptions["status"], + message: OptionalOptions["message"], + multipartDownloadThreshold: OptionalOptions["multipartDownloadThreshold"], + multipartDownloadSize: OptionalOptions["multipartDownloadSize"], +}; - // - for job save and log - - readonly id: string - readonly kodoBrowserVersion: string +export default class DownloadJob extends TransferJob { + static TempFileExt = ".download" + static fromPersistInfo( + id: string, + persistInfo: PersistInfo, + clientOptions: RequiredOptions["clientOptions"], + downloadOptions: { + downloadSpeedLimit: number, + overwrite: boolean, + isDebug: boolean, + userNatureLanguage: NatureLanguage, + } + ): DownloadJob { + return new DownloadJob({ + id, + status: persistInfo.status, + message: persistInfo.message, - // - for UI - - private __status: Status - // speed - speedTimerId?: number = undefined - speed: number = 0 - predictLeftTime: number = 0 - // message - message: string + from: persistInfo.from, + to: persistInfo.to, + prog: persistInfo.prog, - // - for resume from break point - - prog: OptionalOptions["prog"] + clientOptions, + storageClasses: persistInfo.storageClasses, - constructor(config: Options) { - super(); - this.id = `dj-${new Date().getTime()}-${Math.random().toString().substring(2)}`; - this.kodoBrowserVersion = AppConfig.app.version; + overwrite: downloadOptions.overwrite, + region: persistInfo.region, - this.options = { - ...DEFAULT_OPTIONS, - ...config, - }; + multipartDownloadThreshold: persistInfo.multipartDownloadThreshold, + multipartDownloadSize: persistInfo.multipartDownloadSize, + downloadSpeedLimit: downloadOptions.downloadSpeedLimit, + isDebug: downloadOptions.isDebug, + + userNatureLanguage: downloadOptions.userNatureLanguage, + }); + } + + // check duplicate and sanitize filename + static async getTempFilePath(filePath: string, isOverwrite = false) { + let result = `${filePath}${DownloadJob.TempFileExt}`; + + if (isOverwrite) { + return result; + } + + const fileExt = path.extname(filePath); + const filePathWithoutExt = fileExt.length > 0 ? filePath.slice(0, -fileExt.length) : filePath; + let duplicateSuffixNum = 0; + while (true) { + const isFileExists: boolean = await fsPromises.access( + result.slice(0, -DownloadJob.TempFileExt.length), + fsConstants.F_OK, + ) + .then(() => { return true }) + .catch(() => { return false }); + const isTempFileExists: boolean = await fsPromises.access(result, fsConstants.F_OK) + .then(() => { return true }) + .catch(() => { return false }); + if (!isFileExists && ! isTempFileExists) { + break; + } + duplicateSuffixNum += 1; + result = `${filePathWithoutExt}.${duplicateSuffixNum}${fileExt}${DownloadJob.TempFileExt}`; + } + return result; + } + + // - create options - + protected readonly options: Readonly - this.__status = this.options.status; + // - for process control - + tempFilePath: string + downloader?: Downloader + + constructor(config: Options) { + super(config); + + this.options = lodash.merge({}, DEFAULT_OPTIONS, config); this.prog = { ...this.options.prog, loaded: this.options.prog.loaded, }; + this.tempFilePath = `${this.options.to.path}${DownloadJob.TempFileExt}`; this.message = this.options.message; + // hook functions this.startDownload = this.startDownload.bind(this); + this.handleProgress = this.handleProgress.bind(this); + this.handleHeader = this.handleHeader.bind(this); + this.handlePartGet = this.handlePartGet.bind(this); } - private set _status(value: Status) { - this.__status = value; - this.emit("statuschange", this.status); - - if ( - this.status === Status.Failed - || this.status === Status.Stopped - || this.status === Status.Finished - ) { - clearInterval(this.speedTimerId); - this.speed = 0; - this.predictLeftTime = 0; + get uiData() { + return { + ...super.uiData, + to: this.options.to, } } - get status(): Status { - return this.__status - } - - get isStopped(): boolean { - return this.status !== Status.Running; - } - - private get tempfile(): string { - return `${this.options.to.path}.download`; - } - - private get ipcDownloadJob(): IpcDownloadJob { + get persistInfo(): PersistInfo { return { - job: this.id, - key: IpcJobEvent.Download, - clientOptions: { - ...this.options.clientOptions, - // if ucUrl is not undefined, downloader will use it generator url - ucUrl: this.options.clientOptions.ucUrl === '' - ? undefined - : this.options.clientOptions.ucUrl, - backendMode: this.options.backendMode, - - userNatureLanguage: this.options.userNatureLanguage, - }, - options: { - resumeDownload: this.options.resumeDownload, - maxConcurrency: this.options.maxConcurrency, - multipartDownloadThreshold: this.options.multipartDownloadThreshold * 1024 * 1024, - multipartDownloadSize: this.options.multipartDownloadSize * 1024 * 1024, - downloadSpeedLimit: this.options.downloadSpeedLimit, - kodoBrowserVersion: this.kodoBrowserVersion, - }, - params: { - region: this.options.region, - bucket: this.options.from.bucket, - key: this.options.from.key, - localFile: this.tempfile, - domain: this.options.domain, - isDebug: this.options.isDebug + // read-only info + region: this.options.region, + to: this.options.to, + from: this.options.from, + domain: this.options.domain, + storageClasses: this.options.storageClasses, + backendMode: this.options.clientOptions.backendMode, + + // real-time info + prog: { + loaded: this.prog.loaded, + total: this.prog.total, + resumable: this.prog.resumable }, - } + status: this.status, + message: this.message, + multipartDownloadThreshold: this.options.multipartDownloadThreshold, + multipartDownloadSize: this.options.multipartDownloadSize, + }; } - start(prog?: OptionalOptions["prog"]): this { + async start(): Promise { if (this.status === Status.Running || this.status === Status.Finished) { - return this; - } - - if (this.options.isDebug) { - console.log(`Try downloading kodo://${this.options.from.bucket}/${this.options.from.key} to ${this.options.to.path}`); + return; } this.message = ""; this._status = Status.Running; - const job = this.ipcDownloadJob - job.params.downloadedBytes = prog?.loaded; + this.startSpeedCounter(); if (this.options.isDebug) { - console.log(`[JOB] ${JSON.stringify(job)}`); + console.log(`Try downloading kodo://${this.options.from.bucket}/${this.options.from.key} to ${this.options.to.path}`); } - ipcRenderer.on(this.id, this.startDownload); - ipcRenderer.send("asynchronous-job", job); - this.startSpeedCounter(); + // create client + const qiniuClient = createQiniuClient(this.options.clientOptions, { + userNatureLanguage: this.options.userNatureLanguage, + isDebug: this.options.isDebug, + }); - return this; + await qiniuClient.enter( + "downloadFile", + this.startDownload, + { + targetBucket: this.options.from.bucket, + targetKey: this.options.from.key, + }, + ).catch(err => { + if (err === Downloader.userCanceledError) { + this._status = Status.Stopped; + return; + } + this._status = Status.Failed; + this.message = err.toString(); + }); + } + + private async startDownload(client: Adapter) { + client.storageClasses = this.options.storageClasses; + + // check overwrite + // the reason for overwrite when `this.prog.loaded > 0` is to allow + // download from breakpoint to work properly. + const isOverwrite = this.options.overwrite || this.prog.loaded > 0; + this.tempFilePath = await DownloadJob.getTempFilePath( + this.options.to.path, + isOverwrite, + ); + this.options.to.path = this.tempFilePath.slice(0, -DownloadJob.TempFileExt.length); + + // download + this.downloader = new Downloader(client); + await this.downloader.getObjectToFile( + this.options.region, + { + bucket: this.options.from.bucket, + key: this.options.from.key, + }, + this.tempFilePath, + this.options.domain, + { + recoveredFrom: this.prog.resumable + ? this.prog.loaded + : 0, + partSize: this.options.multipartDownloadSize, + chunkTimeout: 30000, + retriesOnSameOffset: 10, + downloadThrottleOption: this.options.downloadSpeedLimit + ? { + rate: this.options.downloadSpeedLimit, + } + : undefined, + getCallback: { + headerCallback: this.handleHeader, + partGetCallback: this.handlePartGet, + progressCallback: this.handleProgress, + }, + }, + ); + + this._status = Status.Verifying; + + // complete download + await fsPromises.rename( + this.tempFilePath, + this.options.to.path, + ); + this._status = Status.Finished; } stop(): this { if (this.status === Status.Stopped) { return this; } + this._status = Status.Stopped; if (this.options.isDebug) { - console.log(`Pausing kodo://${this.options.from.bucket}/${this.options.from.key}`); + console.log(`Pausing ${this.options.from.key}`); } - clearInterval(this.speedTimerId); - - this.speed = 0; - this.predictLeftTime = 0; - - this._status = Status.Stopped; - this.emit("stop"); - - ipcRenderer.send("asynchronous-job", { - job: this.id, - key: IpcJobEvent.Stop, - }); - ipcRenderer.removeListener(this.id, this.startDownload); + if (!this.downloader) { + return this; + } + this.downloader.abort(); + this.downloader = undefined; return this; } @@ -230,139 +325,60 @@ export default class DownloadJob extends Base { if (this.status === Status.Waiting) { return this; } + this._status = Status.Waiting; + if (this.options.isDebug) { - console.log(`Pending kodo://${this.options.from.bucket}/${this.options.from.key}`); + console.log(`Pending ${this.options.from.key}`); } - this._status = Status.Waiting; - this.emit("pause"); + if (!this.downloader) { + return this; + } + this.downloader.abort(); + this.downloader = undefined; return this; } - startDownload(_event: any, data: any) { - if (this.options.isDebug) { - console.log("[IPC MAIN]", data); + tryCleanupDownloadFile(): this { + if ([Status.Finished, Status.Waiting].includes(this.status)) { + return this; } - - switch (data.key) { - case EventKey.Stat: - this.prog.total = data.data.progressTotal; - this.prog.resumable = data.data.progressResumable; - this.emit("progress", this.prog); - return - case EventKey.Progress: - this.prog.loaded = data.data.progressLoaded; - this.prog.resumable = data.data.progressResumable; - this.emit("progress", this.prog); - return; - case EventKey.PartDownloaded: - this.prog.loaded = this.prog.loaded + data.data.size; - this.emit("partcomplete", this.prog); - return - case EventKey.Downloaded: - ipcRenderer.removeListener(this.id, this.startDownload); - this._status = Status.Verifying; - fs.rename(this.tempfile, this.options.to.path, err => { - if (err) { - console.error(`rename file ${this.tempfile} to ${this.options.to.path} error:`, err); - - this._status = Status.Failed; - this.emit("error", err); - } else { - this._status = Status.Finished; - this.emit("complete"); - } - }); - return; - case EventKey.Error: - console.warn("download object error:", data); - ipcRenderer.removeListener(this.id, this.startDownload); - - this.message = data; - this._status = Status.Failed; - this.emit("error", data.error); - return; - case EventKey.Debug: - if (!this.options.isDebug) { - console.log("Debug", data); - } - return; - default: - console.log("Unknown", data); - return; + try { + fs.unlinkSync(this.tempFilePath); + } catch (_err) { + // ignore error } + return this; } - startSpeedCounter() { - const startAt = new Date().getTime(); - - let lastLoaded = this.prog.loaded; - let lastSpeed = 0; - - clearInterval(this.speedTimerId); - const intervalDuration = Duration.Second; - this.speedTimerId = setInterval(() => { - if (this.isStopped) { - this.speed = 0; - this.predictLeftTime = 0; - return; - } - - let avgSpeed = this.prog.loaded / (startAt - new Date().getTime()) * Duration.Second; - this.speed = this.prog.loaded - lastLoaded; - if (this.speed <= 0 || (lastSpeed / this.speed) > 1.1) { - this.speed = lastSpeed * 0.95; - } - if (this.speed < avgSpeed) { - this.speed = avgSpeed; - } + protected handleStatusChange() { + this.options.onStatusChange?.(this.status); + } - lastSpeed = this.prog.loaded; - lastSpeed = this.speed; + private handleProgress(downloaded: number, total: number): void { + if (!this.downloader) { + return; + } + this.prog.loaded = downloaded; + this.prog.total = total; + this.options.onProgress?.(lodash.merge({}, this.prog)); - if (this.options.downloadSpeedLimit && this.speed > this.options.downloadSpeedLimit * 1024) { - this.speed = this.options.downloadSpeedLimit * 1024; - } - this.emit("speedchange", this.speed * 1.2); - - this.predictLeftTime = this.speed <= 0 - ? 0 - : Math.floor((this.prog.total - this.prog.loaded) / this.speed * 1000); - }, intervalDuration) as unknown as number; // hack type problem of nodejs and browser + this.speedCount(this.options.downloadSpeedLimit); } - getInfoForSave() { - return { - // read-only info - storageClasses: this.options.storageClasses, - region: this.options.region, - to: this.options.to, - from: this.options.from, - backendMode: this.options.backendMode, - domain: this.options.domain, - - // real-time info - prog: { - loaded: this.prog.loaded, - total: this.prog.total, - resumable: this.prog.resumable - }, - status: this.status, - message: this.message, - }; + private handleHeader(_objectHeader: ObjectHeader): void { + if (!this.downloader) { + return; + } + // useless? } - tryCleanupDownloadFile(): this { - if ([Status.Finished, Status.Waiting].includes(this.status)) { - return this; + private handlePartGet(_partSize: number): void { + if (!this.downloader) { + return; } - try { - fs.unlinkSync(this.tempfile); - } catch (_err) { - // ignore error - } - return this; + // useless? } } diff --git a/src/common/models/job/transfer-job.ts b/src/common/models/job/transfer-job.ts new file mode 100644 index 00000000..9e8ec5ce --- /dev/null +++ b/src/common/models/job/transfer-job.ts @@ -0,0 +1,166 @@ +import lodash from "lodash"; +import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; + +import Duration from "@common/const/duration"; +import {ClientOptions} from "@common/qiniu"; + +import {Status} from "./types"; +import * as Utils from "./utils"; + +interface RequiredOptions { + clientOptions: ClientOptions, + + from: Utils.LocalPath | Utils.RemotePath + to: Utils.LocalPath | Utils.RemotePath + region: string, +} + +interface OptionalOptions { + id: string, + + status: Status, + message: string, + prog: { + total: number, // Bytes + loaded: number, // Bytes + resumable?: boolean, + }, + + userNatureLanguage: NatureLanguage, +} + +export type Options = RequiredOptions & Partial + +const DEFATUL_OPTIONS: OptionalOptions = { + id: "", + + status: Status.Waiting, + message: "", + prog: { + total: 0, + loaded: 0, + }, + + userNatureLanguage: "zh-CN", +} + +export default abstract class TransferJob { + protected readonly options: Readonly + + readonly id: string + + // - for UI - + private __status: Status + // speed + private lastTimestamp = 0 // ms + private lastLoaded = 0 // Bytes + private speed: number = 0 // Bytes/s + private estimatedTime: number = 0 // seconds + // message + message: string + + // - for resume from break point - + prog: OptionalOptions["prog"] + + protected constructor(config: Options) { + this.id = config.id + ? config.id + : `j-${Date.now()}-${Math.random().toString().substring(2)}`; + + this.options = lodash.merge({}, DEFATUL_OPTIONS, config); + + this.__status = this.options.status; + this.message = this.options.message; + + this.prog = { + ...this.options.prog, + } + + this.speedCount = lodash.throttle(this.speedCount.bind(this), Duration.Second); + } + + get status(): Status { + return this.__status; + } + + get isNotRunning(): boolean { + return this.status !== Status.Running; + } + + get uiData() { + return { + id: this.id, + from: this.options.from, + to: this.options.to, + status: this.status, + message: this.message, + progress: this.prog, + speed: this.speed, + estimatedTime: this.estimatedTime, + } + } + + + abstract start(options?: any): Promise + abstract stop(): this + abstract wait(): this + abstract get persistInfo(): any + + protected abstract handleStatusChange(): void + + // TypeScript specification (8.4.3) says... + // > Accessors for the same member name must specify the same accessibility + protected set _status(value: Status) { + this.__status = value; + this.handleStatusChange(); + + if ( + [ + Status.Failed, + Status.Stopped, + Status.Finished, + Status.Duplicated, + ].includes(this.status) + ) { + this.stopSpeedCounter(); + } + } + + protected startSpeedCounter() { + this.stopSpeedCounter(); + + this.lastTimestamp = Date.now(); + this.lastLoaded = this.prog.loaded; + } + + // call me on progress + protected speedCount(speedLimit: number) { + if (this.isNotRunning) { + this.stopSpeedCounter(); + return; + } + + const nowTimestamp = Date.now(); + const currentSpeed = (this.prog.loaded - this.lastLoaded) / + ((nowTimestamp - this.lastTimestamp) / Duration.Second); + + this.speed = Math.round(currentSpeed); + if (speedLimit > 0) { + this.speed = Math.min(this.speed, speedLimit); + } + this.estimatedTime = Math.max( + Math.round((this.prog.total - this.prog.loaded) / this.speed) * Duration.Second, + 0, + ); + + this.lastLoaded = this.prog.loaded; + this.lastTimestamp = nowTimestamp; + } + + private stopSpeedCounter() { + this.speed = 0; + this.estimatedTime = 0; + this.lastTimestamp = 0; + this.lastLoaded = 0; + } +} diff --git a/src/common/models/job/types.ts b/src/common/models/job/types.ts index 2da90d87..496d803c 100644 --- a/src/common/models/job/types.ts +++ b/src/common/models/job/types.ts @@ -1,10 +1,4 @@ -import { Region } from "kodo-s3-adapter-sdk"; -import { StorageClass } from "kodo-s3-adapter-sdk/dist/adapter"; -import { NatureLanguage } from "kodo-s3-adapter-sdk/dist/uplog"; - // job constructor options -export type BackendMode = "kodo" | "s3"; - export interface UploadedPart { partNumber: number, etag: string, @@ -19,86 +13,3 @@ export enum Status { Duplicated = "duplicated", Verifying = "verifying" } - -// ipc-job -export enum IpcJobEvent { - Upload = "job-upload", - Download = "job-download", - Stop = "job-stop", -} - -export interface IpcUploadJob { - job: string, - key: IpcJobEvent.Upload, - clientOptions: { - accessKey: string, - secretKey: string, - ucUrl?: string, - regions: Region[], - - userNatureLanguage: NatureLanguage, - }, - options: { - resumeUpload: boolean, - maxConcurrency: number, - multipartUploadThreshold: number, - multipartUploadSize: number, - uploadSpeedLimit: number, - kodoBrowserVersion: string, - }, - params: { - region: string, - bucket: string, - key: string, - localFile: string, - isDebug: boolean, - uploadedId?: string, - uploadedParts?: UploadedPart[], - overwriteDup: boolean, - storageClassName: StorageClass["kodoName"], - storageClasses: StorageClass[], - } -} - -export interface IpcDownloadJob { - job: string, - key: IpcJobEvent.Download, - clientOptions: { - accessKey: string, - secretKey: string, - ucUrl?: string, - regions: Region[], - backendMode: BackendMode, - - userNatureLanguage: NatureLanguage, - }, - options: { - resumeDownload: boolean, - maxConcurrency: number, - multipartDownloadThreshold: number, - multipartDownloadSize: number, - downloadSpeedLimit: number, - kodoBrowserVersion: string, - }, - params: { - region: string, - bucket: string, - key: string, - localFile: string, - isDebug: boolean, - domain?: string, - downloadedBytes?: number, - }, -} - -export enum EventKey { - Duplicated = "fileDuplicated", - Stat = "fileStat", - Progress = "progress", - PartUploaded = "filePartUploaded", - PartDownloaded = "filePartDownloaded", - Uploaded = "fileUploaded", - Downloaded = "fileDownloaded", - Error = "error", - Debug = "debug", -} diff --git a/src/common/models/job/upload-job.test.ts b/src/common/models/job/upload-job.test.ts index e31f6333..966ff195 100644 --- a/src/common/models/job/upload-job.test.ts +++ b/src/common/models/job/upload-job.test.ts @@ -1,60 +1,85 @@ -import { Status } from "./types"; -import {uploadOptionsFromNewJob, uploadOptionsFromResumeJob} from "./_mock-helpers_/data"; +import * as MockAuth from "./_mock-helpers_/auth"; +import { mocked } from "ts-jest/utils"; + +jest.mock("kodo-s3-adapter-sdk", () => { + const mockedUploader = jest.fn(); + mockedUploader.constructor = mockedUploader; + mockedUploader.prototype.putObjectFromFile = function () { + return new Promise(resolve => { + setTimeout(resolve, 300) + }); + }; + mockedUploader.prototype.putObjectFromFile = + jest.fn(mockedUploader.prototype.putObjectFromFile); + mockedUploader.prototype.abort = jest.fn(); + + return { + __esModule: true, + ...jest.requireActual("kodo-s3-adapter-sdk"), + Uploader: mockedUploader, + } +}); + +import {Uploader} from "kodo-s3-adapter-sdk"; + +import {BackendMode} from "@common/qiniu"; +import {Status} from "@common/models/job/types"; import UploadJob from "./upload-job"; +import mockFs from "mock-fs"; describe("test models/job/upload-job.ts", () => { - describe("test stop", () => { - it("stop", () => { - const uploadJob = new UploadJob(uploadOptionsFromNewJob); - const spiedEmit = jest.spyOn(uploadJob, "emit"); - spiedEmit.mockImplementation((_eventName: string, ..._args: any[]) => uploadJob); - expect(uploadJob.stop()).toBe(uploadJob); - expect(uploadJob.speed).toBe(0); - expect(uploadJob.predictLeftTime).toBe(0); - expect(uploadJob.status).toBe(Status.Stopped); - expect(uploadJob.emit).toBeCalledWith("statuschange", "stopped"); + const MockedUploader = mocked(Uploader, true); + + describe("test UploadJob methods", () => { + beforeEach(() => { + mockFs({ + "/path/to/dir": { + "file-to-upload.txt": "some content", + }, + }); }); - }); + afterEach(() => { + mockFs.restore(); + }); + it("test UploadJob start", async () => { + const uploadJob = new UploadJob({ + clientOptions: { + accessKey: MockAuth.QINIU_ACCESS_KEY, + secretKey: MockAuth.QINIU_SECRET_KEY, + ucUrl: MockAuth.QINIU_UC, + regions: [], + backendMode: BackendMode.Kodo, + }, + from: { + name: "file-to-upload.txt", + path: "/path/to/dir/file-to-upload.txt", + size: 1024, + mtime: 1658815528906, + }, + to: { + bucket: "fake-bucket", + key: "path/to/file-to-upload.txt", + }, + overwrite: true, + region: "fake-region", + storageClassName: "standard", + storageClasses: [], + }); - // describe("test start", () => { - // it("start()", () => { - // const uploadJob = new UploadJob(uploadOptionsFromNewJob); - // const spiedEmit = jest.spyOn(uploadJob, "emit"); - // spiedEmit.mockImplementation((_eventName: string, ..._args: any[]) => uploadJob); - // expect(uploadJob.start()).toBe(uploadJob); - // expect(uploadJob.message).toBe(""); - // - // // private status flow - // expect(uploadJob.emit).toBeCalledWith("statuschange", Status.Running); - // expect(uploadJob.status).toBe(Status.Running); - // - // // startSpeedCounter flow - // expect(uploadJob.speed).toBe(0); - // expect(uploadJob.predictLeftTime).toBe(0); - // uploadJob.stop(); - // }); - // }); - - describe("test resume upload job", () => { - it("test get persistInfo", () => { - const uploadJob = new UploadJob(uploadOptionsFromResumeJob); - expect(uploadJob.persistInfo).toStrictEqual({ - from: uploadOptionsFromResumeJob.from, - storageClasses: uploadOptionsFromResumeJob.storageClasses, - region: uploadOptionsFromResumeJob.region, - to: uploadOptionsFromResumeJob.to, - overwrite: uploadOptionsFromResumeJob.overwrite, - storageClassName: uploadOptionsFromResumeJob.storageClassName, - backendMode: uploadOptionsFromResumeJob.backendMode, - prog: uploadOptionsFromResumeJob.prog, - status: Status.Waiting, - message: uploadOptionsFromResumeJob.message, - uploadedId: uploadOptionsFromResumeJob.uploadedId, - uploadedParts: uploadOptionsFromResumeJob.uploadedParts.map(p => ({ PartNumber: p.partNumber, ETag: p.etag })), - multipartUploadThreshold: uploadOptionsFromResumeJob.multipartUploadThreshold, - multipartUploadSize: uploadOptionsFromResumeJob.multipartUploadSize, + await uploadJob.start(); + expect(MockedUploader.prototype.putObjectFromFile).toBeCalledTimes(1); + const [ + regionName, + objectInfo, + ] = MockedUploader.prototype.putObjectFromFile.mock.calls[0]; + expect(regionName).toBe("fake-region"); + expect(objectInfo).toStrictEqual({ + bucket: "fake-bucket", + key: "path/to/file-to-upload.txt", + storageClassName: "standard", }); + expect(uploadJob.status).toBe(Status.Finished); }); }); }); diff --git a/src/common/models/job/upload-job.ts b/src/common/models/job/upload-job.ts index 18494ba0..cf638f71 100644 --- a/src/common/models/job/upload-job.ts +++ b/src/common/models/job/upload-job.ts @@ -3,28 +3,21 @@ import {promises as fsPromises} from "fs"; // @ts-ignore import mime from "mime"; import lodash from "lodash"; -import {Qiniu, Region, Uploader} from "kodo-s3-adapter-sdk"; +import {Uploader} from "kodo-s3-adapter-sdk"; import {Adapter, Part, StorageClass} from "kodo-s3-adapter-sdk/dist/adapter"; import {RecoveredOption} from "kodo-s3-adapter-sdk/dist/uploader"; import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; -import Duration from "@common/const/duration"; +import {ClientOptions, createQiniuClient} from "@common/qiniu"; import ByteSize from "@common/const/byte-size"; -import * as AppConfig from "@common/const/app-config"; -import {BackendMode, Status, UploadedPart} from "./types"; -import Base from "./base" +import {Status, UploadedPart} from "./types"; import * as Utils from "./utils"; +import TransferJob from "./transfer-job"; -// if change options, remember to check PersistInfo +// if change options, remember to check `get persistInfo()` interface RequiredOptions { - clientOptions: { - accessKey: string, - secretKey: string, - ucUrl: string, - regions: Region[], - backendMode: BackendMode, - }, + clientOptions: ClientOptions, from: Required, to: Utils.RemotePath, @@ -46,12 +39,13 @@ interface UploadOptions { userNatureLanguage: NatureLanguage, } -interface OptionalOptions extends UploadOptions{ +interface OptionalOptions extends UploadOptions { id: string, uploadedId: string, uploadedParts: UploadedPart[], status: Status, + message: string, prog: { total: number, // Bytes @@ -59,7 +53,10 @@ interface OptionalOptions extends UploadOptions{ resumable?: boolean, }, - message: string, + onStatusChange?: (status: Status) => void, + onProgress?: (prog: UploadJob["prog"]) => void, + onPartCompleted?: (part: Part) => void, + onCompleted?: () => void, } export type Options = RequiredOptions & Partial @@ -113,7 +110,7 @@ type PersistInfo = { multipartUploadSize: OptionalOptions["multipartUploadSize"], } -export default class UploadJob extends Base { +export default class UploadJob extends TransferJob { static fromPersistInfo( id: string, persistInfo: PersistInfo, @@ -156,24 +153,10 @@ export default class UploadJob extends Base { } // - create options - - private readonly options: Readonly - - // - for job save and log - - readonly id: string - readonly kodoBrowserVersion: string + protected readonly options: Readonly private isForceOverwrite: boolean = false - // - for UI - - private __status: Status - // speed - speedTimerId?: number = undefined - speed: number = 0 // Bytes/s - predictLeftTime: number = 0 // seconds - // message - message: string - // - for resume from break point - - prog: OptionalOptions["prog"] uploadedId: string uploadedParts: UploadedPart[] @@ -181,82 +164,45 @@ export default class UploadJob extends Base { uploader?: Uploader constructor(config: Options) { - super(); - this.id = config.id - ? config.id - : `uj-${new Date().getTime()}-${Math.random().toString().substring(2)}`; - this.kodoBrowserVersion = AppConfig.app.version; + super(config) this.options = lodash.merge({}, DEFAULT_OPTIONS, config); - this.__status = this.options.status; - - this.prog = { - ...this.options.prog, - } this.uploadedId = this.options.uploadedId; this.uploadedParts = [ ...this.options.uploadedParts, ]; - this.message = this.options.message; - // hook functions this.startUpload = this.startUpload.bind(this); + this.handleStatusChange = this.handleStatusChange.bind(this); this.handleProgress = this.handleProgress.bind(this); this.handlePartsInit = this.handlePartsInit.bind(this); this.handlePartPutted = this.handlePartPutted.bind(this); } - get accessKey(): string { - return this.options.clientOptions.accessKey; - } - - // TypeScript specification (8.4.3) says... - // > Accessors for the same member name must specify the same accessibility - private set _status(value: Status) { - this.__status = value; - this.emit("statuschange", this.status); - - if ( - this.status === Status.Failed - || this.status === Status.Stopped - || this.status === Status.Finished - || this.status === Status.Duplicated - ) { - this.stopSpeedCounter(); - } - } - - get status(): Status { - return this.__status - } - - get isNotRunning(): boolean { - return this.status !== Status.Running; - } - get uiData() { return { - id: this.id, + ...super.uiData, from: this.options.from, - to: this.options.to, - status: this.status, - speed: this.speed, - estimatedTime: this.predictLeftTime, - progress: this.prog, - message: this.message, - } + }; } async start( - forceOverwrite: boolean = false, + options?: { + forceOverwrite: boolean + }, ): Promise { if (this.status === Status.Running || this.status === Status.Finished) { return; } - if (forceOverwrite) { + this.message = ""; + this._status = Status.Running; + + this.startSpeedCounter(); + + if (options?.forceOverwrite) { this.isForceOverwrite = true; } @@ -264,36 +210,13 @@ export default class UploadJob extends Base { console.log(`Try uploading ${this.options.from.path} to kodo://${this.options.to.bucket}/${this.options.to.key}`); } - this.message = "" - - this._status = Status.Running; - // create client - const qiniu = new Qiniu( - this.options.clientOptions.accessKey, - this.options.clientOptions.secretKey, - this.options.clientOptions.ucUrl, - `Kodo-Browser/${this.kodoBrowserVersion}/ioutil`, - this.options.clientOptions.regions, - ); - const qiniuClient = qiniu.mode( - this.options.clientOptions.backendMode, - { - appName: 'kodo-browser/ioutil', - appVersion: this.kodoBrowserVersion, - appNatureLanguage: this.options.userNatureLanguage, - // disable uplog when use customize cloud - // because there isn't a valid access key of uplog - uplogBufferSize: this.options.clientOptions.ucUrl ? -1 : undefined, - requestCallback: () => { - }, - responseCallback: () => { - }, - }, - ); + const qiniuClient = createQiniuClient(this.options.clientOptions, { + userNatureLanguage: this.options.userNatureLanguage, + isDebug: this.options.isDebug, + }); // upload - this.startSpeedCounter(); await qiniuClient.enter( "uploadFile", this.startUpload, @@ -370,7 +293,7 @@ export default class UploadJob extends Base { this._status = Status.Finished; await fileHandle.close(); - this.emit("complete"); + this.options.onCompleted?.(); } stop(): this { @@ -411,68 +334,6 @@ export default class UploadJob extends Base { return this; } - private startSpeedCounter() { - this.stopSpeedCounter(); - - let lastTimestamp = new Date().getTime(); - let lastLoaded = this.prog.loaded; - let zeroSpeedCounter = 0; - const intervalDuration = Duration.Second; - this.speedTimerId = setInterval(() => { - if (this.isNotRunning) { - this.stopSpeedCounter(); - return; - } - - const nowTimestamp = new Date().getTime(); - const currentSpeed = (this.prog.loaded - lastLoaded) / ((nowTimestamp - lastTimestamp) / Duration.Second); - if (currentSpeed < 1 && zeroSpeedCounter < 3) { - zeroSpeedCounter += 1; - return; - } - - this.speed = Math.round(currentSpeed); - this.predictLeftTime = Math.max( - Math.round((this.prog.total - this.prog.loaded) / this.speed) * Duration.Second, - 0, - ); - - lastLoaded = this.prog.loaded; - lastTimestamp = nowTimestamp; - zeroSpeedCounter = 0; - }, intervalDuration) as unknown as number; // hack type problem of nodejs and browser - } - - private stopSpeedCounter() { - this.speed = 0; - this.predictLeftTime = 0; - clearInterval(this.speedTimerId); - } - - private handleProgress(uploaded: number, total: number) { - if (!this.uploader) { - return; - } - this.prog.loaded = uploaded; - this.prog.total = total; - - this.emit("progress", lodash.merge({}, this.prog)); - } - - private handlePartsInit(initInfo: RecoveredOption) { - this.uploadedId = initInfo.uploadId; - this.uploadedParts = initInfo.parts; - } - - private handlePartPutted(part: Part) { - if (!this.uploader) { - return; - } - this.uploadedParts.push(part); - - this.emit("partcomplete", lodash.merge({}, part)); - } - get persistInfo(): PersistInfo { return { from: this.options.from, @@ -499,4 +360,34 @@ export default class UploadJob extends Base { multipartUploadSize: this.options.multipartUploadSize, }; } + + protected handleStatusChange() { + this.options.onStatusChange?.(this.status); + } + + private handleProgress(uploaded: number, total: number) { + if (!this.uploader) { + return; + } + this.prog.loaded = uploaded; + this.prog.total = total; + + this.options.onProgress?.(lodash.merge({}, this.prog)); + + this.speedCount(this.options.uploadSpeedLimit); + } + + private handlePartsInit(initInfo: RecoveredOption) { + this.uploadedId = initInfo.uploadId; + this.uploadedParts = initInfo.parts; + } + + private handlePartPutted(part: Part) { + if (!this.uploader) { + return; + } + this.uploadedParts.push(part); + + this.options.onPartCompleted?.(lodash.merge({}, part)); + } } diff --git a/src/common/models/job/utils.test.ts b/src/common/models/job/utils.test.ts index 855feeb1..ec66cc39 100644 --- a/src/common/models/job/utils.test.ts +++ b/src/common/models/job/utils.test.ts @@ -6,21 +6,23 @@ import * as JobUtils from "./utils"; describe("test models/job/utils.ts", () => { describe("test parseLocalPath", () => { - it("unix-like path", () => { - expect(JobUtils.parseLocalPath("/path/to/Some/file.txt")) - .toEqual({ - name: "file.txt", - path: "/path/to/Some/file.txt", - }); - }); - // TODO: check on win system - // it("win path", () => { - // expect(JobUtils.parseLocalPath("D:\\path\\to\\Some\\file.txt")) - // .toEqual({ - // name: "file.txt", - // path: "D:\\path\\to\\Some\\file.txt", - // }); - // }); + if (process.platform === "win32") { + it("win path", () => { + expect(JobUtils.parseLocalPath("D:\\path\\to\\Some\\file.txt")) + .toEqual({ + name: "file.txt", + path: "D:\\path\\to\\Some\\file.txt", + }); + }); + } else { + it("unix-like path", () => { + expect(JobUtils.parseLocalPath("/path/to/Some/file.txt")) + .toEqual({ + name: "file.txt", + path: "/path/to/Some/file.txt", + }); + }); + } }); describe("test parseKodoPath", () => { diff --git a/src/common/models/job/utils.ts b/src/common/models/job/utils.ts index 7145d5a8..b4c6b7b4 100644 --- a/src/common/models/job/utils.ts +++ b/src/common/models/job/utils.ts @@ -40,6 +40,10 @@ export function parseKodoPath(kodoPath: string): RemotePath { }; } +export function isLocalPath(p: LocalPath | RemotePath): p is LocalPath { + return p.hasOwnProperty("name"); +} + // get etag function sha1(content: Buffer): Buffer { return crypto.createHash("sha1") @@ -86,7 +90,9 @@ function getEtagByBuffer(content: Buffer, callback: (etag: string) => void): voi } } function getEtagByStream(dataStream: ReadableStream, callback: (etag: string) => void): void { - // 以4M为单位分割，why? + // 以 4M 为单位分割，why? + // 对外暴露的方法仅有自动下载新版本，猜测时用于验证是否下载完整，后续 crc 上线后需要修改这里 + // 4M 是上传新版本到七牛空间时采用的分片 const blockSize = 4*1024*1024; const sha1String: Buffer[] = []; let blockCount = 0; diff --git a/src/common/qiniu-store/lib/consts.js b/src/common/qiniu-store/lib/consts.js deleted file mode 100644 index e4942f3f..00000000 --- a/src/common/qiniu-store/lib/consts.js +++ /dev/null @@ -1,5 +0,0 @@ -'use strict'; - -export const MIN_MULTIPART_SIZE = 4 << 20; // 4 MB -export const MAX_PUTOBJECT_SIZE = (1 << 30) * 5; // 5 GB -export const MAX_MULTIPART_COUNT = 10000; diff --git a/src/common/qiniu-store/lib/ioutil.js b/src/common/qiniu-store/lib/ioutil.js deleted file mode 100644 index 6a3c9ebb..00000000 --- a/src/common/qiniu-store/lib/ioutil.js +++ /dev/null @@ -1,419 +0,0 @@ -import { promises as fsPromises } from 'fs' -import path from 'path' - -import { Qiniu, Uploader, Downloader } from 'kodo-s3-adapter-sdk' -import { EventEmitter } from 'events' -import mime from 'mime' - -import { - MIN_MULTIPART_SIZE, - MAX_PUTOBJECT_SIZE, - MAX_MULTIPART_COUNT, -} from './consts' - -EventEmitter.prototype._maxListeners = 1000 - -class Client { - constructor(clientOptions, options) { - options = options ? options : {}; - - const qiniu = new Qiniu( - clientOptions.accessKey, clientOptions.secretKey, clientOptions.ucUrl, - `Kodo-Browser/${options.kodoBrowserVersion}/ioutil`, - clientOptions.regions); - - const modeOpts = { - appName: 'kodo-browser/ioutil', - appVersion: options.kodoBrowserVersion, - appNatureLanguage: clientOptions.userNatureLanguage, - }; - if (clientOptions.isDebug) { - modeOpts.requestCallback = debugRequest(clientOptions.backendMode); - modeOpts.responseCallback = debugResponse(clientOptions.backendMode); - } - // disable uplog when use customize cloud - // because there isn't a valid access key of uplog - if (clientOptions.ucUrl) { - modeOpts.uplogBufferSize = -1; - } - this.client = qiniu.mode(clientOptions.backendMode, modeOpts); - this.uploader = undefined; - this.downloader = undefined; - - this.resumeUpload = options.resumeUpload === true; - this.multipartUploadThreshold = options.multipartUploadThreshold || (MIN_MULTIPART_SIZE * 10); - this.multipartUploadSize = options.multipartUploadSize || (MIN_MULTIPART_SIZE * 2); - - this.resumeDownload = options.resumeDownload === true; - this.multipartDownloadThreshold = options.multipartDownloadThreshold || (MIN_MULTIPART_SIZE * 10); - this.multipartDownloadSize = options.multipartDownloadSize || (MIN_MULTIPART_SIZE * 2); - - this.uploadSpeedLimit = options.uploadSpeedLimit || false; - this.downloadSpeedLimit = options.downloadSpeedLimit || false; - - if (this.multipartUploadSize < MIN_MULTIPART_SIZE) { - throw new Error('Minimum multipartUploadSize is 4 MB.'); - } - if (this.multipartUploadSize > MAX_PUTOBJECT_SIZE) { - throw new Error('Maximum multipartUploadSize is 5 GB.'); - } - if (this.multipartUploadThreshold < MIN_MULTIPART_SIZE) { - throw new Error('Minimum multipartUploadThreshold is 4 MB.'); - } - if (this.multipartUploadThreshold > MAX_PUTOBJECT_SIZE) { - throw new Error('Maximum multipartUploadThreshold is 5 GB.'); - } - } - - uploadFile(params) { - const self = this; - const localFile = params.localFile; - const contentType = mime.getType(localFile); - const isOverwrite = params.overwriteDup; - let isAborted = false; - - let recoveredOption = undefined; - if (params.uploadedId && params.uploadedParts) { - recoveredOption = { - uploadId: params.uploadedId, - parts: params.uploadedParts, - }; - } - let uploadedPartSize = this.multipartUploadSize; - const uploadThreshold = this.multipartUploadThreshold; - - const eventEmitter = new EventEmitter(); - eventEmitter.setMaxListeners(0); - eventEmitter.progressLoaded = 0; - eventEmitter.progressTotal = 0; - eventEmitter.progressResumable = false; - eventEmitter.abort = handleAbort; - - this.client.enter('uploadFile', (client) => { - client.storageClasses = params.storageClasses; - this.uploader = new Uploader(client); - - if (isOverwrite) { - return startUploadFile(); - } else { - return new Promise((resolve, reject) => { - client.isExists(params.region, { bucket: params.bucket, key: params.key }).then((isExists) => { - if (isExists) { - eventEmitter.emit('fileDuplicated', eventEmitter); - resolve(); - } else { - startUploadFile().then(resolve).catch(reject); - } - }, () => { - startUploadFile().then(resolve).catch(reject); - }); - }); - } - }, { - targetBucket: params.bucket, - targetKey: params.key, - }).finally(() => { - this.uploader = undefined; - }); - - process.on('uncaughtException', (err) => { - handleError({ - error: err.message, - stack: err.stack.split("\n") - }); - }); - - return eventEmitter; - - function startUploadFile() { - return new Promise((resolve, reject) => { - fsPromises.stat(localFile).then((stats) => { - if (isAborted) { - reject(new Error('Aborted')); - return; - } - const partsCount = Math.ceil(stats.size / uploadedPartSize); - if (partsCount > MAX_MULTIPART_COUNT) { - uploadedPartSize = smallestPartSizeFromFileSize(stats.size); - } - if (uploadedPartSize > MAX_PUTOBJECT_SIZE) { - const err = new Error(`File size exceeds maximum object size: ${localFile}`); - err.retryable = false; - handleError(err); - reject(err); - return; - } - eventEmitter.progressLoaded = 0; - eventEmitter.progressTotal = stats.size; - eventEmitter.progressResumable = self.resumeUpload && (!recoveredOption || uploadedPartSize === self.multipartUploadSize); - eventEmitter.emit('fileStat', eventEmitter); - - fsPromises.open(localFile, 'r').then((fileHandle) => { - if (isAborted) { - reject(new Error('Aborted')); - return; - } - - const fileName = path.basename(localFile); - let lastProgressTime = new Date(); - let uploadThrottleOption = undefined; - - if (self.uploadSpeedLimit) { - uploadThrottleOption = { rate: self.uploadSpeedLimit * 1024 }; - } - - self.uploader.putObjectFromFile(params.region, { bucket: params.bucket, key: params.key, storageClassName: params.storageClassName }, fileHandle, stats.size, fileName, { - header: { contentType }, - recovered: recoveredOption, - uploadThreshold: uploadThreshold, - partSize: uploadedPartSize, - putCallback: { - partsInitCallback: (recovered) => { - recoveredOption = recovered; - }, - partPutCallback: (part) => { - if (isAborted) { - return; - } - eventEmitter.emit('filePartUploaded', { - uploadId: recoveredOption.uploadId, - part: part, - }); - }, - progressCallback: (uploaded) => { - if (isAborted) { - return; - } - eventEmitter.progressLoaded = uploaded; - if (eventEmitter.progressLoaded > eventEmitter.progressTotal) { - eventEmitter.progressLoaded = eventEmitter.progressTotal; - } - - const now = new Date(); - if (now - lastProgressTime > 1000) { - lastProgressTime = now; - eventEmitter.emit('progress', eventEmitter); - } - }, - uploadThrottleOption: uploadThrottleOption, - }, - }).then(() => { - if (isAborted) { - reject(new Error('Aborted')); - return; - } - eventEmitter.progressLoaded = eventEmitter.progressTotal; - eventEmitter.emit('progress', eventEmitter); - eventEmitter.emit('fileUploaded', eventEmitter); - resolve(); - }).catch((err) => { - handleError(err); - reject(err); - }); - }).catch((err) => { - err.retryable = false; - handleError(err); - reject(err); - }); - }).catch((err) => { - err.retryable = false; - handleError(err); - reject(err); - }); - }); - } - - function handleError(err) { - if (isAborted) return; - - if (err && err.retryable === false) { - handleAbort(); - } - - if (err.message) { - eventEmitter.emit('error', `${err.name}: ${err.message}`); - } else { - eventEmitter.emit('error', err.name); - } - } - - function handleAbort() { - if (isAborted) return; - isAborted = true; - - if (self.uploader) { - self.uploader.abort(); - } - - eventEmitter.emit('abort', {}); - } - - function smallestPartSizeFromFileSize(fileSize) { - const partSize = Math.ceil(fileSize / MAX_MULTIPART_COUNT); - - if (partSize < MIN_MULTIPART_SIZE) { - return MIN_MULTIPART_SIZE; - } - - return partSize + (MIN_MULTIPART_SIZE - partSize % MIN_MULTIPART_SIZE); - } - } - - downloadFile(params) { - const self = this; - const localFile = params.localFile; - let isAborted = false; - - let downloadedBytes = params.downloadedBytes || 0; - let downloadedPartSize = this.multipartDownloadSize; - - if (!this.resumeDownload) { - downloadedBytes = 0; - } - - const eventEmitter = new EventEmitter(); - eventEmitter.setMaxListeners(0); - eventEmitter.progressLoaded = 0; - eventEmitter.progressTotal = 0; - eventEmitter.progressResumable = self.resumeDownload; - eventEmitter.abort = handleAbort; - - this.client.enter('downloadFile', (client) => { - this.downloader = new Downloader(client); - return startDownloadFile().finally(() => { - this.downloader = undefined; - }); - }, { - targetBucket: params.bucket, - targetKey: params.key, - }); - - process.on('uncaughtException', (err) => { - handleError({ - error: err.message, - stack: err.stack.split("\n") - }); - }); - - return eventEmitter; - - function startDownloadFile() { - let downloadThrottleOption = undefined; - if (self.downloadSpeedLimit) { - downloadThrottleOption = { rate: self.downloadSpeedLimit * 1024 }; - } - - eventEmitter.emit('progress', eventEmitter); - let lastProgressTime = new Date(); - - return self.downloader.getObjectToFile(params.region, { bucket: params.bucket, key: params.key }, localFile, params.domain, { - recoveredFrom: downloadedBytes, - partSize: downloadedPartSize, - chunkTimeout: 30000, - retriesOnSameOffset: 10, - downloadThrottleOption: downloadThrottleOption, - getCallback: { - headerCallback: (header) => { - if (isAborted) { - return; - } - eventEmitter.progressTotal = header.size; - eventEmitter.emit('fileStat', eventEmitter); - }, - partGetCallback: (partSize) => { - if (isAborted) { - return; - } - downloadedBytes += partSize; - eventEmitter.emit('filePartDownloaded', { size: partSize }); - }, - progressCallback: (downloaded, total) => { - if (isAborted) { - return; - } - eventEmitter.progressLoaded = downloaded; - if (eventEmitter.progressLoaded > eventEmitter.progressTotal) { - eventEmitter.progressLoaded = eventEmitter.progressTotal; - } - - const now = new Date(); - if (now - lastProgressTime > 1000) { - lastProgressTime = now; - eventEmitter.emit('progress', eventEmitter); - } - }, - }, - }).then(() => { - if (isAborted) { - return; - } - - eventEmitter.progressLoaded = eventEmitter.progressTotal; - eventEmitter.emit('progress', eventEmitter); - eventEmitter.emit('fileDownloaded', eventEmitter); - }).catch(handleError); - } - - function handleError(err) { - if (isAborted) return; - - if (err && err.retryable === false) { - handleAbort(); - } - - if (err.message) { - eventEmitter.emit('error', `${err.name}: ${err.message}`); - } else { - eventEmitter.emit('error', err.name); - } - } - - function handleAbort() { - if (isAborted) return; - isAborted = true; - - if (self.downloader) { - self.downloader.abort(); - } - - eventEmitter.emit('abort', {}); - } - } -} - -function debugRequest(mode) { - return (request) => { - let url = undefined, method = undefined, headers = undefined; - if (request) { - url = request.url; - method = request.method; - headers = request.headers; - } - console.info('>>', mode, 'REQ_URL:', url, 'REQ_METHOD:', method, 'REQ_HEADERS:', headers); - }; -} - -function debugResponse(mode) { - return (response) => { - let requestUrl = undefined, requestMethod = undefined, requestHeaders = undefined, - responseStatusCode = undefined, responseHeaders = undefined, responseInterval = undefined, responseData = undefined, responseError = undefined; - if (response) { - responseStatusCode = response.statusCode; - responseHeaders = response.headers; - responseInterval = response.interval; - responseData = response.data; - responseError = response.error; - if (response.request) { - requestUrl = response.request.url; - requestMethod = response.request.method; - requestHeaders = response.request.headers; - } - } - console.info('<<', mode, 'REQ_URL:', requestUrl, 'REQ_METHOD:', requestMethod, 'REQ_HEADERS: ', requestHeaders, - 'RESP_STATUS:', responseStatusCode, 'RESP_HEADERS:', responseHeaders, 'RESP_INTERVAL:', responseInterval, 'ms RESP_DATA:', responseData, 'RESP_ERROR:', responseError); - }; -} - -export function createClient(clientOptions, options) { - return new Client(clientOptions, options); -} diff --git a/src/main/util/createClient.ts b/src/common/qiniu/create-client.ts similarity index 96% rename from src/main/util/createClient.ts rename to src/common/qiniu/create-client.ts index 9535a7a2..1191c31b 100644 --- a/src/main/util/createClient.ts +++ b/src/common/qiniu/create-client.ts @@ -4,8 +4,7 @@ import {ModeOptions} from "kodo-s3-adapter-sdk/dist/qiniu"; import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; import * as AppConfig from "@common/const/app-config"; -import {ClientOptions} from "@common/ipc-actions/upload"; -import {BackendMode} from "@common/const/qiniu"; +import {BackendMode, ClientOptions} from "./types"; export default function createQiniuClient( clientOptions: ClientOptions, diff --git a/src/common/qiniu/index.ts b/src/common/qiniu/index.ts new file mode 100644 index 00000000..3f7f8afc --- /dev/null +++ b/src/common/qiniu/index.ts @@ -0,0 +1,2 @@ +export * from "./types"; +export {default as createQiniuClient} from "./create-client"; diff --git a/src/common/qiniu/types.ts b/src/common/qiniu/types.ts new file mode 100644 index 00000000..4b06dece --- /dev/null +++ b/src/common/qiniu/types.ts @@ -0,0 +1,16 @@ +import {Region} from "kodo-s3-adapter-sdk"; + +export enum BackendMode { + Kodo = "kodo", + S3 = "s3", +} + +export interface ClientOptions { + accessKey: string, + secretKey: string, + ucUrl: string, + regions: Region[], + backendMode: BackendMode, + + // storageClasses: StorageClass[], // TODO +} diff --git a/src/main/download-worker.js b/src/main/download-worker.js deleted file mode 100644 index 83a72557..00000000 --- a/src/main/download-worker.js +++ /dev/null @@ -1,95 +0,0 @@ -import { - createClient -} from '../common/qiniu-store/lib/ioutil' - -process.on('uncaughtException', function (err) { - process.send({ - key: 'error', - error: err.message, - stack: err.stack.split("\n") - }); -}); - -process.send({ - key: 'env', - execPath: process.execPath, - version: process.version -}); - -process.on('message', (msg) => { - switch (msg.key) { - case 'stop': - process.exit(0); - break; - - case 'start': - global.Global = { - app: { - id: 'kodo-browser', - logo: 'icons/icon.png', - version: msg.data.options.kodoBrowserVersion, - }, - }; - const client = createClient(msg.data.clientOptions, msg.data.options); - - const downloader = client.downloadFile(msg.data.params); - downloader.on('fileStat', (prog) => { - process.send({ - job: msg.data.job, - key: 'fileStat', - data: { - progressLoaded: 0, - progressTotal: prog.progressTotal, - progressResumable: prog.progressResumable - } - }); - }); - downloader.on('progress', (prog) => { - process.send({ - job: msg.data.job, - key: 'progress', - data: { - progressLoaded: prog.progressLoaded, - progressTotal: prog.progressTotal, - progressResumable: prog.progressResumable - } - }); - }); - downloader.on('filePartDownloaded', (part) => { - process.send({ - job: msg.data.job, - key: 'filePartDownloaded', - data: part || {} - }); - }); - downloader.on('fileDownloaded', (result) => { - process.send({ - job: msg.data.job, - key: 'fileDownloaded', - data: result || {} - }); - }); - downloader.on('error', (err) => { - process.send({ - job: msg.data.job, - key: 'error', - error: err - }); - }); - downloader.on('debug', (data) => { - process.send({ - job: msg.data.job, - key: 'debug', - data: data - }); - }); - - break; - - default: - process.send({ - key: `[Error] ${msg.key}`, - message: msg - }); - } -}); diff --git a/src/main/download-worker.ts b/src/main/download-worker.ts new file mode 100644 index 00000000..1ca869dc --- /dev/null +++ b/src/main/download-worker.ts @@ -0,0 +1,164 @@ +import { + AddedJobsReplyMessage, + DownloadAction, + DownloadMessage, + JobCompletedReplyMessage, + UpdateUiDataReplyMessage +} from "@common/ipc-actions/download"; +import DownloadManager from "./transfer-managers/download-manager"; +import DownloadJob from "@common/models/job/download-job"; +import {Status} from "@common/models/job/types"; + + +// initial DownloadManager Config from argv after `--config-json` +const configStr = process.argv.find((_arg, i, arr) => arr[i - 1] === "--config-json"); +const downloadManagerConfig = configStr ? JSON.parse(configStr) : {}; +downloadManagerConfig.onJobDone = handleJobDone; +const downloadManager = new DownloadManager(downloadManagerConfig); + +process.on("uncaughtException", (err) => { + downloadManager.persistJobs(true); + console.error("download worker: uncaughtException", err); +}); + +process.on("message", (message: DownloadMessage) => { + switch (message.action) { + case DownloadAction.UpdateConfig: { + downloadManager.updateConfig(message.data); + break; + } + case DownloadAction.LoadPersistJobs: { + downloadManager.loadJobsFromStorage( + message.data.clientOptions, + message.data.downloadOptions, + ); + break; + } + case DownloadAction.AddJobs: { + downloadManager.createDownloadJobs( + message.data.remoteObjects, + message.data.destPath, + message.data.clientOptions, + message.data.downloadOptions, + { + jobsAdding: () => { + downloadManager.persistJobs(); + }, + jobsAdded: () => { + const replyMessage: AddedJobsReplyMessage = { + action: DownloadAction.AddedJobs, + data: { + remoteObjects: message.data.remoteObjects, + destPath: message.data.destPath, + }, + }; + process.send?.(replyMessage); + }, + }, + ); + break; + } + case DownloadAction.UpdateUiData: { + const replyMessage: UpdateUiDataReplyMessage = { + action: DownloadAction.UpdateUiData, + data: downloadManager.getJobsUiDataByPage( + message.data.pageNum, + message.data.count, + message.data.query, + ), + } + process.send?.(replyMessage); + break; + } + case DownloadAction.StopJob: { + downloadManager.stopJob(message.data.jobId); + break; + } + case DownloadAction.WaitJob: { + downloadManager.waitJob(message.data.jobId); + break; + } + case DownloadAction.StartJob: { + downloadManager.startJob(message.data.jobId); + break; + } + case DownloadAction.RemoveJob: { + downloadManager.removeJob(message.data.jobId); + break; + } + case DownloadAction.CleanupJobs: { + downloadManager.cleanupJobs(); + break; + } + case DownloadAction.StartAllJobs: { + downloadManager.startAllJobs(); + break; + } + case DownloadAction.StopAllJobs: { + downloadManager.stopAllJobs(); + break; + } + case DownloadAction.RemoveAllJobs: { + downloadManager.removeAllJobs(); + downloadManager.persistJobs(true); + break; + } + default: { + console.warn("Download Manager received unknown action, message:", message); + } + } +}); + + +let isCleanup = false; +function handleExit() { + if (isCleanup) { + return Promise.resolve(); + } + + isCleanup = true; + + downloadManager.stopAllJobs({ + matchStatus: [Status.Waiting], + }); + + return new Promise((resolve, reject) => { + try { + // resolve inflight jobs persisted status not correct + setTimeout(() => { + downloadManager.stopAllJobs({ + matchStatus: [Status.Running], + }); + downloadManager.persistJobs(true); + resolve(); + }, 2000); + } catch { + reject() + } + }); +} + + +process.on("exit", () => { + handleExit(); +}); + +process.on("SIGTERM", () => { + handleExit() + .then(() => { + process.exit(0); + }); +}); + +function handleJobDone(jobId: string, job?: DownloadJob) { + if (job?.status === Status.Finished) { + const jobCompletedReplyMessage: JobCompletedReplyMessage = { + action: DownloadAction.JobCompleted, + data: { + jobsId: jobId, + jobUiData: job.uiData, + } + }; + process.send?.(jobCompletedReplyMessage); + } +} diff --git a/src/main/index.js b/src/main/index.js deleted file mode 100755 index be48d1a6..00000000 --- a/src/main/index.js +++ /dev/null @@ -1,763 +0,0 @@ -const fs = require("fs"); -const os = require("os"); -const path = require("path"); -const electron = require("electron"); -const { - app, - globalShortcut, - dialog, - Menu, - ipcMain, - BrowserWindow -} = electron; -const electronRemote = require('@electron/remote/main'); -electronRemote.initialize(); -const { - fork -} = require("child_process"); -const { UplogBuffer } = require("kodo-s3-adapter-sdk/dist/uplog"); -const { UploadAction } = require("@common/ipc-actions/upload"); - -///***************************************** -let root = path.dirname(__dirname); - -let uiRoot = path.join(root, 'renderer'); -// if (process.env.NODE_ENV != "development") { -// uiRoot = path.join(uiRoot, "app"); -// } - -const iconRoot = path.join(root, 'renderer', 'icons') - -// Keep a global reference of the window object, if you don't, the window will -// be closed automatically when the JavaScript object is garbage collected. -let win; -let winBlockedTid; // time interval for close -let forkedWorkers = new Map(); -let uploadRunning = 0; - -switch (process.platform) { -case "darwin": - app.dock.setIcon( - path.join(iconRoot, "icon.png") - ); - break; -case "linux": - break; -case "win32": - break; -} - -//singleton -app.requestSingleInstanceLock(); -// Someone tried to run a second instance, we should focus our window. -app.on('second-instance', (evt, argv, cwd) => { - if (win) { - if (win.isMinimized()) { - win.restore(); - } - - win.focus(); - - app.quit(); - } -}); -app.releaseSingleInstanceLock(); - -let createWindow = () => { - let opt = { - width: 1280, - height: 800, - minWidth: 1280, - minHeight: 600, - title: "Kodo Browser", - icon: path.join(iconRoot, "icon.ico"), - webPreferences: { - nodeIntegration: true, - contextIsolation: false, - }, - }; - - let confirmForWorkers = (e) => { - const runningJobs = forkedWorkers.size - - 1 + // upload-worker - uploadRunning; // upload running jobs; - - if (runningJobs <= 0) { - return; - } - - // always stop - if (winBlockedTid) { - clearInterval(winBlockedTid); - } - - let confirmCb = (btn) => { - if (process.platform == "darwin") { - switch (btn) { - case 0: - btn = 2; - break; - - case 1: - // ignore - break; - - case 2: - btn = 0; - break; - - default: - btn = 1; - } - } - - switch (btn) { - case 0: - forkedWorkers.forEach((worker) => { - worker.kill(); - }); - - forkedWorkers = new Map(); - - break; - - case 1: - // cancel close - e.preventDefault(); - - break; - - case 2: - // cancel close - e.preventDefault(); - - clearInterval(winBlockedTid); - winBlockedTid = setInterval(() => { - if (runningJobs > 0) { - return; - } - - clearInterval(winBlockedTid); - - win.close(); - }, 3000); - - break; - - default: - // cancel close - e.preventDefault(); - } - }; - - let btns = ["Force Quit", "Cancel", "Waiting for jobs"]; - if (process.platform == "darwin") { - btns = btns.reverse(); - } - - // prevent if there still alive workers. - confirmCb(dialog.showMessageBox({ - type: "warning", - message: `There ${runningJobs > 1 ? "are" : "is"} still ${runningJobs} ${runningJobs > 1 ? "jobs" : "job"} in processing, are you sure to quit?`, - buttons: btns - })); - }; - - if (process.platform == "linux") { - opt.icon = path.join(root, "src", "renderer", "icons", "icon.png"); - } - - // Create the browser window. http://electron.atom.io/docs/api/browserwindow/ - win = new BrowserWindow(opt); - win.setTitle(opt.title); - win.setMenuBarVisibility(false); - electronRemote.enable(win.webContents); - - // Emitted before window reload - win.on("beforeunload", confirmForWorkers); - - let focused = true, shown = true; - - const registerOrUnregisterShortcutForDevTools = () => { - if (process.env.NODE_ENV != "development") { - const shortcut = 'CommandOrControl+Alt+I'; - if (shown && focused) { - if (!globalShortcut.isRegistered(shortcut)) { - globalShortcut.register(shortcut, () => { - win.webContents.toggleDevTools(); - }); - } - } else { - if (globalShortcut.isRegistered(shortcut)) { - globalShortcut.unregister(shortcut); - } - } - } - }; - - win.on("blur", function() { - focused = false; - registerOrUnregisterShortcutForDevTools(); - }); - win.on("focus", function() { - focused = true; - registerOrUnregisterShortcutForDevTools(); - }); - win.on("show", function() { - shown = true; - registerOrUnregisterShortcutForDevTools(); - }); - win.on("hide", function() { - shown = false; - registerOrUnregisterShortcutForDevTools(); - }); - win.on("close", confirmForWorkers); - - // Emitted when the window is closed. - win.on("closed", (e) => { - if (forkedWorkers.size > 0) { - // force cleanup forked workers - forkedWorkers.forEach((worker) => { - worker.kill(); - }); - - forkedWorkers = new Map(); - } - - // Dereference the window object, usually you would store windows - // in an array if your app supports multi windows, this is the time - // when you should delete the corresponding element. - win = null; - }); - - // load the index.html of the app. - win.loadURL(`file://${uiRoot}/index.html`); - - if (process.env.NODE_ENV == "development") { - console.log("run in development"); - - // Open the DevTools. - win.webContents.openDevTools(); - } else if (process.platform === "darwin") { - console.log("run on macos in production"); - // Create the Application's main menu - Menu.setApplicationMenu(Menu.buildFromTemplate(getMenuTemplate())); - } -}; - -///***************************************** -// listener events send from renderer process -ipcMain.on("UploaderManager", (event, message) => { - const processName = "UploaderProcess"; - let uploaderProcess = forkedWorkers.get(processName); - if (!uploaderProcess) { - uploaderProcess = fork( - path.join(root, "main", "uploader-bundle.js"), - // is there a better way to pass parameters? - ['--config-json', JSON.stringify({resumeUpload: true, maxConcurrency: 5})], - { - cwd: root, - silent: false, - }, - ); - forkedWorkers.set(processName, uploaderProcess); - - uploaderProcess.on("exit", () => { - forkedWorkers.delete(processName) - }); - - uploaderProcess.on("message", (message) => { - if (win && !win.isDestroyed()) { - event.sender.send("UploaderManager-reply", message); - } - switch (message.action) { - case UploadAction.UpdateUiData: { - uploadRunning = message.data.running; - break; - } - } - }); - } - - uploaderProcess.send(message); -}); - -ipcMain.on("asynchronous", (event, data) => { - switch (data.key) { - case "getStaticServerPort": - event.sender.send("asynchronous-reply", { - key: data.key, - port: serverPort - }); - break; - - case "openDevTools": - win.webContents.openDevTools(); - break; - - case "reloadWindow": - win.webContents.reload(); - break; - - case "clearCache": - win.webContents.session.clearCache(() => { - console.info('cache cleared'); - }); - break; - } -}); - -ipcMain.on("asynchronous-job", (event, data) => { - switch (data.key) { - case "job-upload": - var forkOptions = { - cwd: root, - stdio: [0, 1, 2, 'ipc'], - silent: true - }; - - var execScript = path.join(root, 'main', 'upload-worker-bundle.js'); - - if (data.params.isDebug) { - event.sender.send(data.job, { - job: data.job, - key: 'debug', - env: { - fork: forkOptions, - script: execScript - }, - data: data - }); - } - - var worker = fork(execScript, [], forkOptions); - forkedWorkers.set(data.job, worker); - - worker.send({ - key: 'start', - data: data - }); - - worker.on('message', function (msg) { - if (!win) return; - - if (data.params.isDebug) { - event.sender.send(data.job, { - job: data.job, - key: 'debug', - message: msg - }); - } - - switch (msg.key) { - case 'fileDuplicated': - event.sender.send(data.job, { - job: data.job, - key: 'fileDuplicated', - }); - - break; - - case 'fileStat': - event.sender.send(data.job, { - job: data.job, - key: 'fileStat', - data: { - progressLoaded: 0, - progressTotal: msg.data.progressTotal, - progressResumable: msg.data.progressResumable - } - }); - - break; - - case 'progress': - event.sender.send(data.job, { - job: data.job, - key: 'progress', - data: { - progressLoaded: msg.data.progressLoaded, - progressTotal: msg.data.progressTotal, - progressResumable: msg.data.progressResumable - } - }); - - break; - - case 'filePartUploaded': - event.sender.send(data.job, { - job: data.job, - key: 'filePartUploaded', - data: msg.data - }); - - break; - - case 'fileUploaded': - event.sender.send(data.job, { - job: data.job, - key: 'fileUploaded', - data: msg.data - }); - - forkedWorkers.delete(data.job); - setTimeout(() => { worker.kill(); }, 2000); - - break; - - case 'error': - event.sender.send(data.job, { - job: data.job, - key: 'error', - error: msg.error, - stack: msg.stack - }); - - forkedWorkers.delete(data.job); - setTimeout(() => { worker.kill(); }, 2000); - - break; - - case 'env': - // ignore - break; - - case 'debug': - console.info({upload_debug: msg}); - event.sender.send(data.job, { - job: data.job, - key: 'debug', - data: msg.data - }); - break; - - default: - event.sender.send(data.job, { - job: data.job, - key: 'unknown', - message: msg - }); - } - }); - worker.on("exit", function (code, signal) { - forkedWorkers.delete(data.job); - - if (!win) return; - - event.sender.send(data.job, { - job: data.job, - key: 'debug', - exit: { - code: code, - signal: signal - } - }); - }); - worker.on("error", function (err) { - forkedWorkers.delete(data.job); - - if (!win) return; - - event.sender.send(data.job, { - job: data.job, - key: 'error', - error: err.message, - stack: err.stack.split("\n") - }); - }); - - break; - - case "job-download": - var forkOptions = { - cwd: root, - stdio: [0, 1, 2, 'ipc'], - silent: true - }; - - var execScript = path.join(root, 'main', 'download-worker-bundle.js'); - - if (data.params.isDebug) { - event.sender.send(data.job, { - job: data.job, - key: 'debug', - env: { - fork: forkOptions, - script: execScript - } - }); - } - - var worker = fork(execScript, [], forkOptions); - forkedWorkers.set(data.job, worker); - - worker.send({ - key: "start", - data: data - }); - - worker.on("message", function (msg) { - if (!win) return; - - if (data.params.isDebug) { - event.sender.send(data.job, { - job: data.job, - key: 'debug', - message: msg - }); - } - - switch (msg.key) { - case 'fileStat': - event.sender.send(data.job, { - job: data.job, - key: 'fileStat', - data: { - progressLoaded: 0, - progressTotal: msg.data.progressTotal, - progressResumable: msg.data.progressResumable - } - }); - - break; - - case 'progress': - event.sender.send(data.job, { - job: data.job, - key: 'progress', - data: { - progressLoaded: msg.data.progressLoaded, - progressTotal: msg.data.progressTotal, - progressResumable: msg.data.progressResumable - } - }); - - break; - - case 'filePartDownloaded': - event.sender.send(data.job, { - job: data.job, - key: 'filePartDownloaded', - data: msg.data - }); - - break; - - case 'fileDownloaded': - event.sender.send(data.job, { - job: data.job, - key: 'fileDownloaded', - data: msg.data - }); - - forkedWorkers.delete(data.job); - setTimeout(() => { worker.kill(); }, 2000); - - break; - - case 'error': - event.sender.send(data.job, { - job: data.job, - key: 'error', - error: msg.error, - stack: msg.stack - }); - - forkedWorkers.delete(data.job); - setTimeout(() => { worker.kill(); }, 2000); - - break; - - case 'env': - // ignore - break; - - case 'debug': - console.info({download_debug: msg}); - event.sender.send(data.job, { - job: data.job, - key: 'debug', - data: msg.data - }); - break; - - default: - event.sender.send(data.job, { - job: data.job, - key: 'unknown', - message: msg - }); - } - }); - worker.on("exit", function (code, signal) { - forkedWorkers.delete(data.job); - - if (!win) return; - - event.sender.send(data.job, { - job: data.job, - key: 'debug', - exit: { - code: code, - signal: signal - } - }); - }); - worker.on("error", function (err) { - forkedWorkers.delete(data.job); - - if (!win) return; - - event.sender.send(data.job, { - job: data.job, - key: 'error', - error: err.message, - stack: err.stack.split("\n") - }); - }); - - break; - - case "job-stop": - var worker = forkedWorkers.get(data.job); - if (worker) { - worker.send({ - key: "stop" - }); - } - - break; - - case "job-stopall": - forkedWorkers.forEach((worker) => { - worker.send({ - key: "stop" - }); - setTimeout(() => { worker.kill(); }, 1000); - }); - - forkedWorkers = new Map(); - - break; - - default: - event.sender.send(data.job, { - job: data.job, - key: 'unknown', - data: data - }); - } -}); - -// This method will be called when Electron has finished -// initialization and is ready to create browser windows. -// Some APIs can only be used after this event occurs. -app.on("ready", () => { - // prevent Uplog always locked - // due to forcing quiting app when locked - UplogBuffer.forceUnlock() - .catch(err => { - console.warn("unlock file failed:", err); - }); - createWindow(); -}); - -app.on("activate", () => { - // On OS X it's common to recreate a window in the app when the - // dock icon is clicked and there are no other windows open. - if (win === null) { - createWindow(); - } -}); - -// Quit when all windows are closed. -app.on("window-all-closed", () => { - // On OS X it is common for applications and their menu bar - // to stay active until the user quits explicitly with Cmd + Q - //if (process.platform !== 'darwin') { - - // resolve inflight jobs persisted status not correct - setTimeout(() => { - app.quit(); - }, 3000); - //} -}); - -// In this file you can include the rest of your app's specific main process -// code. You can also put them in separate files and require them here. -function getMenuTemplate() { - return [{ - label: "Application", - submenu: [{ - label: "About Application", - selector: "orderFrontStandardAboutPanel:" - }, - { - type: "separator" - }, - { - label: "Quit", - accelerator: "Command+Q", - click: function () { - app.quit(); - } - } - ] - }, - { - label: "Edit", - submenu: [{ - label: "Undo", - accelerator: "CmdOrCtrl+Z", - selector: "undo:" - }, - { - label: "Redo", - accelerator: "Shift+CmdOrCtrl+Z", - selector: "redo:" - }, - { - type: "separator" - }, - { - label: "Cut", - accelerator: "CmdOrCtrl+X", - selector: "cut:" - }, - { - label: "Copy", - accelerator: "CmdOrCtrl+C", - selector: "copy:" - }, - { - label: "Paste", - accelerator: "CmdOrCtrl+V", - selector: "paste:" - }, - { - label: "Select All", - accelerator: "CmdOrCtrl+A", - selector: "selectAll:" - } - ] - }, - { - label: "Window", - submenu: [{ - label: "Minimize", - accelerator: "CmdOrCtrl+M", - click: function () { - win.minimize(); - } - }, - { - label: "Close", - accelerator: "CmdOrCtrl+W", - click: function () { - win.close(); - } - } - ] - } - ]; -} diff --git a/src/main/index.ts b/src/main/index.ts new file mode 100755 index 00000000..fee7e908 --- /dev/null +++ b/src/main/index.ts @@ -0,0 +1,461 @@ +import path from "path"; +import {fork, ChildProcess} from "child_process"; + +import { + app, + globalShortcut, + dialog, + Menu, + ipcMain, + BrowserWindow +} from "electron"; +import * as electronRemote from "@electron/remote/main"; +import { UplogBuffer } from "kodo-s3-adapter-sdk/dist/uplog"; + +import {UploadAction, UploadReplyMessage} from "@common/ipc-actions/upload"; +import {DownloadAction, DownloadReplyMessage} from "@common/ipc-actions/download"; + +electronRemote.initialize(); + +///***************************************** +const root = path.dirname(__dirname); +const uiRoot = path.join(root, 'renderer'); +const iconRoot = path.join(root, 'renderer', 'icons') + +// Keep a global reference of the window object, if you don't, the window will +// be closed automatically when the JavaScript object is garbage collected. +let win: BrowserWindow | null; +let winBlockedTid: number; // time interval for close +let forkedWorkers = new Map(); +let uploadRunning = 0; +let downloadRunning = 0; + +switch (process.platform) { +case "darwin": + app.dock.setIcon( + path.join(iconRoot, "icon.png") + ); + break; +case "linux": + break; +case "win32": + break; +} + +//singleton +app.requestSingleInstanceLock(); +// Someone tried to run a second instance, we should focus our window. +app.on('second-instance', (_evt, _argv, _cwd) => { + if (win) { + if (win.isMinimized()) { + win.restore(); + } + + win.focus(); + + app.quit(); + } +}); +app.releaseSingleInstanceLock(); + +let createWindow = () => { + let opt = { + width: 1280, + height: 800, + minWidth: 1280, + minHeight: 600, + title: "Kodo Browser", + icon: path.join(iconRoot, "icon.ico"), + webPreferences: { + nodeIntegration: true, + contextIsolation: false, + }, + }; + + function confirmForWorkers(e: Event) { + const runningJobs = downloadRunning + uploadRunning; + + if (runningJobs <= 0) { + return; + } + + // always stop + if (winBlockedTid) { + clearInterval(winBlockedTid); + } + + function confirmCb (btn: number) { + if (process.platform == "darwin") { + switch (btn) { + case 0: + btn = 2; + break; + + case 1: + // ignore + break; + + case 2: + btn = 0; + break; + + default: + btn = 1; + } + } + + switch (btn) { + case 0: + forkedWorkers.forEach((worker) => { + worker.kill(); + }); + + forkedWorkers = new Map(); + + break; + + case 1: + // cancel close + e.preventDefault(); + + break; + + case 2: + // cancel close + e.preventDefault(); + + clearInterval(winBlockedTid); + winBlockedTid = setInterval(() => { + if (runningJobs > 0) { + return; + } + + clearInterval(winBlockedTid); + + win?.close(); + }, 3000) as unknown as number; // hack type problem + + break; + + default: + // cancel close + e.preventDefault(); + } + } + + let btns = ["Force Quit", "Cancel", "Waiting for jobs"]; + if (process.platform == "darwin") { + btns = btns.reverse(); + } + + // prevent if there still alive workers. + confirmCb(dialog.showMessageBoxSync({ + type: "warning", + message: `There ${runningJobs > 1 ? "are" : "is"} still ${runningJobs} ${runningJobs > 1 ? "jobs" : "job"} in processing, are you sure to quit?`, + buttons: btns + })); + } + + if (process.platform == "linux") { + opt.icon = path.join(root, "src", "renderer", "icons", "icon.png"); + } + + // Create the browser window. http://electron.atom.io/docs/api/browserwindow/ + win = new BrowserWindow(opt); + win.setTitle(opt.title); + win.setMenuBarVisibility(false); + electronRemote.enable(win.webContents); + + let focused = true, shown = true; + + const registerOrUnregisterShortcutForDevTools = () => { + if (process.env.NODE_ENV != "development") { + const shortcut = 'CommandOrControl+Alt+I'; + if (shown && focused) { + if (!globalShortcut.isRegistered(shortcut)) { + globalShortcut.register(shortcut, () => { + win?.webContents.toggleDevTools(); + }); + } + } else { + if (globalShortcut.isRegistered(shortcut)) { + globalShortcut.unregister(shortcut); + } + } + } + }; + + win.on("blur", function() { + focused = false; + registerOrUnregisterShortcutForDevTools(); + }); + win.on("focus", function() { + focused = true; + registerOrUnregisterShortcutForDevTools(); + }); + win.on("show", function() { + shown = true; + registerOrUnregisterShortcutForDevTools(); + }); + win.on("hide", function() { + shown = false; + registerOrUnregisterShortcutForDevTools(); + }); + win.on("close", confirmForWorkers); + + // Emitted when the window is closed. + win.on("closed", () => { + if (forkedWorkers.size > 0) { + // force cleanup forked workers + forkedWorkers.forEach((worker) => { + worker.kill(); + }); + + forkedWorkers = new Map(); + } + + // Dereference the window object, usually you would store windows + // in an array if your app supports multi windows, this is the time + // when you should delete the corresponding element. + win = null; + }); + + // load the index.html of the app. + win.loadURL(`file://${uiRoot}/index.html`); + + if (process.env.NODE_ENV == "development") { + console.log("run in development"); + + // Open the DevTools. + win.webContents.openDevTools(); + } else if (process.platform === "darwin") { + console.log("run on macos in production"); + // Create the Application's main menu + setMenu(); + } +}; + +///***************************************** +// listener events send from renderer process +ipcMain.on("UploaderManager", (event, message) => { + const processName = "UploaderProcess"; + let uploaderProcess = forkedWorkers.get(processName); + if (!uploaderProcess) { + uploaderProcess = fork( + path.join(root, "main", "uploader-bundle.js"), + // is there a better way to pass parameters? + ['--config-json', JSON.stringify({ + resumeUpload: true, + maxConcurrency: 5, + // ... + })], + { + cwd: root, + silent: false, + }, + ); + forkedWorkers.set(processName, uploaderProcess); + + uploaderProcess.on("exit", () => { + forkedWorkers.delete(processName) + }); + + uploaderProcess.on("message", (message: UploadReplyMessage) => { + if (win && !win.isDestroyed()) { + event.sender.send("UploaderManager-reply", message); + } + switch (message.action) { + case UploadAction.UpdateUiData: { + uploadRunning = message.data.running; + break; + } + } + }); + } + + uploaderProcess.send(message); +}); + +ipcMain.on("DownloaderManager", (event, message) => { + const processName = "DownloaderProcess"; + let downloaderProcess = forkedWorkers.get(processName); + if (!downloaderProcess) { + downloaderProcess = fork( + path.join(root, "main", "downloader-bundle.js"), + // is there a better way to pass parameters? + ['--config-json', JSON.stringify({resumeUpload: true, maxConcurrency: 5})], + { + cwd: root, + silent: false, + }, + ); + forkedWorkers.set(processName, downloaderProcess); + + downloaderProcess.on("exit", () => { + forkedWorkers.delete(processName) + }); + + downloaderProcess.on("message", (message: DownloadReplyMessage) => { + if (win && !win.isDestroyed()) { + event.sender.send("DownloaderManager-reply", message); + } + switch (message.action) { + case DownloadAction.UpdateUiData: { + downloadRunning = message.data.running; + break; + } + } + }); + } + + downloaderProcess.send(message); +}); + +ipcMain.on("asynchronous", (_event, data) => { + switch (data.key) { + case "openDevTools": + win?.webContents.openDevTools(); + break; + + case "reloadWindow": + win?.webContents.reload(); + break; + + case "clearCache": + win?.webContents.session.clearCache() + .then(() => { + console.info('cache cleared'); + }); + break; + } +}); + +ipcMain.on("asynchronous-job", (event, data) => { + switch (data.key) { + case "job-stopall": { + forkedWorkers.forEach((worker) => { + worker.send({ + // TODO: both UploadAction.StopAllJobs and DownloadAction.StopAllJobs + action: "StopAllJobs", + }); + }); + break; + } + + default: + event.sender.send(data.job, { + job: data.job, + key: 'unknown', + data: data + }); + } +}); + +// This method will be called when Electron has finished +// initialization and is ready to create browser windows. +// Some APIs can only be used after this event occurs. +app.on("ready", () => { + // prevent Uplog always locked + // due to forcing quiting app when locked + UplogBuffer.forceUnlock() + .catch(err => { + console.warn("unlock file failed:", err); + }); + createWindow(); +}); + +app.on("activate", () => { + // On OS X it's common to recreate a window in the app when the + // dock icon is clicked and there are no other windows open. + if (win === null) { + createWindow(); + } +}); + +// Quit when all windows are closed. +app.on("window-all-closed", () => { + // On OS X it is common for applications and their menu bar + // to stay active until the user quits explicitly with Cmd + Q + //if (process.platform !== 'darwin') { + + // resolve inflight jobs persisted status not correct + setTimeout(() => { + app.quit(); + }, 3000); + //} +}); + +// In this file you can include the rest of your app's specific main process +// code. You can also put them in separate files and require them here. +function setMenu() { + Menu.setApplicationMenu(Menu.buildFromTemplate([ + { + label: "Application", + submenu: [{ + label: "About Application", + }, + { + type: "separator" + }, + { + label: "Quit", + accelerator: "Command+Q", + click: function () { + app.quit(); + } + } + ] + }, + { + label: "Edit", + submenu: [ + { + label: "Undo", + accelerator: "CmdOrCtrl+Z", + }, + { + label: "Redo", + accelerator: "Shift+CmdOrCtrl+Z", + }, + { + type: "separator" + }, + { + label: "Cut", + accelerator: "CmdOrCtrl+X", + }, + { + label: "Copy", + accelerator: "CmdOrCtrl+C", + }, + { + label: "Paste", + accelerator: "CmdOrCtrl+V", + }, + { + label: "Select All", + accelerator: "CmdOrCtrl+A", + } + ] + }, + { + label: "Window", + submenu: [ + { + label: "Minimize", + accelerator: "CmdOrCtrl+M", + click: function () { + win?.minimize(); + } + }, + { + label: "Close", + accelerator: "CmdOrCtrl+W", + click: function () { + win?.close(); + } + } + ] + } + ])); +} diff --git a/src/main/uploader/boundary-const.ts b/src/main/transfer-managers/boundary-const.ts similarity index 100% rename from src/main/uploader/boundary-const.ts rename to src/main/transfer-managers/boundary-const.ts diff --git a/src/main/transfer-managers/download-manager.ts b/src/main/transfer-managers/download-manager.ts new file mode 100644 index 00000000..c4550230 --- /dev/null +++ b/src/main/transfer-managers/download-manager.ts @@ -0,0 +1,291 @@ +import path from "path"; +import fs, {promises as fsPromises} from "fs"; + +import lodash from "lodash"; +import sanitizeFilename from "sanitize-filename"; +import {Adapter, ListedObjects} from "kodo-s3-adapter-sdk/dist/adapter"; + +import {ClientOptions, createQiniuClient} from "@common/qiniu"; +import DownloadJob from "@common/models/job/download-job"; +import {Status} from "@common/models/job/types"; +import {LocalPath, RemotePath} from "@common/models/job/utils"; +import {DownloadOptions, RemoteObject} from "@common/ipc-actions/download"; + +import TransferManager, {TransferManagerConfig} from "./transfer-manager"; + +interface Config { + isOverwrite: boolean; +} + +export default class DownloadManager extends TransferManager { + constructor(config: TransferManagerConfig) { + super(config); + } + + async createDownloadJobs( + remoteObjects: RemoteObject[], // remote object info, including key, size, mtime + destPath: string, // local path + clientOptions: ClientOptions, + downloadOptions: DownloadOptions, + hooks?: { + jobsAdding?: () => void, + jobsAdded?: () => void, + }, + ) { + const qiniuClient = createQiniuClient( + clientOptions, + { + userNatureLanguage: downloadOptions.userNatureLanguage, + isDebug: this.config.isDebug, + }, + ); + qiniuClient.storageClasses = downloadOptions.storageClasses; + + // iterate selected objects + for (let remoteObject of remoteObjects) { + // get remoteBaseDirectory + let remoteBaseDirectory = remoteObject.key.endsWith("/") + ? remoteObject.key.slice(0, -1) + : remoteObject.key; + remoteBaseDirectory = "/" + remoteBaseDirectory; + remoteBaseDirectory = remoteBaseDirectory.slice(1, remoteBaseDirectory.lastIndexOf("/") + 1); + + // walk remote + await this.walkRemoteObject( + qiniuClient, + downloadOptions, + remoteObject, + async (err: Error | null, walkingPath: string, walkingObject: RemoteObject) => { + if (err) { + this.config.onError?.(err); + return; + } + + let relativePath = walkingPath.slice(remoteBaseDirectory.length); + // for windows path + if (path.sep === "\\") { + relativePath = relativePath.replace("/", "\\"); + } + // sanitizeFilename + relativePath = relativePath + .split(path.sep) + .map(n => sanitizeFilename(n)) + .join(path.sep); + const localPath = destPath + relativePath; + + if (walkingObject.isDirectory) { + await this.createDirectory(localPath); + } else if (walkingObject.isFile) { + const from = { + bucket: walkingObject.bucket, + key: walkingPath, + size: walkingObject.size, + mtime: walkingObject.mtime, + }; + const to = { + name: walkingObject.name, + path: localPath, + }; + this.createDownloadJob(from, to, clientOptions, downloadOptions); + + // post add job + hooks?.jobsAdding?.(); + this.scheduleJobs(); + } + } + ); + } + hooks?.jobsAdded?.(); + } + + persistJobs(force: boolean = false): void { + if (force) { + this._persistJobs(); + return; + } + this._persistJobsThrottle(); + } + + loadJobsFromStorage( + clientOptions: Pick, + downloadOptions: Pick, + ): void { + if (!this.config.persistPath) { + return; + } + const persistedJobs: Record = + JSON.parse(fs.readFileSync(this.config.persistPath, "utf-8")); + + Object.entries(persistedJobs) + .forEach(([jobId, persistedJob]) => { + if (this.jobs.get(jobId)) { + return; + } + + const job = DownloadJob.fromPersistInfo( + jobId, + persistedJob, + { + ...clientOptions, + backendMode: persistedJob.backendMode, + }, + { + downloadSpeedLimit: this.config.speedLimit, + overwrite: this.config.isOverwrite, + isDebug: this.config.isDebug, + userNatureLanguage: downloadOptions.userNatureLanguage, + } + ); + + if ([Status.Waiting, Status.Running].includes(job.status)) { + job.stop(); + } + + this.addJob(job); + }); + } + + removeJob(jobId: string) { + this.jobs.get(jobId) + ?.stop() + ?.tryCleanupDownloadFile(); + super.removeJob(jobId); + } + + removeAllJobs() { + this.stopAllJobs(); + this.jobs.forEach(job => { + job.tryCleanupDownloadFile(); + }) + super.removeAllJobs(); + } + + private async walkRemoteObject( + client: Adapter, + downloadOptions: DownloadOptions, + remoteObject: RemoteObject, + callback: (err: Error | null, path: string, info: RemoteObject) => Promise + ): Promise { + if (remoteObject.isFile) { + await callback(null, remoteObject.key, remoteObject); + return; + } + + let nextContinuationToken: string | undefined = undefined; + // iterate fetch list + do { + // fetch list by next mark + const listedObjects: ListedObjects = await client.enter("listFiles", async client => { + return await client.listObjects( + downloadOptions.region, + downloadOptions.bucket, + remoteObject.key, + { + delimiter: "/", + minKeys: 0, + maxKeys: 1000, + nextContinuationToken, + }, + ); + }); + + // iterate objects(include current remoteObject) and callback + for (const obj of listedObjects.objects) { + const isDir = obj.key.endsWith("/"); + let name = isDir ? obj.key.slice(0, -1): obj.key; + name = obj.key.slice(name.lastIndexOf("/") + 1); + await callback(null, obj.key, { + region: downloadOptions.region, + bucket: obj.bucket, + key: obj.key, + name: name, + mtime: obj.lastModified.getTime(), + size: obj.size, + isDirectory: isDir, + isFile: !isDir, + }); + } + + // iterate subdirectories and recursive call + for (const dir of listedObjects.commonPrefixes ?? [] ){ + let name = dir.key.slice(0, -1); + name = dir.key.slice(name.lastIndexOf("/") + 1); + await this.walkRemoteObject( + client, + downloadOptions, + { + region: downloadOptions.region, + bucket: dir.bucket, + key: dir.key, + name: name, + mtime: 0, + size: 0, + isDirectory: true, + isFile: false, + }, + callback, + ); + } + + nextContinuationToken = listedObjects.nextContinuationToken; + } while (nextContinuationToken); + } + + private async createDirectory(path: string): Promise { + try { + await fsPromises.stat(path) + .then(stat => { + if (stat.isDirectory()) { + return; + } + }) + .catch(err => { + if (err.message.indexOf("EEXIST: file already exists") > -1) { + return Promise.resolve(); + } + if (err.message.indexOf("ENOENT: no such file or directory") > -1) { + return fsPromises.mkdir(path); + } + + return Promise.reject(err); + }); + } catch (err: any) { + this.config.onError?.(err); + } + } + + private createDownloadJob( + from: Required, + to: LocalPath, + clientOptions: ClientOptions, + downloadOptions: DownloadOptions, + ): void { + const job = new DownloadJob({ + from: from, + to: to, + prog: { + loaded: 0, + total: from.size, + resumable: this.config.resumable && from.size > this.config.multipartThreshold, + }, + + clientOptions: clientOptions, + storageClasses: downloadOptions.storageClasses, + + overwrite: downloadOptions.isOverwrite, + region: downloadOptions.region, + domain: downloadOptions.domain, + + multipartDownloadThreshold: this.config.multipartThreshold, + multipartDownloadSize: this.config.multipartSize, + downloadSpeedLimit: this.config.speedLimit, + isDebug: this.config.isDebug, + + userNatureLanguage: downloadOptions.userNatureLanguage, + }); + + this.addJob(job); + } + + private _persistJobsThrottle = lodash.throttle(this._persistJobs, 1000); +} diff --git a/src/main/transfer-managers/remote-walker.ts b/src/main/transfer-managers/remote-walker.ts new file mode 100644 index 00000000..3df8d887 --- /dev/null +++ b/src/main/transfer-managers/remote-walker.ts @@ -0,0 +1,19 @@ +// import {Adapter} from "kodo-s3-adapter-sdk/dist/adapter"; +// +// function CreateRemoteWalker(client: Adapter) { +// async function _walkPage() { +// +// } +// +// async function _walk() { +// +// } +// } +// +// export default class RemoteWalker { +// constructor( +// private readonly client: Adapter, +// +// ) { +// } +// } diff --git a/src/main/transfer-managers/transfer-manager.ts b/src/main/transfer-managers/transfer-manager.ts new file mode 100644 index 00000000..d0ae00d4 --- /dev/null +++ b/src/main/transfer-managers/transfer-manager.ts @@ -0,0 +1,285 @@ +import fs from "fs"; + +import TransferJob from "@common/models/job/transfer-job"; +import {isLocalPath} from "@common/models/job/utils"; + +import ByteSize from "@common/const/byte-size"; +import {Status} from "@common/models/job/types"; +import {ClientOptions} from "@common/qiniu"; + +interface OptionalConfig { + // transfer options + resumable: boolean, + maxConcurrency: number, + multipartSize: number, // Bytes + multipartThreshold: number, // Bytes + speedLimit: number, // Bytes + isDebug: boolean, + persistPath: string, + + // hooks + onError?: (err: Error) => void, + onJobDone?: (id: string, job?: Job) => void, +} + +const defaultTransferManagerConfig: OptionalConfig = { + resumable: false, + maxConcurrency: 10, + multipartThreshold: 10 * ByteSize.MB, + multipartSize: 4 * ByteSize.MB, + speedLimit: 0, // unlimited + isDebug: false, + persistPath: "", +} + +export type TransferManagerConfig = Partial> & Opt + +export default abstract class TransferManager { + abstract persistJobs(force: boolean): void + abstract loadJobsFromStorage(clientOptions: ClientOptions, options: any): void + + private running: number = 0 + protected jobs: Map = new Map() + protected jobIds: string[] = [] + protected config: OptionalConfig & Opt + + protected constructor(config: TransferManagerConfig) { + this.config = { + ...defaultTransferManagerConfig, + ...config, + }; + } + + get jobsLength() { + return this.jobs.size; + } + + get jobsSummary(): { + total: number, + finished: number, + running: number, + failed: number, + stopped: number, + } { + let finished = 0; + let failed = 0; + let stopped = 0; + this.jobIds.forEach(id => { + switch (this.jobs.get(id)?.status) { + case Status.Finished: { + finished += 1; + break; + } + case Status.Failed: { + failed += 1; + break; + } + case Status.Stopped: { + stopped += 1; + break; + } + } + }); + return { + failed, + finished, + stopped, + running: this.running, + total: this.jobsLength, + } + } + + updateConfig(config: Partial>) { + this.config = { + ...this.config, + ...config, + }; + } + + getJobsUiDataByPage( + pageNum: number = 0, + count: number = 10, + query?: { + status?: Status, + name?: string, // TODO: Compatible with upload and download + }, + ) { + let list: (Job["uiData"] | undefined)[]; + if (query) { + list = this.jobIds.map(id => this.jobs.get(id)?.uiData) + .filter(job => { + if (!job) { + return false; + } + + // status + const matchStatus: boolean = query.status + ? job.status === query.status + : true; + + // name + let matchName: boolean = true; + if (query.name) { + if (isLocalPath(job.from)) { + matchName = job.from.name.includes(query.name); + } else { + matchName = job.from.key.includes(query.name); + } + } + + // result + return matchStatus && matchName; + }) + .slice(pageNum * count, pageNum * count + count); + } else { + list = this.jobIds.slice(pageNum * count, pageNum * count + count) + .map(id => this.jobs.get(id)?.uiData); + } + return { + list, + ...this.jobsSummary, + }; + } + + waitJob(jobId: string): void { + this.jobs.get(jobId)?.wait(); + this.running -= 1; + this.scheduleJobs(); + } + + startJob(jobId: string, options?: any): void { + this.jobs.get(jobId)?.start(options); + this.running += 1; + } + + stopJob(jobId: string): void { + this.jobs.get(jobId)?.stop(); + this.running -= 1; + this.scheduleJobs(); + } + + removeJob(jobId: string): void { + const indexToRemove = this.jobIds.indexOf(jobId); + if (indexToRemove < 0) { + return; + } + this.jobs.get(jobId) + ?.stop(); + this.jobIds.splice(indexToRemove, 1); + this.jobs.delete(jobId); + this.running -= 1; + this.scheduleJobs(); + } + + cleanupJobs(): void { + const idsToRemove = this.jobIds.filter(id => this.jobs.get(id)?.status === Status.Finished); + this.jobIds = this.jobIds.filter(id => !idsToRemove.includes(id)); + idsToRemove.forEach(id => { + this.jobs.delete(id); + }); + } + + + startAllJobs(): void { + this.jobIds + .map(id => this.jobs.get(id)) + .forEach(job => { + if (!job) { + return; + } + if ([ + Status.Stopped, + Status.Failed, + ].includes(job.status)) { + job.wait(); + } + }); + this.scheduleJobs(); + } + + stopAllJobs({ + matchStatus, + }: { + matchStatus: Status[], + } = { + matchStatus: [], + }): void { + this.jobIds + .map(id => this.jobs.get(id)) + .forEach(job => { + if (!job || ![Status.Running, Status.Waiting].includes(job.status)) { + return; + } + if (!matchStatus.includes(job.status)){ + return; + } + job.stop(); + if (job.status === Status.Running) { + this.running -= 1; + } + }); + } + + removeAllJobs(): void { + this.stopAllJobs(); + this.jobIds = []; + this.jobs.clear(); + } + + protected _persistJobs(): void { + if (!this.config.persistPath) { + return; + } + const persistData: Record = {}; + this.jobIds.forEach(id => { + const job = this.jobs.get(id); + if (!job || job.status === Status.Finished) { + return; + } + persistData[id] = job.persistInfo; + }); + fs.writeFileSync( + this.config.persistPath, + JSON.stringify(persistData), + ); + } + + protected addJob(job: Job) { + this.jobs.set(job.id, job); + this.jobIds.push(job.id); + } + + protected scheduleJobs(): void { + if (this.config.isDebug) { + console.log(`[JOB] max: ${this.config.maxConcurrency}, cur: ${this.running}, jobs: ${this.jobIds.length}`); + } + + this.running = Math.max(0, this.running); + if (this.running >= this.config.maxConcurrency) { + return; + } + + for (let i = 0; i < this.jobIds.length; i++) { + const job = this.jobs.get(this.jobIds[i]); + if (job?.status !== Status.Waiting) { + continue; + } + this.running += 1; + job.start() + .finally(() => { + this.afterJobDone(job.id); + }); + + this.running = Math.max(0, this.running); + if (this.running >= this.config.maxConcurrency) { + return; + } + } + } + + private afterJobDone(id: string): void { + this.running -= 1; + this.scheduleJobs(); + this.config.onJobDone?.(id, this.jobs.get(id)); + } +} diff --git a/src/main/uploader/upload-manager.ts b/src/main/transfer-managers/upload-manager.ts similarity index 53% rename from src/main/uploader/upload-manager.ts rename to src/main/transfer-managers/upload-manager.ts index cc6692fa..40614eab 100644 --- a/src/main/uploader/upload-manager.ts +++ b/src/main/transfer-managers/upload-manager.ts @@ -1,120 +1,46 @@ import path from "path"; import fs, {Stats} from "fs"; + import lodash from "lodash"; // @ts-ignore import Walk from "@root/walk"; import {Adapter} from "kodo-s3-adapter-sdk/dist/adapter"; -import ByteSize from "@common/const/byte-size"; -import {ClientOptions, DestInfo, UploadOptions} from "@common/ipc-actions/upload"; +import {ClientOptions, createQiniuClient} from "@common/qiniu"; +import {DestInfo, UploadOptions} from "@common/ipc-actions/upload"; import UploadJob from "@common/models/job/upload-job"; import {Status} from "@common/models/job/types"; -import createQiniuClient from "../util/createClient"; import {MAX_MULTIPART_COUNT, MIN_MULTIPART_SIZE} from "./boundary-const"; +import TransferManager, {TransferManagerConfig} from "./transfer-manager"; // for walk interface StatsWithName extends Stats { name: string, } -// Manager -interface ManagerConfig { - resumeUpload: boolean, - maxConcurrency: number, - multipartUploadSize: number, // Bytes - multipartUploadThreshold: number, // Bytes - uploadSpeedLimit: number, // Bytes/s - isDebug: boolean, - isSkipEmptyDirectory: boolean - persistPath: string, - - onError?: (err: Error) => void, - onJobDone?: (id: string, job?: UploadJob) => void, - onCreatedDirectory?: (bucket: string, directoryKey: string) => void, -} +interface Config { + isSkipEmptyDirectory: boolean; -const defaultManagerConfig: ManagerConfig = { - resumeUpload: false, - maxConcurrency: 10, - multipartUploadSize: 4 * ByteSize.MB, // 4MB - multipartUploadThreshold: 10 * ByteSize.MB, // 10MB - uploadSpeedLimit: 0, - isDebug: false, - isSkipEmptyDirectory: false, - persistPath: "", + onCreatedDirectory?: (bucket: string, directoryKey: string) => void, } -export default class UploadManager { - private concurrency: number = 0; - private jobs: Map = new Map() - private jobIds: UploadJob["id"][] = [] - private config: Readonly - - constructor(config: Partial) { - this.config = { - ...defaultManagerConfig, - ...config, - }; - } - - get jobsLength() { - return this.jobIds.length; - } - - get jobsSummary(): { - total: number, - finished: number, - running: number, - failed: number, - stopped: number, - } { - let finished = 0; - let failed = 0; - let stopped = 0; - this.jobIds.forEach((id) => { - switch (this.jobs.get(id)?.status) { - case Status.Finished: { - finished += 1; - break; - } - case Status.Failed: { - failed += 1; - break; - } - case Status.Stopped: { - stopped += 1; - break; - } - } - }); - return { - total: this.jobIds.length, - finished: finished, - running: this.concurrency, - failed: failed, - stopped: stopped, - } - } - - updateConfig(config: Partial) { - this.config = { - ...this.config, - ...config, - }; +export default class UploadManager extends TransferManager { + constructor(config: TransferManagerConfig) { + super(config); } async createUploadJobs( filePathnameList: string[], // local file path, required absolute path destInfo: DestInfo, - uploadOptions: UploadOptions, clientOptions: ClientOptions, + uploadOptions: UploadOptions, hooks?: { jobsAdding?: () => void, jobsAdded?: () => void, }, ) { - const qiniu = createQiniuClient( + const qiniuClient = createQiniuClient( clientOptions, { userNatureLanguage: uploadOptions.userNatureLanguage, @@ -137,7 +63,7 @@ export default class UploadManager { async (err: Error, walkingPathname: string, statsWithName: StatsWithName): Promise => { if (err) { this.config.onError?.(err); - return + return; } // remoteKey should be "path/to/file" not "/path/to/file" @@ -152,7 +78,7 @@ export default class UploadManager { const remoteDirectoryKey = path.dirname(remoteKey) + "/"; if (remoteDirectoryKey !== "./" && !directoryToCreate.get(remoteDirectoryKey)) { this.createDirectory( - qiniu, + qiniuClient, { region: destInfo.regionId, bucketName: destInfo.bucketName, @@ -172,7 +98,7 @@ export default class UploadManager { if (!this.config.isSkipEmptyDirectory) { const remoteDirectoryKey = remoteKey + "/"; this.createDirectory( - qiniu, + qiniuClient, { region: destInfo.regionId, bucketName: destInfo.bucketName, @@ -194,7 +120,7 @@ export default class UploadManager { bucket: destInfo.bucketName, key: remoteKey, }; - this.createUploadJob(from, to, uploadOptions, clientOptions, destInfo.regionId); + this.createUploadJob(from, to, clientOptions, uploadOptions, destInfo.regionId); // post add job hooks?.jobsAdding?.(); @@ -240,15 +166,15 @@ export default class UploadManager { private createUploadJob( from: Required, to: UploadJob["options"]["to"], - uploadOptions: UploadOptions, clientOptions: ClientOptions, + uploadOptions: UploadOptions, regionId: string, ): void { // parts count - const partsCount = Math.ceil(from.size / this.config.multipartUploadSize); + const partsCount = Math.ceil(from.size / this.config.multipartSize); // part size - let partSize = this.config.multipartUploadSize; + let partSize = this.config.multipartSize; if (partsCount > MAX_MULTIPART_COUNT) { partSize = Math.ceil(from.size / MAX_MULTIPART_COUNT); if (partSize < MIN_MULTIPART_SIZE) { @@ -265,77 +191,35 @@ export default class UploadManager { prog: { loaded: 0, total: from.size, - resumable: this.config.resumeUpload && from.size > this.config.multipartUploadThreshold, + resumable: this.config.resumable && from.size > this.config.multipartThreshold, }, - clientOptions: { - accessKey: clientOptions.accessKey, - secretKey: clientOptions.secretKey, - ucUrl: clientOptions.ucUrl, - regions: clientOptions.regions, - backendMode: clientOptions.backendMode, - }, + clientOptions, storageClasses: uploadOptions.storageClasses, overwrite: uploadOptions.isOverwrite, region: regionId, storageClassName: uploadOptions.storageClassName, - multipartUploadThreshold: this.config.multipartUploadThreshold, + multipartUploadThreshold: this.config.multipartThreshold, multipartUploadSize: partSize, - uploadSpeedLimit: this.config.uploadSpeedLimit, + uploadSpeedLimit: this.config.speedLimit, isDebug: this.config.isDebug, userNatureLanguage: uploadOptions.userNatureLanguage, - }); - - this.addJob(job); - } - private addJob(job: UploadJob) { - job.on("partcomplete", () => { - this.persistJobs(); - return false; - }); - job.on("complete", () => { - this.persistJobs(); - return false; + onProgress: () => { + this.persistJobs(); + }, + onPartCompleted: () => { + this.persistJobs(); + }, + onCompleted: () => { + this.persistJobs(); + } }); - this.jobs.set(job.id, job); - this.jobIds.push(job.id); - } - - public getJobsUiDataByPage(pageNum: number = 0, count: number = 10, query?: { status?: Status, name?: string }) { - let list: (UploadJob["uiData"] | undefined)[]; - if (query) { - list = this.jobIds.map(id => this.jobs.get(id)?.uiData) - .filter(job => { - const matchStatus = query.status - ? job?.status === query.status - : true; - const matchName = query.name - ? job?.from.name.includes(query.name) - : true; - return matchStatus && matchName; - }) - .slice(pageNum, pageNum * count + count); - } else { - list = this.jobIds.slice(pageNum, pageNum * count + count) - .map(id => this.jobs.get(id)?.uiData); - } - return { - list, - ...this.jobsSummary, - }; - } - - public getJobsUiDataByIds(ids: UploadJob["id"][]) { - return { - list: ids.filter(id => this.jobs.has(id)) - .map(id => this.jobs.get(id)?.uiData), - ...this.jobsSummary, - }; + this.addJob(job); } public persistJobs(force: boolean = false): void { @@ -346,38 +230,19 @@ export default class UploadManager { this._persistJobsThrottle(); } - private _persistJobsThrottle = lodash.throttle(this._persistJobs, 1000); - - private _persistJobs(): void { - if (!this.config.persistPath) { - return; - } - const persistData: Record = {}; - this.jobIds.forEach(id => { - const job = this.jobs.get(id); - if (!job || job.status === Status.Finished) { - return; - } - persistData[id] = job.persistInfo; - }); - fs.writeFileSync( - this.config.persistPath, - JSON.stringify(persistData), - ); - } - public loadJobsFromStorage( clientOptions: Pick, - uploadOptions: Pick + uploadOptions: Pick, ): void { if (!this.config.persistPath) { return; } - const persistedJobs: Record = JSON.parse(fs.readFileSync(this.config.persistPath, "utf-8")); + const persistedJobs: Record = + JSON.parse(fs.readFileSync(this.config.persistPath, "utf-8")); Object.entries(persistedJobs) .forEach(([jobId, persistedJob]) => { if (this.jobs.get(jobId)) { - return + return; } if (!persistedJob.from) { @@ -409,7 +274,7 @@ export default class UploadManager { } // resumable - persistedJob.prog.resumable = this.config.resumeUpload && persistedJob.from.size > this.config.multipartUploadThreshold; + persistedJob.prog.resumable = this.config.resumable && persistedJob.from.size > this.config.multipartThreshold; const job = UploadJob.fromPersistInfo( jobId, @@ -419,7 +284,7 @@ export default class UploadManager { backendMode: persistedJob.backendMode, }, { - uploadSpeedLimit: this.config.uploadSpeedLimit, + uploadSpeedLimit: this.config.speedLimit, isDebug: this.config.isDebug, userNatureLanguage: uploadOptions.userNatureLanguage, }, @@ -433,113 +298,7 @@ export default class UploadManager { }); } - public waitJob(jobId: string): void { - this.jobs.get(jobId)?.wait(); - this.scheduleJobs(); - } - - public startJob(jobId: string, forceOverwrite: boolean = false): void { - this.jobs.get(jobId)?.start(forceOverwrite); - } - - public stopJob(jobId: string): void { - this.jobs.get(jobId)?.stop(); - } - - public removeJob(jobId: string): void { - const indexToRemove = this.jobIds.indexOf(jobId); - if (indexToRemove < 0) { - return; - } - this.jobs.get(jobId)?.stop(); - this.jobIds.splice(indexToRemove, 1); - this.jobs.delete(jobId); - } - - public cleanupJobs(): void { - const idsToRemove = this.jobIds.filter(id => this.jobs.get(id)?.status === Status.Finished); - this.jobIds = this.jobIds.filter(id => !idsToRemove.includes(id)); - idsToRemove.forEach(id => { - this.jobs.delete(id); - }); - } - - public startAllJobs(): void { - this.jobIds - .map(id => this.jobs.get(id)) - .forEach(job => { - if (!job) { - return; - } - if ([ - Status.Stopped, - Status.Failed, - ].includes(job.status)) { - job.wait(); - } - }); - this.scheduleJobs(); - } - - public stopAllJobs({ - matchStatus, - }: { - matchStatus: Status[], - } = { - matchStatus: [], - }): void { - this.jobIds - .map(id => this.jobs.get(id)) - .forEach(job => { - if (!job || ![Status.Running, Status.Waiting].includes(job.status)) { - return; - } - if (!matchStatus.includes(job.status)){ - return; - } - job.stop(); - }); - } - - public removeAllJobs(): void { - this.stopAllJobs(); - this.jobIds = []; - this.jobs.clear(); - } - - private scheduleJobs(): void { - if (this.config.isDebug) { - console.log(`[JOB] upload max: ${this.config.maxConcurrency}, cur: ${this.concurrency}, jobs: ${this.jobIds.length}`); - } - - this.concurrency = Math.max(0, this.concurrency); - if (this.concurrency >= this.config.maxConcurrency) { - return; - } - - for (let i = 0; i < this.jobIds.length; i++) { - const job = this.jobs.get(this.jobIds[i]); - if (job?.status !== Status.Waiting) { - continue; - } - this.concurrency += 1; - job.start() - .finally(() => { - this.afterJobDone(job.id); - }); - - this.concurrency = Math.max(0, this.concurrency); - if (this.concurrency >= this.config.maxConcurrency) { - return; - } - } - } - - private afterJobDone(id: UploadJob["id"]): void { - this.concurrency -= 1; - this.scheduleJobs(); - this.config.onJobDone?.(id, this.jobs.get(id)); - } + private _persistJobsThrottle = lodash.throttle(this._persistJobs, 1000); private afterCreateDirectory({ bucket, diff --git a/src/main/uploader/index.ts b/src/main/upload-worker.ts similarity index 96% rename from src/main/uploader/index.ts rename to src/main/upload-worker.ts index e7a4e9f5..6e8d22db 100644 --- a/src/main/uploader/index.ts +++ b/src/main/upload-worker.ts @@ -6,7 +6,7 @@ import { UploadAction, UploadMessage } from "@common/ipc-actions/upload"; -import UploadManager from "./upload-manager"; +import UploadManager from "./transfer-managers/upload-manager"; import UploadJob from "@common/models/job/upload-job"; import {Status} from "@common/models/job/types"; @@ -39,8 +39,8 @@ process.on("message", (message: UploadMessage) => { uploadManager.createUploadJobs( message.data.filePathnameList, message.data.destInfo, - message.data.uploadOptions, message.data.clientOptions, + message.data.uploadOptions, { jobsAdding: () => { uploadManager.persistJobs(); @@ -52,10 +52,10 @@ process.on("message", (message: UploadMessage) => { filePathnameList: message.data.filePathnameList, destInfo: message.data.destInfo, }, - } + }; process.send?.(replyMessage); - } - } + }, + }, ); break; } @@ -80,7 +80,7 @@ process.on("message", (message: UploadMessage) => { break; } case UploadAction.StartJob: { - uploadManager.startJob(message.data.jobId, message.data.forceOverwrite); + uploadManager.startJob(message.data.jobId, message.data.options); break; } case UploadAction.RemoveJob: { @@ -102,7 +102,7 @@ process.on("message", (message: UploadMessage) => { } case UploadAction.RemoveAllJobs: { uploadManager.removeAllJobs(); - uploadManager.persistJobs(); + uploadManager.persistJobs(true); break; } default: { @@ -141,7 +141,7 @@ function handleExit() { process.on("exit", () => { - handleExit() + handleExit(); }); process.on("SIGTERM", () => { diff --git a/src/renderer/components/services/delay-done.js b/src/renderer/components/services/delay-done.js deleted file mode 100755 index 68c28012..00000000 --- a/src/renderer/components/services/delay-done.js +++ /dev/null @@ -1,63 +0,0 @@ -import webModule from '@/app-module/web' - -const DELAY_DONE_FACTORY_NAME = 'DelayDone' - -webModule.factory(DELAY_DONE_FACTORY_NAME, [ - "$timeout", - function ($timeout) { - var mDelayCall = {}; - - return { - delayRun: delayRun, - seriesRun: seriesRun - }; - - /** - * @param id {String} uniq - * @param timeout {int} ms - * @param times {int} 超过次数也会调, 然后重新统计 - * @param fn {Function} callback - */ - - function delayRun(id, timeout, fn, times) { - if (!mDelayCall[id]) - mDelayCall[id] = { - tid: "", - c: 0 - }; - var n = mDelayCall[id]; - - n.c++; - - if (n.c >= times) { - fn(); - n.c = 0; - } else { - $timeout.cancel(n.tid); - n.tid = $timeout(fn, timeout); - } - } - - function seriesRun(arr, fn, doneFn) { - var len = arr.length; - var c = 0; - - function _dig() { - var n = arr[c]; - fn(n, function () { - c++; - if (c >= len) { - doneFn(); - } else { - $timeout(function () { - _dig(); - }, 1); - } - }); - } - _dig(); - } - } -]); - -export default DELAY_DONE_FACTORY_NAME diff --git a/src/renderer/components/services/download-manager.js b/src/renderer/components/services/download-manager.js deleted file mode 100755 index c2e30519..00000000 --- a/src/renderer/components/services/download-manager.js +++ /dev/null @@ -1,472 +0,0 @@ -import fs from 'fs' -import path from 'path' - -import sanitize from 'sanitize-filename' -import { KODO_MODE } from 'kodo-s3-adapter-sdk' - -import { DownloadJob } from '@common/models/job' -import webModule from '@/app-module/web' - -import NgConfig from '@/ng-config' -import * as AuthInfo from './authinfo' -import NgQiniuClient from './ng-qiniu-client' -import { TOAST_FACTORY_NAME as Toast } from '../directives/toast-list' -import Settings from './settings.ts' - -const DOWNLOAD_MGR_FACTORY_NAME = 'DownloadMgr' - -webModule.factory(DOWNLOAD_MGR_FACTORY_NAME, [ - "$q", - "$timeout", - '$translate', - NgQiniuClient, - NgConfig, - Toast, - function ( - $q, - $timeout, - $translate, - QiniuClient, - Config, - Toast, - ) { - const T = $translate.instant - const pfs = fs.promises - - var $scope; - var concurrency = 0; - var stopCreatingFlag = false; - - return { - init: init, - createDownloadJobs: createDownloadJobs, - trySchedJob: trySchedJob, - trySaveProg: trySaveProg, - - stopCreatingJobs: () => { - stopCreatingFlag = true; - } - }; - - function init(scope) { - $scope = scope; - $scope.lists.downloadJobList = []; - - tryLoadProg().then((progs) => { - angular.forEach(progs, (prog) => { - const job = createJob(prog); - if (job.status === "waiting" || job.status === "running") { - job.stop(); - } - addEvents(job); - }); - }); - } - - /** - * @param briefJob { object: { options: {region, from, to,}}} - * @param briefJob.from {bucket, key} - * @param briefJob.to {name, path} - * @return job { start(), stop(), status, progress } - */ - function createJob(briefJob) { - const { options } = briefJob; - const bucket = options.from.bucket, - key = options.from.key, - region = options.region, - domain = options.domain; - - console.info( - "GET", - "::", - region, - "::", - bucket + "/" + key, - "==>", - options.to.path + "/" + options.to.name - ); - - const config = Config.load(); - - options.clientOptions = { - accessKey: AuthInfo.get().id, - secretKey: AuthInfo.get().secret, - ucUrl: config.ucUrl || "", - regions: config.regions || [], - }; - options.region = region; - options.domain = domain; - options.resumeDownload = (Settings.resumeDownload === 1); - options.multipartDownloadThreshold = Settings.multipartDownloadThreshold; - options.multipartDownloadSize = Settings.multipartDownloadSize; - options.downloadSpeedLimit = Settings.downloadSpeedLimitEnabled === 1 - ? Settings.downloadSpeedLimitKBperSec - : 0; - options.isDebug = (Settings.isDebug === 1); - options.storageClasses = options.storageClasses || []; - options.userNatureLanguage = localStorage.getItem('lang') || 'zh-CN'; - - return new DownloadJob(options); - } - - /** - * 下载 - * @param bucketInfos {array} item={region, bucket, path, name, size=0, itemType='file'} 有可能是目录，需要遍历 - * @param toLocalPath {string} - * @param jobsAddedFn {Function} 加入列表完成回调方法， jobs列表已经稳定 - */ - function createDownloadJobs(bucketInfos, toLocalPath, jobsAddedFn) { - stopCreatingFlag = false; - - const dirPath = bucketInfos[0].path.parentDirectoryPath(); - - loop(bucketInfos, (jobs) => {}, () => { - if (jobsAddedFn) { - jobsAddedFn(); - } - }); - - function loop(arr, callFn, callFn2) { - const t = []; - const len = arr.length; - let c = 0; - let c2 = 0; - - if (len == 0) { - callFn(t); - callFn2(t); - return; - } - - _kdig(); - - function _kdig() { - dig(arr[c], t, () => { - - }, () => { - c2++; - if (c2 >= len) { - callFn2(t); - } - }); - - c++; - if (c == len) { - callFn(t); - } else { - if (stopCreatingFlag) { - return; - } - - $timeout(_kdig, 0); - } - } - } - - function dig(qiniuInfo, t, callFn, callFn2) { - if (stopCreatingFlag) { - return; - } - - const fileName = sanitize(qiniuInfo.path.basename() || qiniuInfo.path.directoryBasename()); - let filePath = ''; - if (path.sep == '\\') { - angular.forEach(path.relative(dirPath.toString().replace(/\\/g, '/'), qiniuInfo.path.toString()).replace(/\\/g, '/').split('/'), (folder) => { - filePath = path.join(filePath, sanitize(folder)); - }); - } else { - angular.forEach(path.relative(dirPath.toString(), qiniuInfo.path.toString()).split('/'), (folder) => { - filePath = path.join(filePath, sanitize(folder)); - }); - } - - if (qiniuInfo.itemType === 'folder') { - // list all files under qiniuInfo.path - function tryLoadFiles(marker) { - QiniuClient - .listFiles(qiniuInfo.region, qiniuInfo.bucket, qiniuInfo.path.toString(), marker, { - maxKeys: 1000, - minKeys: 0, - storageClasses: $scope.currentInfo.availableStorageClasses, - }) - .then((result) => { - var files = result.data; - files.forEach((f) => { - f.region = qiniuInfo.region; - f.bucket = qiniuInfo.bucket; - f.domain = qiniuInfo.domain; - f.qiniuBackendMode = qiniuInfo.qiniuBackendMode; - }); - - loop(files, (jobs) => { - t = t.concat(jobs); - if (result.marker) { - $timeout(() => { tryLoadFiles(result.marker); }, 10); - } else { - if (callFn) callFn(); - } - }, callFn2); - }); - } - - if (!qiniuInfo.path.directoryBasename()) { - Toast.error(T('download.emptyNameFolder.forbidden', { path: qiniuInfo.path.toString() })); - return; - } - - tryLoadFiles(); - } else { - let fileFolders = ''; - if (path.sep == '\\') { - fileFolders = path.dirname(filePath.replace(/\\/g, '/')).split('/'); - } else { - fileFolders = path.dirname(filePath.replace(path.sep, '/')).split('/'); - } - - fileFolders.reduce((prevPromise, folder) => { - return prevPromise.then((localFolder) => { - const absfolder = localFolder.joinFolder(folder); - - return pfs.stat(absfolder.toString()).then((stat) => { - if (stat.isDirectory()) { - return Promise.resolve(absfolder); - } - - return pfs.mkdir(absfolder.toString()).then(() => { - return Promise.resolve(absfolder); - }).catch((err) => { - if (err.message.indexOf('EEXIST: file already exists') > -1) { - return Promise.resolve(absfolder); - } - - throw err; - }); - }).catch((err) => { - return pfs.mkdir(absfolder.toString()).then(() => { - return Promise.resolve(absfolder); - }).catch((err) => { - if (err.message.indexOf('EEXIST: file already exists') > -1) { - return Promise.resolve(absfolder); - } - - throw err; - }); - }); - }); - }, Promise.resolve(toLocalPath)).then((localPath) => { - const ext = path.extname(fileName); - const fileLocalPathWithoutExt = path.normalize(localPath.joinFile(path.basename(fileName, ext)).toString()); - let fileLocalPathWithSuffixWithoutExt = fileLocalPathWithoutExt - - if (!$scope.overwriteDownloading.enabled) { - for (let i = 1; fs.existsSync(fileLocalPathWithSuffixWithoutExt + ext); i++) { - fileLocalPathWithSuffixWithoutExt = `${fileLocalPathWithoutExt}.${i}`; - } - } - - const job = createJob({ - options: { - region: qiniuInfo.region, - from: { - bucket: qiniuInfo.bucket, - key: qiniuInfo.path.toString(), - size: qiniuInfo.size, - mtime: qiniuInfo.lastModified.getTime(), - }, - to: { - name: fileName, - path: fileLocalPathWithSuffixWithoutExt + ext - }, - domain: qiniuInfo.domain.toQiniuDomain(), - storageClasses: $scope.currentInfo.availableStorageClasses, - backendMode: qiniuInfo.domain.qiniuBackendMode(), - }, - }); - addEvents(job); - t.push(job); - - if (callFn) callFn(); - if (callFn2) callFn2(); - }); - } - } - } - - function addEvents(job) { - - $scope.lists.downloadJobList.push(job); - - trySchedJob(); - trySaveProg(); - - $timeout(() => { - $scope.calcTotalProg(); - }); - - job.on("partcomplete", (prog) => { - trySaveProg(); - }); - job.on("statuschange", (status) => { - if (status == "stopped") { - concurrency--; - trySchedJob(); - } - - trySaveProg(); - - $timeout(() => { - $scope.calcTotalProg(); - }); - }); - job.on("speedchange", () => { - $timeout(() => { - $scope.calcTotalProg(); - }); - }); - job.on("complete", () => { - concurrency--; - trySchedJob(); - - $timeout(() => { - $scope.calcTotalProg(); - }); - }); - job.on("error", (err) => { - if (err) { - console.error(`download kodo://${job.options.from.bucket}/${job.options.from.key} error: ${err}`); - } - - concurrency--; - trySchedJob(); - - $timeout(() => { - $scope.calcTotalProg(); - }); - }); - } - - function trySchedJob() { - var maxConcurrency = Settings.maxDownloadConcurrency; - var isDebug = (Settings.isDebug === 1); - - concurrency = Math.max(0, concurrency); - if (isDebug) { - console.log(`[JOB] download max: ${maxConcurrency}, cur: ${concurrency}, jobs: ${$scope.lists.downloadJobList.length}`); - } - - if (concurrency < maxConcurrency) { - const jobs = $scope.lists.downloadJobList; - - const startAllJobsFrom = (i) => { - if (i >= jobs.length) { - return; - } - if (concurrency >= maxConcurrency) { - return; - } - - const job = jobs[i]; - if (isDebug) { - console.log('[JOB] sched ', job.status, ' => ', job.options); - } - if (job.status === "waiting") { - concurrency++; - - if (job.prog.resumable) { - tryLoadProgForJob(job).then((job) => { - job.start(job && job.prog); - }).finally(() => { - startAllJobsFrom(i + 1); - }); - return; - } else { - job.start(); - } - } - $timeout(() => { startAllJobsFrom(i + 1); }); - }; - startAllJobsFrom(0); - } - } - - function trySaveProg() { - var t = {}; - angular.forEach($scope.lists.downloadJobList, function (job) { - if (job.status === "finished") return; - - t[job.id] = job.getInfoForSave(); - }); - - fs.writeFileSync(getDownProgFilePath(), JSON.stringify(t)); - } - - /** - * resolve prog saved - */ - function tryLoadProg() { - let progs = {}; - try { - const data = fs.readFileSync(getDownProgFilePath()); - progs = JSON.parse(data); - } catch (e) {} - - if (!progs) { - return Promise.resolve([]); - } - - const promises = Object.values(progs) - .map(briefJob => ({ - options: briefJob, - })) - .map(jobOptions => tryLoadProgForJob(jobOptions)); - return Promise.all(promises); - } - - function tryLoadProgForJob(job) { - return new Promise((resolve) => { - // next block within `if` handle persist job < v1.0.16 - if (job.prog && job.prog.synced) { - job.prog.loaded = job.prog.synced; - delete job.prog.synced; - - job.from.mtime = new Date(job.from.mtime).getTime(); - } - const options = { - ignoreError: true, - storageClasses: job.storageClasses || [], - }; - if (job.options.backendMode === KODO_MODE) { - options.preferKodoAdapter = true; - } else { - options.preferS3Adapter = true; - } - QiniuClient.headFile(job.options.region, job.options.from.bucket, job.options.from.key, options).then((info) => { - if (info.size !== job.options.from.size || info.lastModified.getTime() !== job.options.from.mtime) { - if (job.prog) { - job.prog.loaded = 0; - } - } - resolve(job); - }).catch(() => { - if (job.prog) { - job.prog.loaded = 0; - } - resolve(job); - }); - }); - } - - // prog save path - function getDownProgFilePath() { - var folder = Global.config_path; - if (!fs.existsSync(folder)) { - fs.mkdirSync(folder); - } - - var username = AuthInfo.get().id || "kodo-browser"; - return path.join(folder, "downprog_" + username + ".json"); - } - } -]); - -export default DOWNLOAD_MGR_FACTORY_NAME diff --git a/src/renderer/components/services/index.js b/src/renderer/components/services/index.js index 26a54018..3e32ca11 100644 --- a/src/renderer/components/services/index.js +++ b/src/renderer/components/services/index.js @@ -4,11 +4,9 @@ import './auth' import './auto-upgrade' import './bookmark.ts' import './cipher.ts' -import './delay-done' import './dialog.s' import './diff-modal' import './domains' -import './download-manager' import './external-path' import './file.s' import './job-util' diff --git a/src/renderer/components/services/ipc-download-manager.ts b/src/renderer/components/services/ipc-download-manager.ts new file mode 100644 index 00000000..3ceb34b9 --- /dev/null +++ b/src/renderer/components/services/ipc-download-manager.ts @@ -0,0 +1,7 @@ +import {ipcRenderer} from "electron"; + +import {DownloadActionFns} from "@common/ipc-actions/download"; + +const ipcDownloadManager = new DownloadActionFns(ipcRenderer, "DownloaderManager"); + +export default ipcDownloadManager; diff --git a/src/renderer/components/services/settings.ts b/src/renderer/components/services/settings.ts index 2a16b024..40125b98 100755 --- a/src/renderer/components/services/settings.ts +++ b/src/renderer/components/services/settings.ts @@ -1,5 +1,6 @@ import ByteSize from "@common/const/byte-size"; import ipcUploadManager from "@/components/services/ipc-upload-manager"; +import ipcDownloadManager from "@/components/services/ipc-download-manager"; export enum SettingKey { IsDebug = "isDebug", @@ -37,6 +38,9 @@ class Settings { ipcUploadManager.updateConfig({ isDebug: v !== 0, }); + ipcDownloadManager.updateConfig({ + isDebug: v !== 0, + }) } // autoUpgrade @@ -54,7 +58,7 @@ class Settings { set resumeUpload(v: number) { localStorage.setItem(SettingKey.ResumeUpload, v.toString()); ipcUploadManager.updateConfig({ - resumeUpload: v !== 0, + resumable: v !== 0, }); } @@ -77,7 +81,7 @@ class Settings { if (v >= 1 && v <= 1024) { localStorage.setItem(SettingKey.MultipartUploadSize, v.toString()); ipcUploadManager.updateConfig({ - multipartUploadSize: v * ByteSize.MB, + multipartSize: v * ByteSize.MB, }); } } @@ -89,7 +93,7 @@ class Settings { set multipartUploadThreshold(v: number) { localStorage.setItem(SettingKey.MultipartUploadThreshold, v.toString()); ipcUploadManager.updateConfig({ - multipartUploadThreshold: v * ByteSize.MB, + multipartThreshold: v * ByteSize.MB, }); } @@ -101,11 +105,11 @@ class Settings { localStorage.setItem(SettingKey.UploadSpeedLimitEnabled, v.toString()); if (v === 0) { ipcUploadManager.updateConfig({ - uploadSpeedLimit: 0, + speedLimit: 0, }); } else { ipcUploadManager.updateConfig({ - uploadSpeedLimit: this.uploadSpeedLimitKBperSec * ByteSize.KB, + speedLimit: this.uploadSpeedLimitKBperSec * ByteSize.KB, }); } } @@ -118,7 +122,7 @@ class Settings { localStorage.setItem(SettingKey.UploadSpeedLimit, v.toString()); if (this.uploadSpeedLimitEnabled) { ipcUploadManager.updateConfig({ - uploadSpeedLimit: v * ByteSize.KB, + speedLimit: v * ByteSize.KB, }); } } @@ -129,6 +133,11 @@ class Settings { } set downloadSpeedLimitKBperSec(v: number) { localStorage.setItem(SettingKey.DownloadSpeedLimit, v.toString()); + if (this.downloadSpeedLimitEnabled){ + ipcDownloadManager.updateConfig({ + speedLimit: v * ByteSize.KB, + }); + } } // resumeDownload @@ -137,6 +146,9 @@ class Settings { } set resumeDownload(v: number) { localStorage.setItem(SettingKey.ResumeDownload, v.toString()); + ipcDownloadManager.updateConfig({ + resumable: v !== 0, + }); } // maxDownloadConcurrency @@ -145,6 +157,9 @@ class Settings { } set maxDownloadConcurrency(v: number) { localStorage.setItem(SettingKey.MaxDownloadConcurrency, v.toString()); + ipcDownloadManager.updateConfig({ + maxConcurrency: v, + }); } // multipartDownloadSize @@ -153,6 +168,9 @@ class Settings { } set multipartDownloadSize(v: number) { localStorage.setItem(SettingKey.MultipartDownloadSize, v.toString()); + ipcDownloadManager.updateConfig({ + multipartSize: v * ByteSize.MB, + }); } // multipartDownloadThreshold @@ -161,6 +179,9 @@ class Settings { } set multipartDownloadThreshold(v: number) { localStorage.setItem(SettingKey.MultipartDownloadThreshold, v.toString()); + ipcDownloadManager.updateConfig({ + multipartThreshold: v * ByteSize.MB, + }); } // downloadSpeedLimitEnabled @@ -169,6 +190,15 @@ class Settings { } set downloadSpeedLimitEnabled(v: number) { localStorage.setItem(SettingKey.DownloadSpeedLimitEnabled, v.toString()); + if (v === 0) { + ipcDownloadManager.updateConfig({ + speedLimit: 0, + }); + } else { + ipcDownloadManager.updateConfig({ + speedLimit: this.downloadSpeedLimitKBperSec * ByteSize.KB, + }); + } } // externalPathEnabled diff --git a/src/renderer/main/files/_/file-list.html b/src/renderer/main/files/_/file-list.html index a2e19350..b88ac8b0 100755 --- a/src/renderer/main/files/_/file-list.html +++ b/src/renderer/main/files/_/file-list.html @@ -1,5 +1,5 @@
      -
      +
      diff --git a/src/renderer/main/files/files.js b/src/renderer/main/files/files.js index abd49015..e32db4e1 100755 --- a/src/renderer/main/files/files.js +++ b/src/renderer/main/files/files.js @@ -1201,7 +1201,7 @@ webModule.controller(FILES_CONTROLLER_NAME, [ folderPaths[0] += path.sep; } const to = qiniuPath.fromLocalPath(folderPaths[0]); - $scope.handlers.downloadFilesHandler([fromInfo], to); + $scope.handlers.downloadFilesHandler([fromInfo], to, angular.copy($scope.currentInfo)); }); } @@ -1484,7 +1484,7 @@ webModule.controller(FILES_CONTROLLER_NAME, [ * @param fromS3Path {array} item={region, bucket, path, name, size } * @param toLocalPath {string} */ - $scope.handlers.downloadFilesHandler(selectedFiles, localPath); + $scope.handlers.downloadFilesHandler(selectedFiles, localPath, angular.copy($scope.currentInfo)); } /** diff --git a/src/renderer/main/files/transfer/downloads.html b/src/renderer/main/files/transfer/downloads.html index 42481bc3..e8b7c1d0 100755 --- a/src/renderer/main/files/transfer/downloads.html +++ b/src/renderer/main/files/transfer/downloads.html @@ -3,18 +3,18 @@
      + class="form-control input-sm" style="width:150px;" ng-model="transSearch.downloadJob">
      + ng-click="transSearch.downloadJob='stopped'"> {{totalStat.downStopped}} + ng-click="transSearch.downloadJob='failed'"> {{totalStat.downFailed}} @@ -68,18 +68,18 @@
      -
        +
        • + ng-repeat="item in lists.downloadJobList track by item.id">
          - {{$index+1}}. {{::item.options.to.name}} + {{$index+1}}. {{::item.to.name}}
          + ng-style="{width: (item.progress.loaded|persent:item.progress.total:item.status) + '%'}">
      @@ -95,28 +95,29 @@
      - + {{item.prog.loaded|sizeFormat}}/{{item.prog.total|sizeFormat}} + ng-if="item.progress.loaded!=item.progress.total">{{item.progress.loaded|sizeFormat}}/{{item.progress.total|sizeFormat}} - , {{item.predictLeftTime|leftTimeFormat}} + , {{item.estimatedTime|leftTimeFormat}}
      - + + tooltip-placement="left" tooltip-append-to-body="true"> + class="btn btn-default btn-xs" uib-tooltip="{{'download.job.retry'|translate}}" + tooltip-placement="left" tooltip-append-to-body="true">
      @@ -126,7 +127,7 @@
    - + {{'loading'|translate}}
    @@ -137,7 +138,7 @@ diff --git a/src/renderer/main/files/transfer/downloads.js b/src/renderer/main/files/transfer/downloads.js index af72fd6b..eb5c8590 100755 --- a/src/renderer/main/files/transfer/downloads.js +++ b/src/renderer/main/files/transfer/downloads.js @@ -1,10 +1,8 @@ import angular from "angular" import webModule from '@/app-module/web' - +import ipcDownloadManager from "@/components/services/ipc-download-manager"; import jobUtil from '@/components/services/job-util' -import DownloadMgr from '@/components/services/download-manager' -import DelayDone from '@/components/services/delay-done' import { TOAST_FACTORY_NAME as Toast } from '@/components/directives/toast-list' import { OVERWRITE_DOWNLOADING } from '@/const/setting-keys' import { DIALOG_FACTORY_NAME as Dialog} from '@/components/services/dialog.s' @@ -16,8 +14,6 @@ webModule.controller(TRANSFER_DOWNLOADS_CONTROLLER_NAME, [ "$timeout", "$translate", jobUtil, - DownloadMgr, - DelayDone, Toast, Dialog, function ( @@ -25,8 +21,6 @@ webModule.controller(TRANSFER_DOWNLOADS_CONTROLLER_NAME, [ $timeout, $translate, jobUtil, - DownloadMgr, - DelayDone, Toast, Dialog ) { @@ -39,59 +33,50 @@ webModule.controller(TRANSFER_DOWNLOADS_CONTROLLER_NAME, [ clearAll: clearAll, stopAll: stopAll, startAll: startAll, + stopItem: stopItem, checkStartJob: checkStartJob, - sch: { - downname: null - }, - schKeyFn: function (item) { - return ( - item.options.to.name + - " " + - item.status + - " " + - jobUtil.getStatusLabel(item.status) - ); - }, - limitToNum: 100, loadMoreDownloadItems: loadMoreItems }); function loadMoreItems() { - var len = $scope.lists.downloadJobList.length; - if ($scope.limitToNum < len) { - $scope.limitToNum += Math.min(100, len - $scope.limitToNum); + const len = $scope.totalStat.down; + if ($scope.lists.downloadJobListLimit < len) { + $scope.lists.downloadJobListLimit += Math.min(100, len - $scope.lists.uploadJobListLimit); } } function triggerOverwriting() { $scope.overwriteDownloading.enabled = !$scope.overwriteDownloading.enabled; localStorage.setItem(OVERWRITE_DOWNLOADING, $scope.overwriteDownloading.enabled); + ipcDownloadManager.updateConfig({ + isOverwrite: $scope.overwriteDownloading.enabled, + }); } - function checkStartJob(item) { - item.wait(); + function stopItem(item) { + ipcDownloadManager.stopJob({ + jobId: item.id, + }); + } - DownloadMgr.trySchedJob(); + function checkStartJob(item) { + ipcDownloadManager.waitJob({ + jobId: item.id, + }); } function showRemoveItem(item) { - if (item.status == "finished") { + if (item.status === "finished") { doRemove(item); } else { - var title = T("remove.from.list.title"); //'从列表中移除' - var message = T("remove.from.list.message"); //'确定移除该下载任务?' + const title = T("remove.from.list.title"); //'从列表中移除' + const message = T("remove.from.list.message"); //'确定移除该下载任务?' Dialog.confirm( title, message, (btn) => { if (btn) { - if (item.status == "running" || - item.status == "waiting" || - item.status == "verifying") { - item.stop(); - } - doRemove(item); } }, @@ -101,125 +86,53 @@ webModule.controller(TRANSFER_DOWNLOADS_CONTROLLER_NAME, [ } function doRemove(item) { - var jobs = $scope.lists.downloadJobList; - for (var i = 0; i < jobs.length; i++) { - if (item === jobs[i]) { - jobs.splice(i, 1); - item.tryCleanupDownloadFile(); - break; - } - } - - $timeout(() => { - DownloadMgr.trySaveProg(); - $scope.calcTotalProg(); + ipcDownloadManager.removeJob({ + jobId: item.id, }); } function clearAllCompleted() { - var jobs = $scope.lists.downloadJobList; - for (var i = 0; i < jobs.length; i++) { - if ("finished" == jobs[i].status) { - jobs.splice(i, 1); - i--; - } - } - - $timeout(() => { - $scope.calcTotalProg(); - }); + ipcDownloadManager.cleanUpJobs(); } function clearAll() { if (!$scope.lists.downloadJobList || - $scope.lists.downloadJobList.length == 0) { + $scope.lists.downloadJobList.length === 0) { return; } - var title = T("clear.all.title"); //清空所有 - var message = T("clear.all.download.message"); //确定清空所有下载任务? + const title = T("clear.all.title"); //清空所有 + const message = T("clear.all.download.message"); //确定清空所有下载任务? Dialog.confirm( title, message, (btn) => { if (btn) { - var jobs = $scope.lists.downloadJobList; - for (var i = 0; i < jobs.length; i++) { - var job = jobs[i]; - if (job.status == "running" || - job.status == "waiting" || - job.status == "verifying") { - job.stop(); - } - - jobs.splice(i, 1); - job.tryCleanupDownloadFile(); - i--; - } - - $timeout(() => { - DownloadMgr.trySaveProg(); - $scope.calcTotalProg(); - }); + ipcDownloadManager.removeAllJobs(); } }, 1 ); } - var stopFlag = false; - function stopAll() { - var jobs = $scope.lists.downloadJobList; - if (jobs && jobs.length > 0) { - stopFlag = true; - - DownloadMgr.stopCreatingJobs(); - - Toast.info(T("pause.on")); //'正在暂停...' - $scope.allActionBtnDisabled = true; - - angular.forEach(jobs, (job) => { - if (job.prog.resumable && ( - job.status == "running" || - job.status == "waiting" || - job.status == "verifying" - )) - n.stop(); - }); - Toast.success(T("pause.success")); //'暂停成功' - - $timeout(() => { - DownloadMgr.trySaveProg(); - $scope.allActionBtnDisabled = false; - }); - } + Toast.info(T("pause.on")); //'正在暂停...' + ipcDownloadManager.stopAllJobs(); + Toast.success(T("pause.success")); //'暂停成功' + + $timeout(() => { + $scope.allActionBtnDisabled = false; + }); } function startAll() { - stopFlag = false; - - //串行 - var jobs = $scope.lists.downloadJobList; - if (jobs && jobs.length > 0) { - $scope.allActionBtnDisabled = true; - DelayDone.seriesRun( - jobs, - (job, fn) => { - if (stopFlag) return; - - if (job && (job.status == "stopped" || job.status == "failed")) { - job.wait(); - } + $scope.allActionBtnDisabled = true; - fn(); - }, - () => { - DownloadMgr.trySchedJob(); - $scope.allActionBtnDisabled = false; - } - ); - } + ipcDownloadManager.startAllJobs(); + + $timeout(() => { + $scope.allActionBtnDisabled = false; + }); } } ]); diff --git a/src/renderer/main/files/transfer/frame.html b/src/renderer/main/files/transfer/frame.html index 39fa2e7e..a1580d44 100755 --- a/src/renderer/main/files/transfer/frame.html +++ b/src/renderer/main/files/transfer/frame.html @@ -5,7 +5,7 @@ - {{totalStat.downDone}}/{{lists.downloadJobList.length}} + {{totalStat.downDone}}/{{totalStat.down}}
@@ -21,7 +21,7 @@
  • {{'download'|translate}} - +
    • diff --git a/src/renderer/main/files/transfer/frame.js b/src/renderer/main/files/transfer/frame.js index d8c83667..fb376e57 100755 --- a/src/renderer/main/files/transfer/frame.js +++ b/src/renderer/main/files/transfer/frame.js @@ -14,7 +14,6 @@ import safeApply from '@/components/services/safe-apply'; import ipcUploadManager from '@/components/services/ipc-upload-manager'; import NgConfig from '@/ng-config' -import DownloadMgr from '@/components/services/download-manager' import { TOAST_FACTORY_NAME as Toast } from '@/components/directives/toast-list' import { EMPTY_FOLDER_UPLOADING, @@ -28,6 +27,8 @@ import './uploads' import './frame.css' import {Status} from "@common/models/job/types"; +import ipcDownloadManager from "@/components/services/ipc-download-manager"; +import {DownloadAction} from "@common/ipc-actions/download"; const TRANSFER_FRAME_CONTROLLER_NAME = 'transferFrameCtrl' @@ -36,30 +37,31 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ "$translate", safeApply, NgConfig, - DownloadMgr, Toast, function ( $scope, $translate, safeApply, ngConfig, - DownloadMgr, Toast, ) { const T = $translate.instant; let uploaderTimer; + let downloaderTimer; angular.extend($scope, { transTab: 1, transSearch: { uploadJob: "", + downloadJob: "", }, lists: { uploadJobList: [], uploadJobListLimit: 100, downloadJobList: [], + downloadJobListLimit: 100, }, emptyFolderUploading: { enabled: localStorage.getItem(EMPTY_FOLDER_UPLOADING) !== null @@ -80,6 +82,10 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ upDone: 0, upStopped: 0, upFailed: 0, + + // download + down: 0, + downRunning: 0, downDone: 0, downStopped: 0, downFailed: 0 @@ -93,10 +99,11 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ $scope.handlers.downloadFilesHandler = downloadFilesHandler; initUploaderIpc(); - DownloadMgr.init($scope); + initDownloaderIpc(); $scope.$on('$destroy', () => { clearInterval(uploaderTimer); + clearInterval(downloaderTimer); }); // init Uploader IPC @@ -160,11 +167,11 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ }); }); ipcUploadManager.updateConfig({ - resumeUpload: Settings.resumeUpload !== 0, + resumable: Settings.resumeUpload !== 0, maxConcurrency: Settings.maxUploadConcurrency, - multipartUploadSize: Settings.multipartUploadSize * ByteSize.MB, - multipartUploadThreshold: Settings.multipartUploadThreshold * ByteSize.MB, - uploadSpeedLimit: Settings.uploadSpeedLimitEnabled === 0 + multipartSize: Settings.multipartUploadSize * ByteSize.MB, + multipartThreshold: Settings.multipartUploadThreshold * ByteSize.MB, + speedLimit: Settings.uploadSpeedLimitEnabled === 0 ? 0 : Settings.uploadSpeedLimitKBperSec * ByteSize.KB, isDebug: Settings.isDebug !== 0, @@ -251,55 +258,174 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ }); } + function initDownloaderIpc() { + ipcRenderer.on("DownloaderManager-reply", (_event, message) => { + safeApply($scope, () => { + switch (message.action) { + case DownloadAction.UpdateUiData: { + $scope.lists.downloadJobList = message.data.list; + $scope.totalStat.down = message.data.total; + $scope.totalStat.downDone = message.data.finished; + $scope.totalStat.downStopped = message.data.stopped; + $scope.totalStat.downFailed = message.data.failed; + break; + } + case DownloadAction.AddedJobs: { + Toast.info(T("download.addtolist.success")); + $scope.transTab = 2; + $scope.toggleTransVisible(true); + + AuditLog.log( + AuditLog.Action.DownloadFilesStart, + { + from: message.data.remoteObjects.map((remoteObject) => { + return { + regionId: remoteObject.region, + bucket: remoteObject.bucketName, + path: remoteObject.key, + }; + }), + to: message.data.destPath, + }, + ); + + break; + } + } + }); + }); + ipcDownloadManager.updateConfig({ + resumable: Settings.resumeDownload !== 0, + maxConcurrency: Settings.maxDownloadConcurrency, + multipartSize: Settings.multipartDownloadSize * ByteSize.MB, + multipartThreshold: Settings.multipartDownloadThreshold * ByteSize.MB, + speedLimit: Settings.downloadSpeedLimitEnabled === 0 + ? 0 + : Settings.downloadSpeedLimitKBperSec * ByteSize.KB, + isDebug: Settings.isDebug !== 0, + isOverwrite: $scope.overwriteDownloading.enabled, + persistPath: getProgFilePath(), + }); + ipcDownloadManager.loadPersistJobs({ + clientOptions: { + accessKey: AuthInfo.get().id, + secretKey: AuthInfo.get().secret, + ucUrl: ngConfig.load().ucUrl || "", + regions: ngConfig.load().regions || [], + }, + downloadOptions: { + userNatureLanguage: localStorage.getItem("lang") || "zh-CN", + }, + }); + downloaderTimer = setInterval(() => { + let query; + if ($scope.transSearch.downloadJob) { + if (Object.values(Status).includes($scope.transSearch.downloadJob.trim())) { + query = { + status: $scope.transSearch.downloadJob.trim(), + }; + } else { + query = { + name: $scope.transSearch.downloadJob.trim(), + }; + } + } + ipcDownloadManager.updateUiData({ + pageNum: 0, + count: $scope.lists.downloadJobListLimit, + query: query, + }) + }, 1000); + + function getProgFilePath() { + const folder = Global.config_path; + if (!fs.existsSync(folder)) { + fs.mkdirSync(folder); + } + + const username = AuthInfo.get().id || "kodo-browser"; + return path.join(folder, "downprog_" + username + ".json"); + } + } + /** * download - * @param fromRemotePath {array} item={region, bucket, path, name, domain, size=0, itemType='file'}, create folder if required - * @param toLocalPath {string} + * @param {object[]} fromRemotePath + * @param {string} fromRemotePath.bucket + * @param {string} fromRemotePath.region + * @param {string} fromRemotePath.name + * @param {number} [fromRemotePath.size] + * @param {Date} [fromRemotePath.lastModified] + * @param {object} fromRemotePath.path + * @param {boolean} fromRemotePath.path.isDir + * @param {string[]} fromRemotePath.path.dirSegments + * @param {string} fromRemotePath.path.sep + * @param {object} fromRemotePath.domain + * @param {string} fromRemotePath.domain.region + * @param {string} fromRemotePath.domain.bucket + * @param {string} fromRemotePath[].qiniuBackendMode + * @param {string} toLocalPath + * @param {object} bucketInfo + * @param {object[]} bucketInfo.availableStorageClasses + * @param {string} bucketInfo.regionId + * @param {string} bucketInfo.qiniuBackendMode + * @param {string} bucketInfo.bucketName + * @param {string} bucketInfo.bucketId */ - function downloadFilesHandler(fromRemotePath, toLocalPath) { + function downloadFilesHandler(fromRemotePath, toLocalPath, bucketInfo) { Toast.info(T("download.addtolist.on")); - DownloadMgr.createDownloadJobs(fromRemotePath, toLocalPath, function (isCancelled) { - Toast.info(T("download.addtolist.success")); - - AuditLog.log( - AuditLog.Action.DownloadFilesStart, - { - from: fromRemotePath.map((entry) => { - return { regionId: entry.region, bucket: entry.bucketName, path: entry.path.toString() }; - }), - to: toLocalPath, - }, - ); - - $scope.transTab = 2; - $scope.toggleTransVisible(true); - }); - } - - function calcTotalProg() { - let c = 0, c2 = 0, cf = 0, cf2 = 0, cs = 0, cs2 = 0; - angular.forEach($scope.lists.downloadJobList, function (n) { - if (n.status === 'running') { - c2++; - } - if (n.status === 'waiting') { - c2++; - } - if (n.status === 'failed') { - cf2++; - } - if (n.status === 'stopped') { - c2++; - cs2++; + const remoteObjects = fromRemotePath.map(item => { + // key + let key = item.path.dirSegments.join(item.path.sep); + if (item.path.isDir) { + key += "/"; + } else if (key) { + key += `/${item.name}`; + } else { + key = item.name; } + + // result + return { + region: item.region, + bucket: item.bucket, + key: key, + name: item.name, + size: item.path.isDir ? 0 : item.size, + mtime: item.path.isDir ? 0 : item.lastModified, + isDirectory: item.path.isDir, + isFile: !item.path.isDir, + }; }); + ipcDownloadManager.addJobs({ + remoteObjects: remoteObjects, + destPath: toLocalPath.toString(), + downloadOptions: { + region: bucketInfo.regionId, + bucket: bucketInfo.bucketName, + domain: $scope.selectedDomain.domain.toQiniuDomain(), + isOverwrite: $scope.overwriteDownloading.enabled, + storageClasses: bucketInfo.availableStorageClasses, + userNatureLanguage: localStorage.getItem('lang') || 'zh-CN', + }, + clientOptions: { + accessKey: AuthInfo.get().id, + secretKey: AuthInfo.get().secret, + ucUrl: ngConfig.load().ucUrl || "", + regions: ngConfig.load().regionId || [], + backendMode: bucketInfo.qiniuBackendMode, + }, + }); + } - $scope.totalStat.running = $scope.totalStat.upRunning + c2; - $scope.totalStat.total = $scope.totalStat.up + $scope.lists.downloadJobList.length; - $scope.totalStat.downDone = $scope.lists.downloadJobList.length - c2; - $scope.totalStat.downStopped = cs2; - $scope.totalStat.downFailed = cf2; + function calcTotalProg() { + $scope.totalStat.running = + $scope.totalStat.upRunning + + $scope.totalStat.downRunning; + $scope.totalStat.total = + $scope.totalStat.up + + $scope.totalStat.down; } } ]); diff --git a/webpack/paths.js b/webpack/paths.js index bd916ded..0e6555a4 100644 --- a/webpack/paths.js +++ b/webpack/paths.js @@ -27,9 +27,9 @@ module.exports = { appWebpackCache: resolveApp('node_modules/.cache'), appMain: resolveApp('src/main'), - appMainIndex: resolveApp('src/main/index.js'), - appMainDownloadWorker: resolveApp('src/main/download-worker.js'), - appMainUploader: resolveApp('src/main/uploader/index.ts'), + appMainIndex: resolveApp('src/main/index.ts'), + appMainDownloader: resolveApp('src/main/download-worker.ts'), + appMainUploader: resolveApp('src/main/upload-worker.ts'), appBuildMain: resolveApp('dist/main'), appRenderer: resolveApp('src/renderer'), diff --git a/webpack/webpack-main.config.js b/webpack/webpack-main.config.js index 62c4c32f..9cac1883 100644 --- a/webpack/webpack-main.config.js +++ b/webpack/webpack-main.config.js @@ -16,8 +16,8 @@ module.exports = function(webpackEnv) { }, entry: { main: paths.appMainIndex, - 'download-worker': paths.appMainDownloadWorker, - 'uploader': paths.appMainUploader, + downloader: paths.appMainDownloader, + uploader: paths.appMainUploader, }, output: { filename: '[name]-bundle.js', From 054f4553fcd411619fa9a994b39d2730a08bca10 Mon Sep 17 00:00:00 2001 From: LiHS Date: Tue, 26 Jul 2022 14:55:19 +0800 Subject: [PATCH 5/8] remove useless code --- src/common/ipc-actions/common.ts | 0 src/common/ipc-actions/download.ts | 7 -- src/common/ipc-actions/upload.ts | 2 +- src/common/models/job/download-job.ts | 7 +- src/common/models/job/index.ts | 2 - src/common/models/job/transfer-job.ts | 7 +- src/common/models/job/types.ts | 18 ++++ src/common/models/job/upload-job.ts | 7 +- src/common/models/job/utils.test.ts | 96 ------------------- src/common/qiniu/etag.test.ts | 45 +++++++++ .../{models/job/utils.ts => qiniu/etag.ts} | 42 -------- src/common/qiniu/index.ts | 1 + src/common/qiniu/types.ts | 2 - src/main/download-worker.ts | 3 +- .../transfer-managers/download-manager.ts | 5 +- src/main/transfer-managers/remote-walker.ts | 19 ---- .../transfer-managers/transfer-manager.ts | 2 +- src/main/upload-worker.ts | 3 +- .../components/services/auto-upgrade.js | 4 +- src/renderer/main/files/transfer/frame.js | 12 --- 20 files changed, 83 insertions(+), 201 deletions(-) delete mode 100644 src/common/ipc-actions/common.ts delete mode 100644 src/common/models/job/index.ts delete mode 100644 src/common/models/job/utils.test.ts create mode 100644 src/common/qiniu/etag.test.ts rename src/common/{models/job/utils.ts => qiniu/etag.ts} (72%) delete mode 100644 src/main/transfer-managers/remote-walker.ts diff --git a/src/common/ipc-actions/common.ts b/src/common/ipc-actions/common.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/src/common/ipc-actions/download.ts b/src/common/ipc-actions/download.ts index cbb898a6..f0f26298 100644 --- a/src/common/ipc-actions/download.ts +++ b/src/common/ipc-actions/download.ts @@ -18,13 +18,6 @@ export interface RemoteObject { isFile: boolean, } -// -// export interface FromInfo { -// regionId: string, -// bucketName: string, -// objectList: RemoteObject[], -// } - export interface DownloadOptions { region: string, bucket: string, diff --git a/src/common/ipc-actions/upload.ts b/src/common/ipc-actions/upload.ts index 65c239b8..8d5b4f83 100644 --- a/src/common/ipc-actions/upload.ts +++ b/src/common/ipc-actions/upload.ts @@ -3,7 +3,7 @@ import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; import {ClientOptions} from "@common/qiniu"; import StorageClass from "@common/models/storage-class"; -import {UploadJob} from "@common/models/job"; +import UploadJob from "@common/models/job/upload-job"; import {Status} from "@common/models/job/types"; // some types maybe should in models diff --git a/src/common/models/job/download-job.ts b/src/common/models/job/download-job.ts index 10df908e..4b94ef5f 100644 --- a/src/common/models/job/download-job.ts +++ b/src/common/models/job/download-job.ts @@ -8,15 +8,14 @@ import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; import {ClientOptions, createQiniuClient} from "@common/qiniu"; -import {Status} from "./types"; -import * as Utils from "./utils"; +import {LocalPath, RemotePath, Status} from "./types"; import TransferJob from "./transfer-job"; interface RequiredOptions { clientOptions: ClientOptions, - from: Required, - to: Utils.LocalPath + from: Required, + to: LocalPath region: string, overwrite: boolean, diff --git a/src/common/models/job/index.ts b/src/common/models/job/index.ts deleted file mode 100644 index d6b91e1f..00000000 --- a/src/common/models/job/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { default as UploadJob, Options as UploadJobOptions } from './upload-job' -export { default as DownloadJob } from './download-job' diff --git a/src/common/models/job/transfer-job.ts b/src/common/models/job/transfer-job.ts index 9e8ec5ce..204e265e 100644 --- a/src/common/models/job/transfer-job.ts +++ b/src/common/models/job/transfer-job.ts @@ -4,14 +4,13 @@ import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; import Duration from "@common/const/duration"; import {ClientOptions} from "@common/qiniu"; -import {Status} from "./types"; -import * as Utils from "./utils"; +import {LocalPath, RemotePath, Status} from "./types"; interface RequiredOptions { clientOptions: ClientOptions, - from: Utils.LocalPath | Utils.RemotePath - to: Utils.LocalPath | Utils.RemotePath + from: LocalPath | RemotePath + to: LocalPath | RemotePath region: string, } diff --git a/src/common/models/job/types.ts b/src/common/models/job/types.ts index 496d803c..ee912727 100644 --- a/src/common/models/job/types.ts +++ b/src/common/models/job/types.ts @@ -13,3 +13,21 @@ export enum Status { Duplicated = "duplicated", Verifying = "verifying" } + +export interface LocalPath { + name: string, + path: string, + size?: number, // bytes + mtime?: number, // ms timestamp +} + +export interface RemotePath { + bucket: string, + key: string, + size?: number, // bytes + mtime?: number, // ms timestamp +} + +export function isLocalPath(p: LocalPath | RemotePath): p is LocalPath { + return p.hasOwnProperty("name"); +} diff --git a/src/common/models/job/upload-job.ts b/src/common/models/job/upload-job.ts index cf638f71..848c49c9 100644 --- a/src/common/models/job/upload-job.ts +++ b/src/common/models/job/upload-job.ts @@ -11,16 +11,15 @@ import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; import {ClientOptions, createQiniuClient} from "@common/qiniu"; import ByteSize from "@common/const/byte-size"; -import {Status, UploadedPart} from "./types"; -import * as Utils from "./utils"; +import {LocalPath, RemotePath, Status, UploadedPart} from "./types"; import TransferJob from "./transfer-job"; // if change options, remember to check `get persistInfo()` interface RequiredOptions { clientOptions: ClientOptions, - from: Required, - to: Utils.RemotePath, + from: Required, + to: RemotePath, region: string, overwrite: boolean, diff --git a/src/common/models/job/utils.test.ts b/src/common/models/job/utils.test.ts deleted file mode 100644 index ec66cc39..00000000 --- a/src/common/models/job/utils.test.ts +++ /dev/null @@ -1,96 +0,0 @@ -import mockFs from "mock-fs"; - -import { Readable as ReadableStream } from "stream"; - -import * as JobUtils from "./utils"; - -describe("test models/job/utils.ts", () => { - describe("test parseLocalPath", () => { - if (process.platform === "win32") { - it("win path", () => { - expect(JobUtils.parseLocalPath("D:\\path\\to\\Some\\file.txt")) - .toEqual({ - name: "file.txt", - path: "D:\\path\\to\\Some\\file.txt", - }); - }); - } else { - it("unix-like path", () => { - expect(JobUtils.parseLocalPath("/path/to/Some/file.txt")) - .toEqual({ - name: "file.txt", - path: "/path/to/Some/file.txt", - }); - }); - } - }); - - describe("test parseKodoPath", () => { - it("normal", () => { - expect( - JobUtils.parseKodoPath("kodo://kodo-browser/path/to/some/file.txt") - ) - .toEqual({ - bucket: "kodo-browser", - key: "path", - // first look(by the name) it should be - // key: "path/to/some/file.txt", - // but old function return as such - // TODO: not sure is it a bug, or intentionally - }); - }); - - it("check protocol failed", () => { - expect(() => { - JobUtils.parseKodoPath("kodo-browser/path/to/some/file.txt") - }) - .toThrow("Invalid kodo path"); - }); - }); - - describe("test getEtag", () => { - it("by Buffer", async () => { - const etag = await new Promise(resolve => { - JobUtils.getEtag(Buffer.from("Hello kodo browser!"), resolve); - }); - const oldEtag = await new Promise(resolve => { - JobUtils.getEtag(Buffer.from("Hello kodo browser!"), resolve); - }); - // console.log(etag); - // expect(etag).toBe("FvV1cV1RzovZlzHlGApMIXL2RRHW"); - expect(etag).toBe(oldEtag); - }); - it("by path(string)", async () => { - mockFs({ - "/path/to/get/etag": "Hello kodo browser!", - }); - const etag = await new Promise(resolve => { - JobUtils.getEtag("/path/to/get/etag", resolve); - }); - expect(etag).toBe("FvV1cV1RzovZlzHlGApMIXL2RRHW"); - mockFs.restore(); - }); - it("by ReadableStream", async () => { - class MyStream extends ReadableStream { - data = "Hello kodo browser!" - - _read(size: number) { - if (this.data.length) { - let chunk: string; - [chunk, this.data] = [this.data.slice(0, size), this.data.slice(size)]; - this.push(chunk); - } else { - this.push(null); - } - } - } - - const readableStream = new MyStream(); - - const etag = await new Promise(resolve => { - JobUtils.getEtag(readableStream, resolve); - }); - expect(etag).toBe("FvV1cV1RzovZlzHlGApMIXL2RRHW"); - }); - }); -}); diff --git a/src/common/qiniu/etag.test.ts b/src/common/qiniu/etag.test.ts new file mode 100644 index 00000000..3b11c508 --- /dev/null +++ b/src/common/qiniu/etag.test.ts @@ -0,0 +1,45 @@ +import mockFs from "mock-fs"; +import {Readable as ReadableStream} from "stream"; + +import {getEtag} from "./etag"; + +describe("test getEtag", () => { + it("by Buffer", async () => { + const etag = await new Promise(resolve => { + getEtag(Buffer.from("Hello kodo browser!"), resolve); + }); + expect(etag).toBe("FvV1cV1RzovZlzHlGApMIXL2RRHW"); + }); + it("by path(string)", async () => { + mockFs({ + "/path/to/get/etag": "Hello kodo browser!", + }); + const etag = await new Promise(resolve => { + getEtag("/path/to/get/etag", resolve); + }); + expect(etag).toBe("FvV1cV1RzovZlzHlGApMIXL2RRHW"); + mockFs.restore(); + }); + it("by ReadableStream", async () => { + class MyStream extends ReadableStream { + data = "Hello kodo browser!" + + _read(size: number) { + if (this.data.length) { + let chunk: string; + [chunk, this.data] = [this.data.slice(0, size), this.data.slice(size)]; + this.push(chunk); + } else { + this.push(null); + } + } + } + + const readableStream = new MyStream(); + + const etag = await new Promise(resolve => { + getEtag(readableStream, resolve); + }); + expect(etag).toBe("FvV1cV1RzovZlzHlGApMIXL2RRHW"); + }); +}); diff --git a/src/common/models/job/utils.ts b/src/common/qiniu/etag.ts similarity index 72% rename from src/common/models/job/utils.ts rename to src/common/qiniu/etag.ts index b4c6b7b4..8c17c0fc 100644 --- a/src/common/models/job/utils.ts +++ b/src/common/qiniu/etag.ts @@ -1,49 +1,7 @@ -import path from "path"; import crypto from "crypto"; import { Readable as ReadableStream } from "stream"; import fs from "fs"; -import * as KodoNav from "@/const/kodo-nav" - -export interface LocalPath { - name: string, - path: string, - size?: number, // bytes - mtime?: number, // ms timestamp -} -// parseLocalPath: get name and path from local path -export function parseLocalPath(p: string): LocalPath { - return { - name: path.basename(p), - path: p, - }; -} - -export interface RemotePath { - bucket: string, - key: string, - size?: number, // bytes - mtime?: number, // ms timestamp -} -// parseKodoPath: get bucket and key from KodoPath -export function parseKodoPath(kodoPath: string): RemotePath { - if (!kodoPath.startsWith(KodoNav.ADDR_KODO_PROTOCOL)) { - throw Error("Invalid kodo path"); - } - - const [bucket, key] = kodoPath - .substring(KodoNav.ADDR_KODO_PROTOCOL.length) - .split('/', 2); - return { - bucket, - key: key.replace(/\\/g, "/"), - }; -} - -export function isLocalPath(p: LocalPath | RemotePath): p is LocalPath { - return p.hasOwnProperty("name"); -} - // get etag function sha1(content: Buffer): Buffer { return crypto.createHash("sha1") diff --git a/src/common/qiniu/index.ts b/src/common/qiniu/index.ts index 3f7f8afc..5163e954 100644 --- a/src/common/qiniu/index.ts +++ b/src/common/qiniu/index.ts @@ -1,2 +1,3 @@ export * from "./types"; export {default as createQiniuClient} from "./create-client"; +export * from "./etag"; diff --git a/src/common/qiniu/types.ts b/src/common/qiniu/types.ts index 4b06dece..e59a6e5c 100644 --- a/src/common/qiniu/types.ts +++ b/src/common/qiniu/types.ts @@ -11,6 +11,4 @@ export interface ClientOptions { ucUrl: string, regions: Region[], backendMode: BackendMode, - - // storageClasses: StorageClass[], // TODO } diff --git a/src/main/download-worker.ts b/src/main/download-worker.ts index 1ca869dc..824307bd 100644 --- a/src/main/download-worker.ts +++ b/src/main/download-worker.ts @@ -9,7 +9,6 @@ import DownloadManager from "./transfer-managers/download-manager"; import DownloadJob from "@common/models/job/download-job"; import {Status} from "@common/models/job/types"; - // initial DownloadManager Config from argv after `--config-json` const configStr = process.argv.find((_arg, i, arr) => arr[i - 1] === "--config-json"); const downloadManagerConfig = configStr ? JSON.parse(configStr) : {}; @@ -84,6 +83,7 @@ process.on("message", (message: DownloadMessage) => { } case DownloadAction.RemoveJob: { downloadManager.removeJob(message.data.jobId); + downloadManager.persistJobs(); break; } case DownloadAction.CleanupJobs: { @@ -160,5 +160,6 @@ function handleJobDone(jobId: string, job?: DownloadJob) { } }; process.send?.(jobCompletedReplyMessage); + downloadManager.persistJobs(); } } diff --git a/src/main/transfer-managers/download-manager.ts b/src/main/transfer-managers/download-manager.ts index c4550230..58f59db3 100644 --- a/src/main/transfer-managers/download-manager.ts +++ b/src/main/transfer-managers/download-manager.ts @@ -8,7 +8,6 @@ import {Adapter, ListedObjects} from "kodo-s3-adapter-sdk/dist/adapter"; import {ClientOptions, createQiniuClient} from "@common/qiniu"; import DownloadJob from "@common/models/job/download-job"; import {Status} from "@common/models/job/types"; -import {LocalPath, RemotePath} from "@common/models/job/utils"; import {DownloadOptions, RemoteObject} from "@common/ipc-actions/download"; import TransferManager, {TransferManagerConfig} from "./transfer-manager"; @@ -255,8 +254,8 @@ export default class DownloadManager extends TransferManager, - to: LocalPath, + from: Required, + to: DownloadJob["options"]["to"], clientOptions: ClientOptions, downloadOptions: DownloadOptions, ): void { diff --git a/src/main/transfer-managers/remote-walker.ts b/src/main/transfer-managers/remote-walker.ts deleted file mode 100644 index 3df8d887..00000000 --- a/src/main/transfer-managers/remote-walker.ts +++ /dev/null @@ -1,19 +0,0 @@ -// import {Adapter} from "kodo-s3-adapter-sdk/dist/adapter"; -// -// function CreateRemoteWalker(client: Adapter) { -// async function _walkPage() { -// -// } -// -// async function _walk() { -// -// } -// } -// -// export default class RemoteWalker { -// constructor( -// private readonly client: Adapter, -// -// ) { -// } -// } diff --git a/src/main/transfer-managers/transfer-manager.ts b/src/main/transfer-managers/transfer-manager.ts index d0ae00d4..310ac337 100644 --- a/src/main/transfer-managers/transfer-manager.ts +++ b/src/main/transfer-managers/transfer-manager.ts @@ -1,7 +1,7 @@ import fs from "fs"; import TransferJob from "@common/models/job/transfer-job"; -import {isLocalPath} from "@common/models/job/utils"; +import {isLocalPath} from "@common/models/job/types"; import ByteSize from "@common/const/byte-size"; import {Status} from "@common/models/job/types"; diff --git a/src/main/upload-worker.ts b/src/main/upload-worker.ts index 6e8d22db..eb742cfd 100644 --- a/src/main/upload-worker.ts +++ b/src/main/upload-worker.ts @@ -19,7 +19,7 @@ const uploadManager = new UploadManager(uploadManagerConfig); process.on("uncaughtException", (err) => { uploadManager.persistJobs(true); - console.error(err); + console.error("upload worker: uncaughtException", err); }); process.on("message", (message: UploadMessage) => { @@ -173,4 +173,5 @@ function handleCreatedDirectory(bucket: string, directoryKey: string) { }, }; process.send?.(createdDirectoryReplyMessage); + uploadManager.persistJobs(); } diff --git a/src/renderer/components/services/auto-upgrade.js b/src/renderer/components/services/auto-upgrade.js index f0efbacf..80c021ae 100755 --- a/src/renderer/components/services/auto-upgrade.js +++ b/src/renderer/components/services/auto-upgrade.js @@ -4,7 +4,7 @@ import path from 'path' import request from 'request' import downloadsFolder from 'downloads-folder' -import * as util from '@common/models/job/utils' +import {getEtag} from '@common/qiniu' import webModule from '@/app-module/web' import { upgrade } from '@/customize' @@ -82,7 +82,7 @@ webModule.factory(AUTO_UPGRADE_SVS_FACTORY_NAME, [ this.check = function (expected, callback) { //crc console.log("etag check"); - return util.getEtag(to + ".download", function(actual) { + return getEtag(to + ".download", function(actual) { if (expected !== `"${actual}"`) { callback(new Error(`Etag check failed, expected: ${expected}, actual: "${actual}"`)); } else { diff --git a/src/renderer/main/files/transfer/frame.js b/src/renderer/main/files/transfer/frame.js index fb376e57..8f8c6498 100755 --- a/src/renderer/main/files/transfer/frame.js +++ b/src/renderer/main/files/transfer/frame.js @@ -90,8 +90,6 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ downStopped: 0, downFailed: 0 }, - - calcTotalProg: calcTotalProg }); // functions in parent scope @@ -288,7 +286,6 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ to: message.data.destPath, }, ); - break; } } @@ -418,15 +415,6 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ }, }); } - - function calcTotalProg() { - $scope.totalStat.running = - $scope.totalStat.upRunning + - $scope.totalStat.downRunning; - $scope.totalStat.total = - $scope.totalStat.up + - $scope.totalStat.down; - } } ]); From ff6f742b369767955188259f94c0f7447e7d9767 Mon Sep 17 00:00:00 2001 From: LiHS Date: Wed, 27 Jul 2022 18:41:07 +0800 Subject: [PATCH 6/8] fix region in download/upload workers --- src/main/download-worker.ts | 39 +++++++++++++++++++++++++++++++++---- src/main/upload-worker.ts | 39 +++++++++++++++++++++++++++++++++---- src/renderer/config.ts | 3 +++ 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/src/main/download-worker.ts b/src/main/download-worker.ts index 824307bd..f5ff429e 100644 --- a/src/main/download-worker.ts +++ b/src/main/download-worker.ts @@ -1,3 +1,5 @@ +import {Region} from "kodo-s3-adapter-sdk"; + import { AddedJobsReplyMessage, DownloadAction, @@ -5,9 +7,10 @@ import { JobCompletedReplyMessage, UpdateUiDataReplyMessage } from "@common/ipc-actions/download"; -import DownloadManager from "./transfer-managers/download-manager"; -import DownloadJob from "@common/models/job/download-job"; import {Status} from "@common/models/job/types"; +import DownloadJob from "@common/models/job/download-job"; + +import DownloadManager from "./transfer-managers/download-manager"; // initial DownloadManager Config from argv after `--config-json` const configStr = process.argv.find((_arg, i, arr) => arr[i - 1] === "--config-json"); @@ -28,7 +31,21 @@ process.on("message", (message: DownloadMessage) => { } case DownloadAction.LoadPersistJobs: { downloadManager.loadJobsFromStorage( - message.data.clientOptions, + { + ...message.data.clientOptions, + // regions ars serialized, so need new it. + // reference src/renderer/config.ts load(): result + regions: message.data.clientOptions.regions.map(serializedRegion => { + const r = new Region( + serializedRegion.id, + serializedRegion.s3Id, + serializedRegion.label, + ); + r.ucUrls = serializedRegion.ucUrls; + r.s3Urls = serializedRegion.s3Urls; + return r; + }), + }, message.data.downloadOptions, ); break; @@ -37,7 +54,21 @@ process.on("message", (message: DownloadMessage) => { downloadManager.createDownloadJobs( message.data.remoteObjects, message.data.destPath, - message.data.clientOptions, + { + ...message.data.clientOptions, + // regions ars serialized, so need new it. + // reference src/renderer/config.ts load(): result + regions: message.data.clientOptions.regions.map(serializedRegion => { + const r = new Region( + serializedRegion.id, + serializedRegion.s3Id, + serializedRegion.label, + ); + r.ucUrls = serializedRegion.ucUrls; + r.s3Urls = serializedRegion.s3Urls; + return r; + }), + }, message.data.downloadOptions, { jobsAdding: () => { diff --git a/src/main/upload-worker.ts b/src/main/upload-worker.ts index eb742cfd..298b5d85 100644 --- a/src/main/upload-worker.ts +++ b/src/main/upload-worker.ts @@ -1,3 +1,5 @@ +import {Region} from "kodo-s3-adapter-sdk"; + import { AddedJobsReplyMessage, CreatedDirectoryReplyMessage, @@ -6,9 +8,10 @@ import { UploadAction, UploadMessage } from "@common/ipc-actions/upload"; -import UploadManager from "./transfer-managers/upload-manager"; -import UploadJob from "@common/models/job/upload-job"; import {Status} from "@common/models/job/types"; +import UploadJob from "@common/models/job/upload-job"; + +import UploadManager from "./transfer-managers/upload-manager"; // initial UploadManager Config from argv after `--config-json` const configStr = process.argv.find((_arg, i, arr) => arr[i - 1] === "--config-json"); @@ -30,7 +33,21 @@ process.on("message", (message: UploadMessage) => { } case UploadAction.LoadPersistJobs: { uploadManager.loadJobsFromStorage( - message.data.clientOptions, + { + ...message.data.clientOptions, + // regions ars serialized, so need new it. + // reference src/renderer/config.ts load(): result + regions: message.data.clientOptions.regions.map(serializedRegion => { + const r = new Region( + serializedRegion.id, + serializedRegion.s3Id, + serializedRegion.label, + ); + r.ucUrls = serializedRegion.ucUrls; + r.s3Urls = serializedRegion.s3Urls; + return r; + }), + }, message.data.uploadOptions, ); break; @@ -39,7 +56,21 @@ process.on("message", (message: UploadMessage) => { uploadManager.createUploadJobs( message.data.filePathnameList, message.data.destInfo, - message.data.clientOptions, + { + ...message.data.clientOptions, + // regions ars serialized, so need new it. + // reference src/renderer/config.ts load(): result + regions: message.data.clientOptions.regions.map(serializedRegion => { + const r = new Region( + serializedRegion.id, + serializedRegion.s3Id, + serializedRegion.label, + ); + r.ucUrls = serializedRegion.ucUrls; + r.s3Urls = serializedRegion.s3Urls; + return r; + }), + }, message.data.uploadOptions, { jobsAdding: () => { diff --git a/src/renderer/config.ts b/src/renderer/config.ts index aecb8abf..334d41e9 100644 --- a/src/renderer/config.ts +++ b/src/renderer/config.ts @@ -95,6 +95,9 @@ export function load(isUsingPublic?: boolean): Config { ucUrl, regions: (cachedConfig.regions ?? []).map(r => { const region = new Region('', r.id, r.label); + // if change region's properties, need change this too: + // src/main/upload-worker.ts LoadPersistJobs, createUploadJobs + // src/main/download-worker.ts LoadPersistJobs, createUploadJobs region.ucUrls = [ucUrl]; region.s3Urls = [r.endpoint]; return region; From 379e837d077095faf0c5f33c29ca3dd9e47a614c Mon Sep 17 00:00:00 2001 From: LiHS Date: Fri, 29 Jul 2022 11:39:40 +0800 Subject: [PATCH 7/8] bump kodo-s3-adapter-sdk to v0.2.31 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 84b1f3ba..1eef1eb3 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ "jquery.qrcode": "^1.0.3", "js-base64": "^3.4.5", "js-md5": "^0.7.3", - "kodo-s3-adapter-sdk": "0.2.30", + "kodo-s3-adapter-sdk": "0.2.31", "lodash": "^4.17.21", "mime": "^2.3.1", "moment": "^2.22.2", diff --git a/yarn.lock b/yarn.lock index c19ed0c6..0739c3a3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6906,10 +6906,10 @@ kleur@^3.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== -kodo-s3-adapter-sdk@0.2.30: - version "0.2.30" - resolved "https://registry.yarnpkg.com/kodo-s3-adapter-sdk/-/kodo-s3-adapter-sdk-0.2.30.tgz#6f9ff034eb75e33979aa638fbd2c921df2a903b4" - integrity sha512-E1BpNCSSwSG40wJAPT3yAz04jLB4g/FG6qB6uYQNNNXtsZ6JNjlZui1KJq8vuf9b7I+i6zYkpQW860V8CEQySg== +kodo-s3-adapter-sdk@0.2.31: + version "0.2.31" + resolved "https://registry.yarnpkg.com/kodo-s3-adapter-sdk/-/kodo-s3-adapter-sdk-0.2.31.tgz#bafdcc178e9e7984c53a44bd3d2e1f5957fc84fb" + integrity sha512-aHvFFoFJ4memT2MuktlkiE7fHm8UQXfcwSWTf0sUTqFtFzCUJyi8FIxH20fTjSSjEFirydOrQxcZZMS0qjtf6g== dependencies: async-lock "^1.2.4" aws-sdk "^2.800.0" From d86e2ad2874bc0a977c83e0843fff28245b15054 Mon Sep 17 00:00:00 2001 From: LiHS Date: Fri, 29 Jul 2022 12:40:46 +0800 Subject: [PATCH 8/8] fix tests for settings(015a4a0) --- .../components/services/settings.test.ts | 156 ++++++++++++++++-- 1 file changed, 142 insertions(+), 14 deletions(-) diff --git a/src/renderer/components/services/settings.test.ts b/src/renderer/components/services/settings.test.ts index 1a49d97e..09ac536a 100644 --- a/src/renderer/components/services/settings.test.ts +++ b/src/renderer/components/services/settings.test.ts @@ -7,10 +7,11 @@ jest.mock("electron", () => ({ } })); -import { ipcRenderer } from "electron"; -import Settings, { SettingKey } from "./settings"; +import {ipcRenderer} from "electron"; import {UploadAction} from "@common/ipc-actions/upload"; +import {DownloadAction} from "@common/ipc-actions/download"; import ByteSize from "@common/const/byte-size"; +import Settings, {SettingKey} from "./settings"; // WARNING: The getter tests in "no data in storage" section is // for testing default value. @@ -105,7 +106,7 @@ describe("test settings.ts", () => { it("isDebug setter", () => { Settings.isDebug = 1; expect(Settings.isDebug).toBe(1); - expect(ipcRenderer.send).toHaveBeenLastCalledWith( + expect(ipcRenderer.send).toHaveBeenCalledWith( "UploaderManager", { action: UploadAction.UpdateConfig, @@ -114,9 +115,18 @@ describe("test settings.ts", () => { }, }, ); + expect(ipcRenderer.send).toHaveBeenCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + isDebug: true, + }, + }, + ); Settings.isDebug = 0; expect(Settings.isDebug).toBe(0); - expect(ipcRenderer.send).toHaveBeenLastCalledWith( + expect(ipcRenderer.send).toHaveBeenCalledWith( "UploaderManager", { action: UploadAction.UpdateConfig, @@ -125,6 +135,15 @@ describe("test settings.ts", () => { }, }, ); + expect(ipcRenderer.send).toHaveBeenCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + isDebug: false, + }, + }, + ); }); // autoUpgrade @@ -150,7 +169,7 @@ describe("test settings.ts", () => { { action: UploadAction.UpdateConfig, data: { - resumeUpload: true, + resumable: true, }, }, ); @@ -161,7 +180,7 @@ describe("test settings.ts", () => { { action: UploadAction.UpdateConfig, data: { - resumeUpload: false, + resumable: false, }, }, ); @@ -208,7 +227,7 @@ describe("test settings.ts", () => { { action: UploadAction.UpdateConfig, data: { - multipartUploadSize: 10 * ByteSize.MB, + multipartSize: 10 * ByteSize.MB, }, }, ); @@ -219,7 +238,7 @@ describe("test settings.ts", () => { { action: UploadAction.UpdateConfig, data: { - multipartUploadSize: 8 * ByteSize.MB, + multipartSize: 8 * ByteSize.MB, }, }, ); @@ -243,7 +262,7 @@ describe("test settings.ts", () => { { action: UploadAction.UpdateConfig, data: { - multipartUploadThreshold: 110 * ByteSize.MB, + multipartThreshold: 110 * ByteSize.MB, }, }, ); @@ -254,7 +273,7 @@ describe("test settings.ts", () => { { action: UploadAction.UpdateConfig, data: { - multipartUploadThreshold: 100 * ByteSize.MB, + multipartThreshold: 100 * ByteSize.MB, }, }, ); @@ -272,7 +291,7 @@ describe("test settings.ts", () => { { action: UploadAction.UpdateConfig, data: { - uploadSpeedLimit: Settings.uploadSpeedLimitKBperSec * ByteSize.KB, + speedLimit: Settings.uploadSpeedLimitKBperSec * ByteSize.KB, }, }, ); @@ -283,7 +302,7 @@ describe("test settings.ts", () => { { action: UploadAction.UpdateConfig, data: { - uploadSpeedLimit: 0, + speedLimit: 0, }, }, ); @@ -302,7 +321,7 @@ describe("test settings.ts", () => { { action: UploadAction.UpdateConfig, data: { - uploadSpeedLimit: 2048 * ByteSize.KB, + speedLimit: 2048 * ByteSize.KB, }, }, ); @@ -313,7 +332,7 @@ describe("test settings.ts", () => { { action: UploadAction.UpdateConfig, data: { - uploadSpeedLimit: 1024 * ByteSize.KB, + speedLimit: 1024 * ByteSize.KB, }, }, ); @@ -326,8 +345,26 @@ describe("test settings.ts", () => { it("resumeDownload setter", () => { Settings.resumeDownload = 1; expect(Settings.resumeDownload).toBe(1); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + resumable: true, + }, + }, + ); Settings.resumeDownload = 0; expect(Settings.resumeDownload).toBe(0); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + resumable: false, + }, + }, + ); }); // maxDownloadConcurrency @@ -337,8 +374,26 @@ describe("test settings.ts", () => { it("maxDownloadConcurrency setter", () => { Settings.maxDownloadConcurrency = 2; expect(Settings.maxDownloadConcurrency).toBe(2); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + maxConcurrency: 2, + }, + }, + ); Settings.maxDownloadConcurrency = 1; expect(Settings.maxDownloadConcurrency).toBe(1); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + maxConcurrency: 1, + }, + }, + ); }); // multipartDownloadSize @@ -348,8 +403,26 @@ describe("test settings.ts", () => { it("multipartDownloadSize setter", () => { Settings.multipartDownloadSize = 4; expect(Settings.multipartDownloadSize).toBe(4); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + multipartSize: 4 * ByteSize.MB, + }, + }, + ); Settings.multipartDownloadSize = 8; expect(Settings.multipartDownloadSize).toBe(8); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + multipartSize: 8 * ByteSize.MB, + }, + }, + ); }); // multipartDownloadThreshold @@ -359,8 +432,26 @@ describe("test settings.ts", () => { it("multipartDownloadThreshold setter", () => { Settings.multipartDownloadThreshold = 110; expect(Settings.multipartDownloadThreshold).toBe(110); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + multipartThreshold: 110 * ByteSize.MB, + }, + }, + ); Settings.multipartDownloadThreshold = 100; expect(Settings.multipartDownloadThreshold).toBe(100); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + multipartThreshold: 100 * ByteSize.MB, + }, + }, + ); }); // downloadSpeedLimitEnabled @@ -370,8 +461,26 @@ describe("test settings.ts", () => { it("downloadSpeedLimitEnabled setter", () => { Settings.downloadSpeedLimitEnabled = 1; expect(Settings.downloadSpeedLimitEnabled).toBe(1); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + speedLimit: Settings.uploadSpeedLimitKBperSec * ByteSize.KB, + }, + }, + ); Settings.downloadSpeedLimitEnabled = 0; expect(Settings.downloadSpeedLimitEnabled).toBe(0); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + speedLimit: 0, + }, + }, + ); }); // downloadSpeedLimitKBperSec @@ -379,10 +488,29 @@ describe("test settings.ts", () => { expect(Settings.downloadSpeedLimitKBperSec).toBe(1024); }); it("downloadSpeedLimitKBperSec setter", () => { + Settings.downloadSpeedLimitEnabled = 1; Settings.downloadSpeedLimitKBperSec = 2048; expect(Settings.downloadSpeedLimitKBperSec).toBe(2048); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + speedLimit: 2048 * ByteSize.KB, + }, + }, + ); Settings.downloadSpeedLimitKBperSec = 1024; expect(Settings.downloadSpeedLimitKBperSec).toBe(1024); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "DownloaderManager", + { + action: DownloadAction.UpdateConfig, + data: { + speedLimit: 1024 * ByteSize.KB, + }, + }, + ); }); // externalPathEnabled