diff --git a/package.json b/package.json index 7b980cbd..5cb662a9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kodo-browser", - "version": "1.0.17", + "version": "1.0.18", "license": "Apache-2.0", "author": { "name": "Rong Zhou", @@ -17,7 +17,7 @@ "dev": "./node_modules/.bin/webpack --env development -c ./webpack/webpack.config.js && ./node_modules/.bin/npm-run-all --parallel watch dev:run", "dev:run": "./node_modules/.bin/cross-env NODE_ENV=development electron ./dist/main/main-bundle.js", "prod": "./node_modules/.bin/cross-env NODE_ENV=production electron ./dist", - "watch": "./node_modules/.bin/webpack --env development -c ./webpack/webpack-renderer.config.js -w", + "watch": "./node_modules/.bin/webpack --env development -c ./webpack/webpack.config.js -w", "build": "./node_modules/.bin/webpack --env production --env sourcemap -c ./webpack/webpack.config.js", "build:mac": "./node_modules/.bin/gulp mac", "pkg:mac": "./node_modules/.bin/gulp maczip", @@ -55,6 +55,7 @@ "@babel/preset-env": "^7.1.6", "@types/codemirror": "^5.60.5", "@types/jest": "^27.0.2", + "@types/lodash": "^4.14.182", "@types/mock-fs": "^4.13.1", "archiver": "^3.0.3", "copy-webpack-plugin": "^9.0.1", @@ -84,6 +85,7 @@ "webpack-cli": "^4.9.0" }, "dependencies": { + "@root/walk": "^1.1.0", "@uirouter/angularjs": "^1.0.20", "angular": "1.7.9", "angular-sanitize": "1.7.5", @@ -102,7 +104,8 @@ "jquery.qrcode": "^1.0.3", "js-base64": "^3.4.5", "js-md5": "^0.7.3", - "kodo-s3-adapter-sdk": "0.2.29", + "kodo-s3-adapter-sdk": "0.2.30", + "lodash": "^4.17.21", "mime": "^2.3.1", "moment": "^2.22.2", "qiniu-path": "^0.0.3", diff --git a/src/renderer/const/app-config.ts b/src/common/const/app-config.ts similarity index 100% rename from src/renderer/const/app-config.ts rename to src/common/const/app-config.ts diff --git a/src/common/const/byte-size.ts b/src/common/const/byte-size.ts new file mode 100644 index 00000000..fb8939be --- /dev/null +++ b/src/common/const/byte-size.ts @@ -0,0 +1,8 @@ +enum ByteSize { + KB = 1024, + MB = 1024 * KB, + GB = 1024 * MB, + TB = 1024 * GB, +} + +export default ByteSize; diff --git a/src/renderer/const/duration.ts b/src/common/const/duration.ts similarity index 100% rename from src/renderer/const/duration.ts rename to src/common/const/duration.ts diff --git a/src/common/const/qiniu.ts b/src/common/const/qiniu.ts new file mode 100644 index 00000000..86d713a1 --- /dev/null +++ b/src/common/const/qiniu.ts @@ -0,0 +1,4 @@ +export enum BackendMode { + Kodo = "kodo", + S3 = "s3", +} diff --git a/src/common/ipc-actions/upload.ts b/src/common/ipc-actions/upload.ts new file mode 100644 index 00000000..e2292dac --- /dev/null +++ b/src/common/ipc-actions/upload.ts @@ -0,0 +1,286 @@ +import {IpcRenderer} from "electron"; +import {Region} from "kodo-s3-adapter-sdk"; +import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; +import {BackendMode} from "@common/const/qiniu"; +import {Status} from "@common/models/job/types"; +import StorageClass from "@common/models/storage-class"; +import {UploadJob} from "@common/models/job"; + +// some types maybe should in models +export interface DestInfo { + regionId: string, + bucketName: string, + key: string, +} + +export interface UploadOptions { + isOverwrite: boolean, + storageClassName: StorageClass["kodoName"], + storageClasses: StorageClass[], + userNatureLanguage: NatureLanguage, +} + +// TODO: merge with `RequiredOptions['clientOptions']` in upload-job.ts +export interface ClientOptions { + accessKey: string, + secretKey: string, + ucUrl: string, + regions: Region[], + backendMode: BackendMode, +} + +// action names +export enum UploadAction { + UpdateConfig = "UpdateConfig", + LoadPersistJobs = "LoadPersistJobs", + AddJobs = "AddJobs", + StopJob = "StopJob", + WaitJob = "WaitJob", + StartJob = "StartJob", + RemoveJob = "RemoveJob", + CleanupJobs = "CleanupJobs", + StartAllJobs = "StartAllJobs", + StopAllJobs = "StopAllJobs", + RemoveAllJobs = "RemoveAllJobs", + + // common + UpdateUiData = "UpdateUiData", + + // reply only + AddedJobs = "AddedJobs", + JobCompleted = "JobCompleted", + CreatedDirectory = "CreatedDirectory", +} + +// actions with payload data +export interface UpdateConfigMessage { + action: UploadAction.UpdateConfig, + data: Partial<{ + resumeUpload: boolean, + maxConcurrency: number, + multipartUploadSize: number, // Bytes + multipartUploadThreshold: number, // Bytes + uploadSpeedLimit: number, // Bytes/s + isDebug: boolean, + isSkipEmptyDirectory: boolean, + persistPath: string, + }>, +} + +export interface LoadPersistJobsMessage { + action: UploadAction.LoadPersistJobs, + data: { + clientOptions: Pick, + uploadOptions: Pick, + }, +} + +export interface AddJobsMessage { + action: UploadAction.AddJobs, + data: { + filePathnameList: string[], + destInfo: DestInfo, + uploadOptions: UploadOptions, + clientOptions: ClientOptions, + }, +} + +export interface UpdateUiDataMessage { + action: UploadAction.UpdateUiData, + data: { + pageNum: number, + count: number, + query?: { status?: Status, name?: string }, + }, +} + +export interface UpdateUiDataReplyMessage { + action: UploadAction.UpdateUiData, + data: { + list: (UploadJob["uiData"] | undefined)[], + total: number, + finished: number, + running: number, + failed: number, + stopped: number, + }, +} + +export interface StopJobMessage { + action: UploadAction.StopJob, + data: { + jobId: string, + }, +} + +export interface WaitJobMessage { + action: UploadAction.WaitJob, + data: { + jobId: string, + }, +} + +export interface StartJobMessage { + action: UploadAction.StartJob, + data: { + jobId: string, + forceOverwrite?: boolean, + }, +} + +export interface RemoveJobMessage { + action: UploadAction.RemoveJob, + data: { + jobId: string, + }, +} + +export interface CleanupJobMessage { + action: UploadAction.CleanupJobs, + data?: {}, +} + +export interface StartAllJobsMessage { + action: UploadAction.StartAllJobs, + data?: {}, +} + +export interface StopAllJobsMessage { + action: UploadAction.StopAllJobs, + data?: {}, +} + +export interface RemoveAllJobsMessage { + action: UploadAction.RemoveAllJobs, + data?: {}, +} + +export interface AddedJobsReplyMessage { + action: UploadAction.AddedJobs, + data: { + filePathnameList: string[], + destInfo: DestInfo, + }, +} + +export interface JobCompletedReplyMessage { + action: UploadAction.JobCompleted, + data: { + jobId: string, + jobUiData: UploadJob["uiData"], + }, +} + +export interface CreatedDirectoryReplyMessage { + action: UploadAction.CreatedDirectory, + data: { + bucket: string, + directoryKey: string, + }, +} + +export type UploadMessage = UpdateConfigMessage + | LoadPersistJobsMessage + | AddJobsMessage + | UpdateUiDataMessage + | StopJobMessage + | WaitJobMessage + | StartJobMessage + | RemoveJobMessage + | CleanupJobMessage + | StartAllJobsMessage + | StopAllJobsMessage + | RemoveAllJobsMessage + +// send actions functions +export class UploadActionFns { + constructor( + private readonly ipc: IpcRenderer, + private readonly channel: string, + ) { + } + + updateConfig(data: UpdateConfigMessage['data']) { + this.ipc.send(this.channel, { + action: UploadAction.UpdateConfig, + data, + }); + } + + loadPersistJobs(data: LoadPersistJobsMessage['data']) { + this.ipc.send(this.channel, { + action: UploadAction.LoadPersistJobs, + data, + }); + } + + addJobs(data: AddJobsMessage['data']) { + this.ipc.send(this.channel, { + action: UploadAction.AddJobs, + data, + }); + } + + updateUiData(data: UpdateUiDataMessage['data']) { + this.ipc.send(this.channel, { + action: UploadAction.UpdateUiData, + data, + }); + } + + waitJob(data: WaitJobMessage['data']) { + this.ipc.send(this.channel, { + action: UploadAction.WaitJob, + data, + }); + } + + startJob(data: StartJobMessage['data']) { + this.ipc.send(this.channel, { + action: UploadAction.StartJob, + data, + }); + } + + stopJob(data: StopJobMessage['data']) { + this.ipc.send(this.channel, { + action: UploadAction.StopJob, + data, + }); + } + + removeJob(data: RemoveJobMessage['data']) { + this.ipc.send(this.channel, { + action: UploadAction.RemoveJob, + data, + }); + } + + cleanUpJobs() { + this.ipc.send(this.channel, { + action: UploadAction.CleanupJobs, + data: {}, + }); + } + + startAllJobs() { + this.ipc.send(this.channel, { + action: UploadAction.StartAllJobs, + data: {}, + }); + } + + stopAllJobs() { + this.ipc.send(this.channel, { + action: UploadAction.StopAllJobs, + data: {}, + }); + } + + removeAllJobs() { + this.ipc.send(this.channel, { + action: UploadAction.RemoveAllJobs, + data: {}, + }); + } +} diff --git a/src/renderer/models/job/_mock-helpers_/data.ts b/src/common/models/job/_mock-helpers_/data.ts similarity index 89% rename from src/renderer/models/job/_mock-helpers_/data.ts rename to src/common/models/job/_mock-helpers_/data.ts index f1eaa1ef..e3ffcd54 100644 --- a/src/renderer/models/job/_mock-helpers_/data.ts +++ b/src/common/models/job/_mock-helpers_/data.ts @@ -3,7 +3,8 @@ export const uploadOptionsFromNewJob = { "accessKey": "NgKd0BmebvsFERFEBfKVVZGeGn7VsZQe_H_AunOC", "secretKey": "lp4Zv3Gi_7CHtxNTcJx2Pum5hUJB3gHROcg4vp0i", ucUrl: "", - "regions": [] + "regions": [], + backendMode: "kodo" as const, }, // ↑ manual add ↓ from file storageClasses: [], // break change from 1.0.16, older task will be uploaded with standard storage class @@ -14,15 +15,16 @@ export const uploadOptionsFromNewJob = { }, "from": { "name": "out.gif", - "path": "/local/path/to/out.gif" + "path": "/local/path/to/out.gif", + "size": 1024, + "mtime": 1651042948862, }, "overwrite": false, storageClassName: "Standard" as const, - backendMode: "kodo" as const, "resumeUpload": false, "multipartUploadSize": 16, "multipartUploadThreshold": 100, - uploadSpeedLimit: 0, // break change from 1.0.16, 0 means no limit + uploadSpeedLimit: 0, // break change from 1.0.16, 0 means no limit. remove persist from 1.0.18 "isDebug": false, userNatureLanguage: 'zh-CN' as const, // break change from 1.0.16 @@ -33,9 +35,11 @@ export const uploadOptionsFromResumeJob = { "accessKey": "NgKd0BmebvsFERFEBfKVVZGeGn7VsZQe_H_AunOC", "secretKey": "lp4Zv3Gi_7CHtxNTcJx2Pum5hUJB3gHROcg4vp0i", ucUrl: "", // break change from 1.0.16, "" means public - "regions": [] + "regions": [], + backendMode: "kodo" as const, }, // ↑ manual add ↓ from file + storageClasses: [], // break change from 1.0.16, older task will be uploaded with standard storage class "region": "cn-east-1", "to": { "bucket": "kodo-browser-dev", @@ -52,7 +56,6 @@ export const uploadOptionsFromResumeJob = { "loaded": 17825792, "resumable": false }, - "status": "stopped", "message": "", "uploadedId": "61a5f55fda9ed605fd263bc2region02z0", "uploadedParts": [ @@ -75,7 +78,7 @@ export const uploadOptionsFromResumeJob = { "resumeUpload": false, "multipartUploadSize": 16, "multipartUploadThreshold": 100, - "uploadSpeedLimit": false, + "uploadSpeedLimit": 0, "isDebug": false, userNatureLanguage: 'zh-CN' as const, // break change from 1.0.16 diff --git a/src/renderer/models/job/base.test.ts b/src/common/models/job/base.test.ts similarity index 100% rename from src/renderer/models/job/base.test.ts rename to src/common/models/job/base.test.ts diff --git a/src/renderer/models/job/base.ts b/src/common/models/job/base.ts similarity index 100% rename from src/renderer/models/job/base.ts rename to src/common/models/job/base.ts diff --git a/src/renderer/models/job/download-job.test.ts b/src/common/models/job/download-job.test.ts similarity index 99% rename from src/renderer/models/job/download-job.test.ts rename to src/common/models/job/download-job.test.ts index 6c498bde..653eabd5 100644 --- a/src/renderer/models/job/download-job.test.ts +++ b/src/common/models/job/download-job.test.ts @@ -8,7 +8,7 @@ jest.mock("electron", () => ({ })); import { ipcRenderer } from "electron"; -import * as AppConfig from "@/const/app-config"; +import * as AppConfig from "@common/const/app-config"; import { EventKey, IpcJobEvent, Status } from "./types"; import { downloadOptionsFromResumeJob } from "./_mock-helpers_/data"; diff --git a/src/renderer/models/job/download-job.ts b/src/common/models/job/download-job.ts similarity index 99% rename from src/renderer/models/job/download-job.ts rename to src/common/models/job/download-job.ts index ba29e71b..14db6ecc 100644 --- a/src/renderer/models/job/download-job.ts +++ b/src/common/models/job/download-job.ts @@ -4,8 +4,8 @@ import { Region } from "kodo-s3-adapter-sdk"; import { StorageClass } from "kodo-s3-adapter-sdk/dist/adapter"; import { NatureLanguage } from "kodo-s3-adapter-sdk/dist/uplog"; -import Duration from "@/const/duration"; -import * as AppConfig from "@/const/app-config"; +import Duration from "@common/const/duration"; +import * as AppConfig from "@common/const/app-config"; import { BackendMode, EventKey, IpcDownloadJob, IpcJobEvent, Status } from "./types"; import Base from "./base"; diff --git a/src/renderer/models/job/index.ts b/src/common/models/job/index.ts similarity index 100% rename from src/renderer/models/job/index.ts rename to src/common/models/job/index.ts diff --git a/src/renderer/models/job/types.ts b/src/common/models/job/types.ts similarity index 98% rename from src/renderer/models/job/types.ts rename to src/common/models/job/types.ts index a45bf004..2da90d87 100644 --- a/src/renderer/models/job/types.ts +++ b/src/common/models/job/types.ts @@ -35,7 +35,6 @@ export interface IpcUploadJob { secretKey: string, ucUrl?: string, regions: Region[], - backendMode: BackendMode, userNatureLanguage: NatureLanguage, }, diff --git a/src/common/models/job/upload-job.test.ts b/src/common/models/job/upload-job.test.ts new file mode 100644 index 00000000..e31f6333 --- /dev/null +++ b/src/common/models/job/upload-job.test.ts @@ -0,0 +1,60 @@ +import { Status } from "./types"; +import {uploadOptionsFromNewJob, uploadOptionsFromResumeJob} from "./_mock-helpers_/data"; + +import UploadJob from "./upload-job"; + +describe("test models/job/upload-job.ts", () => { + describe("test stop", () => { + it("stop", () => { + const uploadJob = new UploadJob(uploadOptionsFromNewJob); + const spiedEmit = jest.spyOn(uploadJob, "emit"); + spiedEmit.mockImplementation((_eventName: string, ..._args: any[]) => uploadJob); + expect(uploadJob.stop()).toBe(uploadJob); + expect(uploadJob.speed).toBe(0); + expect(uploadJob.predictLeftTime).toBe(0); + expect(uploadJob.status).toBe(Status.Stopped); + expect(uploadJob.emit).toBeCalledWith("statuschange", "stopped"); + }); + }); + + // describe("test start", () => { + // it("start()", () => { + // const uploadJob = new UploadJob(uploadOptionsFromNewJob); + // const spiedEmit = jest.spyOn(uploadJob, "emit"); + // spiedEmit.mockImplementation((_eventName: string, ..._args: any[]) => uploadJob); + // expect(uploadJob.start()).toBe(uploadJob); + // expect(uploadJob.message).toBe(""); + // + // // private status flow + // expect(uploadJob.emit).toBeCalledWith("statuschange", Status.Running); + // expect(uploadJob.status).toBe(Status.Running); + // + // // startSpeedCounter flow + // expect(uploadJob.speed).toBe(0); + // expect(uploadJob.predictLeftTime).toBe(0); + // uploadJob.stop(); + // }); + // }); + + describe("test resume upload job", () => { + it("test get persistInfo", () => { + const uploadJob = new UploadJob(uploadOptionsFromResumeJob); + expect(uploadJob.persistInfo).toStrictEqual({ + from: uploadOptionsFromResumeJob.from, + storageClasses: uploadOptionsFromResumeJob.storageClasses, + region: uploadOptionsFromResumeJob.region, + to: uploadOptionsFromResumeJob.to, + overwrite: uploadOptionsFromResumeJob.overwrite, + storageClassName: uploadOptionsFromResumeJob.storageClassName, + backendMode: uploadOptionsFromResumeJob.backendMode, + prog: uploadOptionsFromResumeJob.prog, + status: Status.Waiting, + message: uploadOptionsFromResumeJob.message, + uploadedId: uploadOptionsFromResumeJob.uploadedId, + uploadedParts: uploadOptionsFromResumeJob.uploadedParts.map(p => ({ PartNumber: p.partNumber, ETag: p.etag })), + multipartUploadThreshold: uploadOptionsFromResumeJob.multipartUploadThreshold, + multipartUploadSize: uploadOptionsFromResumeJob.multipartUploadSize, + }); + }); + }); +}); diff --git a/src/common/models/job/upload-job.ts b/src/common/models/job/upload-job.ts new file mode 100644 index 00000000..4f9e1319 --- /dev/null +++ b/src/common/models/job/upload-job.ts @@ -0,0 +1,496 @@ +import {promises as fsPromises} from 'fs'; + +// @ts-ignore +import mime from "mime"; +import lodash from "lodash"; +import {Qiniu, Region, Uploader} from "kodo-s3-adapter-sdk"; +import {Adapter, Part, StorageClass} from "kodo-s3-adapter-sdk/dist/adapter"; +import {RecoveredOption} from "kodo-s3-adapter-sdk/dist/uploader"; +import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; + +import Duration from "@common/const/duration"; +import ByteSize from "@common/const/byte-size"; +import * as AppConfig from "@common/const/app-config"; + +import {BackendMode, Status, UploadedPart} from "./types"; +import Base from "./base" +import * as Utils from "./utils"; + +// if change options, remember to check PersistInfo +interface RequiredOptions { + clientOptions: { + accessKey: string, + secretKey: string, + ucUrl: string, + regions: Region[], + backendMode: BackendMode, + }, + + from: Required, + to: Utils.RemotePath, + region: string, + + overwrite: boolean, + storageClassName: StorageClass["kodoName"], + storageClasses: StorageClass[], +} + +interface UploadOptions { + multipartUploadThreshold: number, // Bytes + multipartUploadSize: number, // Bytes + uploadSpeedLimit: number, // Bytes/s + + isDebug: boolean, + + // could be removed if there is a better uplog + userNatureLanguage: NatureLanguage, +} + +interface OptionalOptions extends UploadOptions{ + id: string, + uploadedId: string, + uploadedParts: UploadedPart[], + + status: Status, + + prog: { + total: number, // Bytes + loaded: number, // Bytes + resumable?: boolean, + }, + + message: string, +} + +export type Options = RequiredOptions & Partial + +const DEFAULT_OPTIONS: OptionalOptions = { + id: '', + + multipartUploadThreshold: 10 * ByteSize.MB, + multipartUploadSize: 4 * ByteSize.MB, + uploadSpeedLimit: 0, // 0 means no limit + uploadedId: "", + uploadedParts: [], + + status: Status.Waiting, + + prog: { + total: 0, + loaded: 0, + }, + + message: "", + isDebug: false, + + userNatureLanguage: "zh-CN", +}; + +type PersistInfo = { + from: RequiredOptions['from'], + storageClasses: RequiredOptions['storageClasses'], + region: RequiredOptions['region'], + to: RequiredOptions['to'], + overwrite: RequiredOptions['overwrite'], + storageClassName: RequiredOptions['storageClassName'], + // if we can remove backendMode? + // because we always use the client current backendMode. + backendMode: RequiredOptions['clientOptions']['backendMode'], + prog: OptionalOptions['prog'], + status: OptionalOptions['status'], + message: OptionalOptions['message'], + uploadedId: OptionalOptions['uploadedId'], + // ugly. if can do some break changes, make it be + // `uploadedParts: OptionalOptions['uploadedParts'],` + uploadedParts: { + PartNumber: UploadedPart['partNumber'], + ETag: UploadedPart['etag'], + }[], + multipartUploadThreshold: OptionalOptions["multipartUploadThreshold"], + multipartUploadSize: OptionalOptions["multipartUploadSize"], +} + +export default class UploadJob extends Base { + static fromPersistInfo( + id: string, + persistInfo: PersistInfo, + clientOptions: RequiredOptions['clientOptions'], + uploadOptions: { + uploadSpeedLimit: number, + isDebug: boolean, + userNatureLanguage: NatureLanguage, + }, + ): UploadJob { + return new UploadJob({ + id, + status: persistInfo.status, + message: persistInfo.message, + + from: persistInfo.from, + to: persistInfo.to, + prog: persistInfo.prog, + + clientOptions, + storageClasses: persistInfo.storageClasses, + + overwrite: persistInfo.overwrite, + region: persistInfo.region, + storageClassName: persistInfo.storageClassName, + + uploadedId: persistInfo.uploadedId, + uploadedParts: persistInfo.uploadedParts.map(part => ({ + partNumber: part.PartNumber, + etag: part.ETag, + })), + + multipartUploadThreshold: persistInfo.multipartUploadThreshold, + multipartUploadSize: persistInfo.multipartUploadSize, + uploadSpeedLimit: uploadOptions.uploadSpeedLimit, + isDebug: uploadOptions.isDebug, + + userNatureLanguage: uploadOptions.userNatureLanguage, + }); + } + + // - create options - + private readonly options: Readonly + + // - for job save and log - + readonly id: string + readonly kodoBrowserVersion: string + private isForceOverwrite: boolean = false + + // - for UI - + private __status: Status + // speed + speedTimerId?: number = undefined + speed: number = 0 // Bytes/s + predictLeftTime: number = 0 // seconds + // message + message: string + + // - for resume from break point - + prog: OptionalOptions["prog"] + uploadedId: string + uploadedParts: UploadedPart[] + + // - for process control - + uploader?: Uploader + + constructor(config: Options) { + super(); + this.id = config.id + ? config.id + : `uj-${new Date().getTime()}-${Math.random().toString().substring(2)}`; + this.kodoBrowserVersion = AppConfig.app.version; + + this.options = lodash.merge({}, DEFAULT_OPTIONS, config); + + this.__status = this.options.status; + + this.prog = { + ...this.options.prog, + } + this.uploadedId = this.options.uploadedId; + this.uploadedParts = [ + ...this.options.uploadedParts, + ]; + + this.message = this.options.message; + + // hook functions + this.startUpload = this.startUpload.bind(this); + this.handleProgress = this.handleProgress.bind(this); + this.handlePartsInit = this.handlePartsInit.bind(this); + this.handlePartPutted = this.handlePartPutted.bind(this); + } + + get accessKey(): string { + return this.options.clientOptions.accessKey; + } + + // TypeScript specification (8.4.3) says... + // > Accessors for the same member name must specify the same accessibility + private set _status(value: Status) { + this.__status = value; + this.emit("statuschange", this.status); + + if ( + this.status === Status.Failed + || this.status === Status.Stopped + || this.status === Status.Finished + || this.status === Status.Duplicated + ) { + this.stopSpeedCounter(); + } + } + + get status(): Status { + return this.__status + } + + get isNotRunning(): boolean { + return this.status !== Status.Running; + } + + get uiData() { + return { + id: this.id, + from: this.options.from, + to: this.options.to, + status: this.status, + speed: this.speed, + estimatedTime: this.predictLeftTime, + progress: this.prog, + message: this.message, + } + } + + async start( + forceOverwrite: boolean = false, + ): Promise { + if (this.status === Status.Running || this.status === Status.Finished) { + return; + } + + if (forceOverwrite) { + this.isForceOverwrite = true; + } + + if (this.options.isDebug) { + console.log(`Try uploading ${this.options.from.path} to kodo://${this.options.to.bucket}/${this.options.to.key}`); + } + + this.message = "" + + this._status = Status.Running; + + // create client + const qiniu = new Qiniu( + this.options.clientOptions.accessKey, + this.options.clientOptions.secretKey, + this.options.clientOptions.ucUrl, + `Kodo-Browser/${this.kodoBrowserVersion}/ioutil`, + this.options.clientOptions.regions, + ); + const qiniuClient = qiniu.mode( + this.options.clientOptions.backendMode, + { + appName: 'kodo-browser/ioutil', + appVersion: this.kodoBrowserVersion, + appNatureLanguage: this.options.userNatureLanguage, + // disable uplog when use customize cloud + // because there isn't a valid access key of uplog + uplogBufferSize: this.options.clientOptions.ucUrl ? -1 : undefined, + requestCallback: () => { + }, + responseCallback: () => { + }, + }, + ); + + // upload + this.startSpeedCounter(); + await qiniuClient.enter( + 'uploadFile', + this.startUpload, + { + targetBucket: this.options.to.bucket, + targetKey: this.options.to.key, + }, + ).catch(err => { + if (err === Uploader.userCanceledError) { + this._status = Status.Stopped; + return; + } + this._status = Status.Failed; + this.message = err.toString(); + }); + } + + private async startUpload(client: Adapter) { + client.storageClasses = this.options.storageClasses; + const isOverwrite = this.isForceOverwrite || this.options.overwrite; + if (!isOverwrite) { + const isExists = await client.isExists( + this.options.region, + { + bucket: this.options.to.bucket, + key: this.options.to.key, + }, + ); + if (isExists) { + this._status = Status.Duplicated; + return; + } + } + + this.uploader = new Uploader(client); + const fileHandle = await fsPromises.open(this.options.from.path, 'r'); + await this.uploader.putObjectFromFile( + this.options.region, + { + bucket: this.options.to.bucket, + key: this.options.to.key, + storageClassName: this.options.storageClassName, + }, + fileHandle, + this.options.from.size, + this.options.from.name, + { + header: { + contentType: mime.getType(this.options.from.path) + }, + recovered: this.uploadedId && this.uploadedParts + ? { + uploadId: this.uploadedId, + parts: this.uploadedParts, + } + : undefined, + uploadThreshold: this.options.multipartUploadThreshold, + partSize: this.options.multipartUploadSize, + putCallback: { + partsInitCallback: this.handlePartsInit, + partPutCallback: this.handlePartPutted, + progressCallback: this.handleProgress, + }, + uploadThrottleOption: this.options.uploadSpeedLimit > 0 + ? { + rate: this.options.uploadSpeedLimit, + } + : undefined, + } + ); + this._status = Status.Finished; + + await fileHandle.close(); + this.emit("complete"); + } + + stop(): this { + if (this.status === Status.Stopped) { + return this; + } + this._status = Status.Stopped; + + if (this.options.isDebug) { + console.log(`Pausing ${this.options.from.path}`); + } + + if (!this.uploader) { + return this; + } + this.uploader.abort(); + this.uploader = undefined; + + return this; + } + + wait(): this { + if (this.status === Status.Waiting) { + return this; + } + this._status = Status.Waiting; + + if (this.options.isDebug) { + console.log(`Pending ${this.options.from.path}`); + } + + if (!this.uploader) { + return this; + } + this.uploader.abort(); + this.uploader = undefined; + + return this; + } + + private startSpeedCounter() { + this.stopSpeedCounter(); + + let lastTimestamp = new Date().getTime(); + let lastLoaded = this.prog.loaded; + let zeroSpeedCounter = 0; + const intervalDuration = Duration.Second; + this.speedTimerId = setInterval(() => { + if (this.isNotRunning) { + this.stopSpeedCounter(); + return; + } + + const nowTimestamp = new Date().getTime(); + const currentSpeed = (this.prog.loaded - lastLoaded) / ((nowTimestamp - lastTimestamp) / Duration.Second); + if (currentSpeed < 1 && zeroSpeedCounter < 3) { + zeroSpeedCounter += 1; + return; + } + + this.speed = Math.round(currentSpeed); + this.predictLeftTime = Math.max( + Math.round((this.prog.total - this.prog.loaded) / this.speed) * Duration.Second, + 0, + ); + + lastLoaded = this.prog.loaded; + lastTimestamp = nowTimestamp; + zeroSpeedCounter = 0; + }, intervalDuration) as unknown as number; // hack type problem of nodejs and browser + } + + private stopSpeedCounter() { + this.speed = 0; + this.predictLeftTime = 0; + clearInterval(this.speedTimerId); + } + + private handleProgress(uploaded: number, total: number) { + if (!this.uploader) { + return; + } + this.prog.loaded = uploaded; + this.prog.total = total; + + this.emit("progress", lodash.merge({}, this.prog)); + } + + private handlePartsInit(initInfo: RecoveredOption) { + this.uploadedId = initInfo.uploadId; + this.uploadedParts = initInfo.parts; + } + + private handlePartPutted(part: Part) { + if (!this.uploader) { + return; + } + this.uploadedParts.push(part); + + this.emit("partcomplete", lodash.merge({}, part)); + } + + get persistInfo(): PersistInfo { + return { + from: this.options.from, + storageClasses: this.options.storageClasses, + region: this.options.region, + to: this.options.to, + overwrite: this.options.overwrite, + storageClassName: this.options.storageClassName, + backendMode: this.options.clientOptions.backendMode, + + // real-time info + prog: { + loaded: this.prog.loaded, + total: this.prog.total, + resumable: this.prog.resumable + }, + status: this.status, + message: this.message, + uploadedId: this.uploadedId, + uploadedParts: this.uploadedParts.map((part) => { + return {PartNumber: part.partNumber, ETag: part.etag}; + }), + multipartUploadThreshold: this.options.multipartUploadThreshold, + multipartUploadSize: this.options.multipartUploadSize, + }; + } +} diff --git a/src/renderer/models/job/utils.test.ts b/src/common/models/job/utils.test.ts similarity index 100% rename from src/renderer/models/job/utils.test.ts rename to src/common/models/job/utils.test.ts diff --git a/src/renderer/models/job/utils.ts b/src/common/models/job/utils.ts similarity index 96% rename from src/renderer/models/job/utils.ts rename to src/common/models/job/utils.ts index 32a06f65..7145d5a8 100644 --- a/src/renderer/models/job/utils.ts +++ b/src/common/models/job/utils.ts @@ -8,8 +8,8 @@ import * as KodoNav from "@/const/kodo-nav" export interface LocalPath { name: string, path: string, - size?: number, - mtime?: number, + size?: number, // bytes + mtime?: number, // ms timestamp } // parseLocalPath: get name and path from local path export function parseLocalPath(p: string): LocalPath { @@ -22,8 +22,8 @@ export function parseLocalPath(p: string): LocalPath { export interface RemotePath { bucket: string, key: string, - size?: number, - mtime?: number, + size?: number, // bytes + mtime?: number, // ms timestamp } // parseKodoPath: get bucket and key from KodoPath export function parseKodoPath(kodoPath: string): RemotePath { diff --git a/src/common/models/storage-class.ts b/src/common/models/storage-class.ts new file mode 100644 index 00000000..23af7e0e --- /dev/null +++ b/src/common/models/storage-class.ts @@ -0,0 +1,7 @@ +export default interface StorageClass { + fileType: number, + kodoName: string, + s3Name: string, + billingI18n: Record, + nameI18n: Record, +} diff --git a/src/main/index.js b/src/main/index.js index a63a5fd2..4cc9b185 100755 --- a/src/main/index.js +++ b/src/main/index.js @@ -14,6 +14,7 @@ const { fork } = require("child_process"); const { UplogBuffer } = require("kodo-s3-adapter-sdk/dist/uplog"); +const { UploadAction } = require("@common/ipc-actions/upload"); ///***************************************** let root = path.dirname(__dirname); @@ -30,6 +31,7 @@ const iconRoot = path.join(root, 'renderer', 'icons') let win; let winBlockedTid; // time interval for close let forkedWorkers = new Map(); +let uploadRunning = 0; switch (process.platform) { case "darwin": @@ -70,7 +72,11 @@ let createWindow = () => { }; let confirmForWorkers = (e) => { - if (forkedWorkers.size <= 0) { + const runningJobs = forkedWorkers.size - + 1 + // upload-worker + uploadRunning; // upload running jobs; + + if (runningJobs <= 0) { return; } @@ -119,8 +125,9 @@ let createWindow = () => { // cancel close e.preventDefault(); + clearInterval(winBlockedTid); winBlockedTid = setInterval(() => { - if (forkedWorkers.size > 0) { + if (runningJobs > 0) { return; } @@ -145,7 +152,7 @@ let createWindow = () => { // prevent if there still alive workers. confirmCb(dialog.showMessageBox({ type: "warning", - message: `There ${forkedWorkers.size > 1 ? "are" : "is"} still ${forkedWorkers.size} ${forkedWorkers.size > 1 ? "jobs" : "job"} in processing, are you sure to quit?`, + message: `There ${runningJobs > 1 ? "are" : "is"} still ${runningJobs} ${runningJobs > 1 ? "jobs" : "job"} in processing, are you sure to quit?`, buttons: btns })); }; @@ -233,6 +240,41 @@ let createWindow = () => { ///***************************************** // listener events send from renderer process +ipcMain.on("UploaderManager", (event, message) => { + const processName = "UploaderProcess"; + let uploaderProcess = forkedWorkers.get(processName); + if (!uploaderProcess) { + uploaderProcess = fork( + path.join(root, "main", "uploader-bundle.js"), + // is there a better way to pass parameters? + ['--config-json', JSON.stringify({resumeUpload: true, maxConcurrency: 5})], + { + cwd: root, + silent: false, + }, + ); + forkedWorkers.set(processName, uploaderProcess); + + uploaderProcess.on("exit", () => { + forkedWorkers.delete(processName) + }); + + uploaderProcess.on("message", (message) => { + if (win && !win.isDestroyed()) { + event.sender.send("UploaderManager-reply", message); + } + switch (message.action) { + case UploadAction.UpdateUiData: { + uploadRunning = message.data.running; + break; + } + } + }); + } + + uploaderProcess.send(message); +}); + ipcMain.on("asynchronous", (event, data) => { switch (data.key) { case "getStaticServerPort": @@ -626,7 +668,11 @@ app.on("window-all-closed", () => { // On OS X it is common for applications and their menu bar // to stay active until the user quits explicitly with Cmd + Q //if (process.platform !== 'darwin') { - app.quit(); + + // resolve inflight jobs persisted status not correct + setTimeout(() => { + app.quit(); + }, 3000); //} }); diff --git a/src/main/upload-worker.js b/src/main/upload-worker.js deleted file mode 100644 index 249afc35..00000000 --- a/src/main/upload-worker.js +++ /dev/null @@ -1,101 +0,0 @@ -import { - createClient -} from '../common/qiniu-store/lib/ioutil' - -process.on('uncaughtException', function (err) { - process.send({ - key: 'error', - error: err.message, - stack: err.stack.split("\n") - }); -}); - -process.send({ - key: 'env', - execPath: process.execPath, - version: process.version -}); - -process.on('message', function (msg) { - switch (msg.key) { - case 'stop': - process.exit(0); - break; - - case 'start': - global.Global = { - app: { - id: 'kodo-browser', - logo: 'icons/icon.png', - version: msg.data.options.kodoBrowserVersion, - }, - }; - const client = createClient(msg.data.clientOptions, msg.data.options); - - const uploader = client.uploadFile(msg.data.params); - uploader.on('fileDuplicated', (prog) => { - process.send({ - job: msg.data.job, - key: 'fileDuplicated' - }); - }); - uploader.on('fileStat', (prog) => { - process.send({ - job: msg.data.job, - key: 'fileStat', - data: { - progressLoaded: 0, - progressTotal: prog.progressTotal, - progressResumable: prog.progressResumable - } - }); - }); - uploader.on('progress', (prog) => { - process.send({ - job: msg.data.job, - key: 'progress', - data: { - progressLoaded: prog.progressLoaded, - progressTotal: prog.progressTotal, - progressResumable: prog.progressResumable - } - }); - }); - uploader.on('filePartUploaded', (part) => { - process.send({ - job: msg.data.job, - key: 'filePartUploaded', - data: part || {} - }); - }); - uploader.on('fileUploaded', (result) => { - process.send({ - job: msg.data.job, - key: 'fileUploaded', - data: result || {} - }); - }); - uploader.on('error', (err) => { - process.send({ - job: msg.data.job, - key: 'error', - error: err - }); - }); - uploader.on('debug', (data) => { - process.send({ - job: msg.data.job, - key: 'debug', - data: data - }); - }); - - break; - - default: - process.send({ - key: `[Error] ${msg.key}`, - message: msg - }); - } -}); diff --git a/src/main/uploader/boundary-const.ts b/src/main/uploader/boundary-const.ts new file mode 100644 index 00000000..7e73bead --- /dev/null +++ b/src/main/uploader/boundary-const.ts @@ -0,0 +1,4 @@ +import ByteSize from "@common/const/byte-size"; + +export const MIN_MULTIPART_SIZE = 4 * ByteSize.MB; +export const MAX_MULTIPART_COUNT = 10000; diff --git a/src/main/uploader/index.ts b/src/main/uploader/index.ts new file mode 100644 index 00000000..7c9ea218 --- /dev/null +++ b/src/main/uploader/index.ts @@ -0,0 +1,176 @@ +import { + AddedJobsReplyMessage, + CreatedDirectoryReplyMessage, + JobCompletedReplyMessage, + UpdateUiDataReplyMessage, + UploadAction, + UploadMessage +} from "@common/ipc-actions/upload"; +import UploadManager from "./upload-manager"; +import UploadJob from "@common/models/job/upload-job"; +import {Status} from "@common/models/job/types"; + +// initial UploadManager Config from argv after `--config-json` +const configStr = process.argv.find((_arg, i, arr) => arr[i - 1] === "--config-json"); +const uploadManagerConfig = configStr ? JSON.parse(configStr) : {}; +uploadManagerConfig.onJobDone = handleJobDone; +uploadManagerConfig.onCreatedDirectory = handleCreatedDirectory; +const uploadManager = new UploadManager(uploadManagerConfig); + +process.on("uncaughtException", (err) => { + uploadManager.persistJobs(true); + console.error(err); +}); + +process.on("message", (message: UploadMessage) => { + switch (message.action) { + case UploadAction.UpdateConfig: { + uploadManager.updateConfig(message.data); + break; + } + case UploadAction.LoadPersistJobs: { + uploadManager.loadJobsFromStorage( + message.data.clientOptions, + message.data.uploadOptions, + ); + break; + } + case UploadAction.AddJobs: { + uploadManager.createUploadJobs( + message.data.filePathnameList, + message.data.destInfo, + message.data.uploadOptions, + message.data.clientOptions, + { + jobsAdding: () => { + uploadManager.persistJobs(); + }, + jobsAdded: () => { + const replyMessage: AddedJobsReplyMessage = { + action: UploadAction.AddedJobs, + data: { + filePathnameList: message.data.filePathnameList, + destInfo: message.data.destInfo, + }, + } + process.send?.(replyMessage); + } + } + ); + break; + } + case UploadAction.UpdateUiData: { + const replyMessage: UpdateUiDataReplyMessage = { + action: UploadAction.UpdateUiData, + data: uploadManager.getJobsUiDataByPage( + message.data.pageNum, + message.data.count, + message.data.query, + ), + } + process.send?.(replyMessage); + break; + } + case UploadAction.StopJob: { + uploadManager.stopJob(message.data.jobId); + break; + } + case UploadAction.WaitJob: { + uploadManager.waitJob(message.data.jobId); + break; + } + case UploadAction.StartJob: { + uploadManager.startJob(message.data.jobId, message.data.forceOverwrite); + break; + } + case UploadAction.RemoveJob: { + uploadManager.removeJob(message.data.jobId); + uploadManager.persistJobs(); + break; + } + case UploadAction.CleanupJobs: { + uploadManager.cleanupJobs(); + break; + } + case UploadAction.StartAllJobs: { + uploadManager.startAllJobs(); + break; + } + case UploadAction.StopAllJobs: { + uploadManager.stopAllJobs(); + break; + } + case UploadAction.RemoveAllJobs: { + uploadManager.removeAllJobs(); + uploadManager.persistJobs(); + break; + } + default: { + console.warn("Upload Manager received unknown action, message:", message); + } + } +}); + +let isCleanup = false; +function handleExit() { + if (isCleanup) { + return Promise.resolve(); + } + + isCleanup = true; + + uploadManager.stopAllJobs({ + matchStatus: [Status.Waiting], + }); + + return new Promise((resolve, reject) => { + try { + // resolve inflight jobs persisted status not correct + setTimeout(() => { + uploadManager.stopAllJobs({ + matchStatus: [Status.Running], + }); + uploadManager.persistJobs(true); + resolve(); + }, 2000); + } catch { + reject() + } + }); +} + + +process.on("exit", () => { + handleExit() +}); + +process.on('SIGTERM', () => { + handleExit() + .then(() => { + process.exit(0); + }); +}); + +function handleJobDone(jobId: string, job?: UploadJob) { + if (job?.status === Status.Finished) { + const jobCompletedReplyMessage: JobCompletedReplyMessage = { + action: UploadAction.JobCompleted, + data: { + jobId, + jobUiData: job.uiData, + }, + }; + process.send?.(jobCompletedReplyMessage); + } +} + +function handleCreatedDirectory(bucket: string, directoryKey: string) { + const createdDirectoryReplyMessage: CreatedDirectoryReplyMessage = { + action: UploadAction.CreatedDirectory, + data: { + bucket, + directoryKey, + }, + }; + process.send?.(createdDirectoryReplyMessage); +} diff --git a/src/main/uploader/upload-manager.ts b/src/main/uploader/upload-manager.ts new file mode 100644 index 00000000..cc6692fa --- /dev/null +++ b/src/main/uploader/upload-manager.ts @@ -0,0 +1,553 @@ +import path from "path"; +import fs, {Stats} from "fs"; +import lodash from "lodash"; +// @ts-ignore +import Walk from "@root/walk"; +import {Adapter} from "kodo-s3-adapter-sdk/dist/adapter"; + +import ByteSize from "@common/const/byte-size"; +import {ClientOptions, DestInfo, UploadOptions} from "@common/ipc-actions/upload"; +import UploadJob from "@common/models/job/upload-job"; +import {Status} from "@common/models/job/types"; + +import createQiniuClient from "../util/createClient"; +import {MAX_MULTIPART_COUNT, MIN_MULTIPART_SIZE} from "./boundary-const"; + +// for walk +interface StatsWithName extends Stats { + name: string, +} + +// Manager +interface ManagerConfig { + resumeUpload: boolean, + maxConcurrency: number, + multipartUploadSize: number, // Bytes + multipartUploadThreshold: number, // Bytes + uploadSpeedLimit: number, // Bytes/s + isDebug: boolean, + isSkipEmptyDirectory: boolean + persistPath: string, + + onError?: (err: Error) => void, + onJobDone?: (id: string, job?: UploadJob) => void, + onCreatedDirectory?: (bucket: string, directoryKey: string) => void, +} + +const defaultManagerConfig: ManagerConfig = { + resumeUpload: false, + maxConcurrency: 10, + multipartUploadSize: 4 * ByteSize.MB, // 4MB + multipartUploadThreshold: 10 * ByteSize.MB, // 10MB + uploadSpeedLimit: 0, + isDebug: false, + isSkipEmptyDirectory: false, + persistPath: "", +} + +export default class UploadManager { + private concurrency: number = 0; + private jobs: Map = new Map() + private jobIds: UploadJob["id"][] = [] + private config: Readonly + + constructor(config: Partial) { + this.config = { + ...defaultManagerConfig, + ...config, + }; + } + + get jobsLength() { + return this.jobIds.length; + } + + get jobsSummary(): { + total: number, + finished: number, + running: number, + failed: number, + stopped: number, + } { + let finished = 0; + let failed = 0; + let stopped = 0; + this.jobIds.forEach((id) => { + switch (this.jobs.get(id)?.status) { + case Status.Finished: { + finished += 1; + break; + } + case Status.Failed: { + failed += 1; + break; + } + case Status.Stopped: { + stopped += 1; + break; + } + } + }); + return { + total: this.jobIds.length, + finished: finished, + running: this.concurrency, + failed: failed, + stopped: stopped, + } + } + + updateConfig(config: Partial) { + this.config = { + ...this.config, + ...config, + }; + } + + async createUploadJobs( + filePathnameList: string[], // local file path, required absolute path + destInfo: DestInfo, + uploadOptions: UploadOptions, + clientOptions: ClientOptions, + hooks?: { + jobsAdding?: () => void, + jobsAdded?: () => void, + }, + ) { + const qiniu = createQiniuClient( + clientOptions, + { + userNatureLanguage: uploadOptions.userNatureLanguage, + isDebug: this.config.isDebug, + }, + ); + const walk = Walk.create({ + withFileStats: true, + }); + + for (const filePathname of filePathnameList) { + const directoryToCreate = new Map(); + const remoteBaseDirectory = destInfo.key.endsWith("/") + ? destInfo.key.slice(0, -1) + : destInfo.key; + const localBaseDirectory = path.dirname(filePathname); + + await walk( + filePathname, + async (err: Error, walkingPathname: string, statsWithName: StatsWithName): Promise => { + if (err) { + this.config.onError?.(err); + return + } + + // remoteKey should be "path/to/file" not "/path/to/file" + let remoteKey = remoteBaseDirectory + walkingPathname.slice(localBaseDirectory.length); + // for windows path + if (path.sep === "\\") { + remoteKey = remoteKey.replace(/\\/g, "/"); + } + remoteKey = remoteKey.startsWith("/") ? remoteKey.slice(1) : remoteKey; + + // if enable skip empty directory upload. + const remoteDirectoryKey = path.dirname(remoteKey) + "/"; + if (remoteDirectoryKey !== "./" && !directoryToCreate.get(remoteDirectoryKey)) { + this.createDirectory( + qiniu, + { + region: destInfo.regionId, + bucketName: destInfo.bucketName, + key: remoteDirectoryKey, + directoryName: path.basename(remoteDirectoryKey), + }, + ) + .then(data => this.afterCreateDirectory(data)); + directoryToCreate.set(remoteDirectoryKey, true); + } + + if (statsWithName.isDirectory()) { + // we need to determine whether the directory is empty. + // and in this electron version(nodejs v10.x) read the directory again + // is too waste. so we create later. + // if we are nodejs > v12.12,use opendir API to determine empty. + if (!this.config.isSkipEmptyDirectory) { + const remoteDirectoryKey = remoteKey + "/"; + this.createDirectory( + qiniu, + { + region: destInfo.regionId, + bucketName: destInfo.bucketName, + key: remoteDirectoryKey, + directoryName: statsWithName.name, + }, + ) + .then(data => this.afterCreateDirectory(data)); + directoryToCreate.set(remoteDirectoryKey, true); + } + } else if (statsWithName.isFile()) { + const from = { + name: statsWithName.name, + path: walkingPathname, + size: statsWithName.size, + mtime: statsWithName.mtime.getTime(), + }; + const to = { + bucket: destInfo.bucketName, + key: remoteKey, + }; + this.createUploadJob(from, to, uploadOptions, clientOptions, destInfo.regionId); + + // post add job + hooks?.jobsAdding?.(); + this.scheduleJobs(); + } else { + console.warn("file can't upload", "local:", walkingPathname, "remoteKey:", remoteKey); + } + }, + ); + } + hooks?.jobsAdded?.(); + } + + private async createDirectory( + client: Adapter, + options: { + region: string, + bucketName: string, + key: string, + directoryName: string, + }, + ) { + await client.enter("createFolder", async client => { + await client.putObject( + options.region, + { + bucket: options.bucketName, + key: options.key, + }, + Buffer.alloc(0), + options.directoryName, + ); + }, { + targetBucket: options.bucketName, + targetKey: options.key, + }); + return { + bucket: options.bucketName, + key: options.key, + }; + } + + private createUploadJob( + from: Required, + to: UploadJob["options"]["to"], + uploadOptions: UploadOptions, + clientOptions: ClientOptions, + regionId: string, + ): void { + // parts count + const partsCount = Math.ceil(from.size / this.config.multipartUploadSize); + + // part size + let partSize = this.config.multipartUploadSize; + if (partsCount > MAX_MULTIPART_COUNT) { + partSize = Math.ceil(from.size / MAX_MULTIPART_COUNT); + if (partSize < MIN_MULTIPART_SIZE) { + partSize = MIN_MULTIPART_SIZE + } else { + // Why? + partSize += MIN_MULTIPART_SIZE - partSize % MIN_MULTIPART_SIZE + } + } + + const job = new UploadJob({ + from: from, + to: to, + prog: { + loaded: 0, + total: from.size, + resumable: this.config.resumeUpload && from.size > this.config.multipartUploadThreshold, + }, + + clientOptions: { + accessKey: clientOptions.accessKey, + secretKey: clientOptions.secretKey, + ucUrl: clientOptions.ucUrl, + regions: clientOptions.regions, + backendMode: clientOptions.backendMode, + }, + storageClasses: uploadOptions.storageClasses, + + overwrite: uploadOptions.isOverwrite, + region: regionId, + storageClassName: uploadOptions.storageClassName, + + multipartUploadThreshold: this.config.multipartUploadThreshold, + multipartUploadSize: partSize, + uploadSpeedLimit: this.config.uploadSpeedLimit, + isDebug: this.config.isDebug, + + userNatureLanguage: uploadOptions.userNatureLanguage, + }); + + this.addJob(job); + } + + private addJob(job: UploadJob) { + job.on("partcomplete", () => { + this.persistJobs(); + return false; + }); + job.on("complete", () => { + this.persistJobs(); + return false; + }); + + this.jobs.set(job.id, job); + this.jobIds.push(job.id); + } + + public getJobsUiDataByPage(pageNum: number = 0, count: number = 10, query?: { status?: Status, name?: string }) { + let list: (UploadJob["uiData"] | undefined)[]; + if (query) { + list = this.jobIds.map(id => this.jobs.get(id)?.uiData) + .filter(job => { + const matchStatus = query.status + ? job?.status === query.status + : true; + const matchName = query.name + ? job?.from.name.includes(query.name) + : true; + return matchStatus && matchName; + }) + .slice(pageNum, pageNum * count + count); + } else { + list = this.jobIds.slice(pageNum, pageNum * count + count) + .map(id => this.jobs.get(id)?.uiData); + } + return { + list, + ...this.jobsSummary, + }; + } + + public getJobsUiDataByIds(ids: UploadJob["id"][]) { + return { + list: ids.filter(id => this.jobs.has(id)) + .map(id => this.jobs.get(id)?.uiData), + ...this.jobsSummary, + }; + } + + public persistJobs(force: boolean = false): void { + if (force) { + this._persistJobs(); + return; + } + this._persistJobsThrottle(); + } + + private _persistJobsThrottle = lodash.throttle(this._persistJobs, 1000); + + private _persistJobs(): void { + if (!this.config.persistPath) { + return; + } + const persistData: Record = {}; + this.jobIds.forEach(id => { + const job = this.jobs.get(id); + if (!job || job.status === Status.Finished) { + return; + } + persistData[id] = job.persistInfo; + }); + fs.writeFileSync( + this.config.persistPath, + JSON.stringify(persistData), + ); + } + + public loadJobsFromStorage( + clientOptions: Pick, + uploadOptions: Pick + ): void { + if (!this.config.persistPath) { + return; + } + const persistedJobs: Record = JSON.parse(fs.readFileSync(this.config.persistPath, "utf-8")); + Object.entries(persistedJobs) + .forEach(([jobId, persistedJob]) => { + if (this.jobs.get(jobId)) { + return + } + + if (!persistedJob.from) { + this.config.onError?.(new Error("load jobs from storage error: lost job.from")); + return; + } + + if (!fs.existsSync(persistedJob.from.path)) { + this.config.onError?.(new Error(`load jobs from storage error: local file not found\nfile path: ${persistedJob.from.path}`)); + return; + } + + // TODO: Is the `if` useless? Why `size` or `mtime` doesn't exist? + if (!persistedJob.from?.size || !persistedJob.from?.mtime) { + persistedJob.prog.loaded = 0; + persistedJob.uploadedParts = []; + } + + const fileStat = fs.statSync(persistedJob.from.path); + if ( + fileStat.size !== persistedJob.from.size || + Math.floor(fileStat.mtimeMs) !== persistedJob.from.mtime + ) { + persistedJob.from.size = fileStat.size; + persistedJob.from.mtime = Math.floor(fileStat.mtimeMs); + persistedJob.prog.loaded = 0; + persistedJob.prog.total = fileStat.size; + persistedJob.uploadedParts = []; + } + + // resumable + persistedJob.prog.resumable = this.config.resumeUpload && persistedJob.from.size > this.config.multipartUploadThreshold; + + const job = UploadJob.fromPersistInfo( + jobId, + persistedJob, + { + ...clientOptions, + backendMode: persistedJob.backendMode, + }, + { + uploadSpeedLimit: this.config.uploadSpeedLimit, + isDebug: this.config.isDebug, + userNatureLanguage: uploadOptions.userNatureLanguage, + }, + ); + + if ([Status.Waiting, Status.Running].includes(job.status)) { + job.stop(); + } + + this.addJob(job); + }); + } + + public waitJob(jobId: string): void { + this.jobs.get(jobId)?.wait(); + this.scheduleJobs(); + } + + public startJob(jobId: string, forceOverwrite: boolean = false): void { + this.jobs.get(jobId)?.start(forceOverwrite); + } + + public stopJob(jobId: string): void { + this.jobs.get(jobId)?.stop(); + } + + public removeJob(jobId: string): void { + const indexToRemove = this.jobIds.indexOf(jobId); + if (indexToRemove < 0) { + return; + } + this.jobs.get(jobId)?.stop(); + this.jobIds.splice(indexToRemove, 1); + this.jobs.delete(jobId); + } + + public cleanupJobs(): void { + const idsToRemove = this.jobIds.filter(id => this.jobs.get(id)?.status === Status.Finished); + this.jobIds = this.jobIds.filter(id => !idsToRemove.includes(id)); + idsToRemove.forEach(id => { + this.jobs.delete(id); + }); + } + + public startAllJobs(): void { + this.jobIds + .map(id => this.jobs.get(id)) + .forEach(job => { + if (!job) { + return; + } + if ([ + Status.Stopped, + Status.Failed, + ].includes(job.status)) { + job.wait(); + } + }); + this.scheduleJobs(); + } + + public stopAllJobs({ + matchStatus, + }: { + matchStatus: Status[], + } = { + matchStatus: [], + }): void { + this.jobIds + .map(id => this.jobs.get(id)) + .forEach(job => { + if (!job || ![Status.Running, Status.Waiting].includes(job.status)) { + return; + } + if (!matchStatus.includes(job.status)){ + return; + } + job.stop(); + }); + } + + public removeAllJobs(): void { + this.stopAllJobs(); + this.jobIds = []; + this.jobs.clear(); + } + + private scheduleJobs(): void { + if (this.config.isDebug) { + console.log(`[JOB] upload max: ${this.config.maxConcurrency}, cur: ${this.concurrency}, jobs: ${this.jobIds.length}`); + } + + this.concurrency = Math.max(0, this.concurrency); + if (this.concurrency >= this.config.maxConcurrency) { + return; + } + + for (let i = 0; i < this.jobIds.length; i++) { + const job = this.jobs.get(this.jobIds[i]); + if (job?.status !== Status.Waiting) { + continue; + } + this.concurrency += 1; + job.start() + .finally(() => { + this.afterJobDone(job.id); + }); + + this.concurrency = Math.max(0, this.concurrency); + if (this.concurrency >= this.config.maxConcurrency) { + return; + } + } + } + + private afterJobDone(id: UploadJob["id"]): void { + this.concurrency -= 1; + this.scheduleJobs(); + this.config.onJobDone?.(id, this.jobs.get(id)); + } + + private afterCreateDirectory({ + bucket, + key, + }: { + bucket: string, + key: string, + }) { + this.config.onCreatedDirectory?.(bucket, key); + } +} diff --git a/src/main/util/createClient.ts b/src/main/util/createClient.ts new file mode 100644 index 00000000..c10c23a3 --- /dev/null +++ b/src/main/util/createClient.ts @@ -0,0 +1,74 @@ +import {Qiniu} from "kodo-s3-adapter-sdk"; +import {Adapter, RequestInfo, ResponseInfo} from "kodo-s3-adapter-sdk/dist/adapter"; +import {ModeOptions} from "kodo-s3-adapter-sdk/dist/qiniu"; +import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog"; + +import * as AppConfig from "@common/const/app-config"; +import {ClientOptions} from "@common/ipc-actions/upload"; +import {BackendMode} from "@common/const/qiniu"; + +export default function createQiniuClient( + clientOptions: ClientOptions, + options: { + userNatureLanguage: NatureLanguage, + isDebug: boolean, + }, +): Adapter { + const qiniu = new Qiniu( + clientOptions.accessKey, + clientOptions.secretKey, + clientOptions.ucUrl, + `Kodo-Browser/${AppConfig.app.version}/ioutil`, + clientOptions.regions, + ); + const modeOptions: ModeOptions = { + appName: 'kodo-browser/ioutil', + appVersion: AppConfig.app.version, + appNatureLanguage: options.userNatureLanguage, + // disable uplog when use customize cloud + // because there isn't a valid access key of uplog + uplogBufferSize: clientOptions.ucUrl ? -1 : undefined, + }; + if (options.isDebug) { + modeOptions.requestCallback = debugRequest(clientOptions.backendMode); + modeOptions.responseCallback = debugResponse(clientOptions.backendMode); + } + return qiniu.mode( + clientOptions.backendMode, + modeOptions, + ); +} + + +function debugRequest(mode: BackendMode) { + return (request: RequestInfo) => { + let url = undefined, method = undefined, headers = undefined; + if (request) { + url = request.url; + method = request.method; + headers = request.headers; + } + console.info('>>', mode, 'REQ_URL:', url, 'REQ_METHOD:', method, 'REQ_HEADERS:', headers); + }; +} + +function debugResponse(mode: BackendMode) { + return (response: ResponseInfo) => { + let requestUrl = undefined, requestMethod = undefined, requestHeaders = undefined, + responseStatusCode = undefined, responseHeaders = undefined, responseInterval = undefined, responseData = undefined, responseError = undefined; + if (response) { + responseStatusCode = response.statusCode; + responseHeaders = response.headers; + responseInterval = response.interval; + responseData = response.data; + responseError = response.error; + if (response.request) { + requestUrl = response.request.url; + requestMethod = response.request.method; + requestHeaders = response.request.headers; + } + } + console.info('<<', mode, 'REQ_URL:', requestUrl, 'REQ_METHOD:', requestMethod, 'REQ_HEADERS: ', requestHeaders, + 'RESP_STATUS:', responseStatusCode, 'RESP_HEADERS:', responseHeaders, 'RESP_INTERVAL:', responseInterval, 'ms RESP_DATA:', responseData, 'RESP_ERROR:', responseError); + }; +} diff --git a/src/renderer/components/filters/formatter.ts b/src/renderer/components/filters/formatter.ts index 9b74a997..5e0c6964 100755 --- a/src/renderer/components/filters/formatter.ts +++ b/src/renderer/components/filters/formatter.ts @@ -1,10 +1,11 @@ import moment from "moment/moment" -import Duration from "@/const/duration"; +import Duration from "@common/const/duration"; import * as FileItem from "@/models/file-item"; import { leftTime } from "@/components/services/util" import { FileExtensionType, getFileType } from '@/components/services/file.s' +import ByteSize from "@common/const/byte-size"; export const sub = { name: "sub", @@ -95,6 +96,7 @@ export const leftTimeFormat = { export const sizeFormat = { name: "sizeFormat", + // n: Bytes fn: (n: number, isApproximate: boolean = true): string => { if (n == 0 || !n || n < 0) { return "0"; @@ -103,33 +105,33 @@ export const sizeFormat = { const t = []; let left = n; - const gb = Math.floor(n / Math.pow(1024, 3)); + const gb = Math.floor(n / ByteSize.GB); if (gb > 0) { if (isApproximate) { - return Math.round(n * 100 / Math.pow(1024, 3)) / 100 + "GB"; + return Math.round(n * 100 / ByteSize.GB) / 100 + "GB"; } else { t.push(gb + "G"); - left = left % Math.pow(1024, 3); + left = left % ByteSize.GB; } } - const mb = Math.floor(left / Math.pow(1024, 2)); + const mb = Math.floor(left / ByteSize.MB); if (mb > 0) { if (isApproximate) { - return Math.round(100 * left / Math.pow(1024, 2)) / 100 + "MB"; + return Math.round(100 * left / ByteSize.MB) / 100 + "MB"; } else { t.push(mb + "M"); - left = left % Math.pow(1024, 2); + left = left % ByteSize.MB; } } - const kb = Math.floor(left / 1024); + const kb = Math.floor(left / ByteSize.KB); if (kb > 0) { if (isApproximate) { - return Math.round(100 * left / 1024) / 100 + "KB"; + return Math.round(100 * left / ByteSize.KB) / 100 + "KB"; } else { t.push(kb + "K"); - left = left % 1024; + left = left % ByteSize.KB; } } diff --git a/src/renderer/components/services/ak-history.test.ts b/src/renderer/components/services/ak-history.test.ts index 6db2485b..ac3266bb 100644 --- a/src/renderer/components/services/ak-history.test.ts +++ b/src/renderer/components/services/ak-history.test.ts @@ -1,7 +1,7 @@ import mockFs from "mock-fs"; import fs from 'fs' -import { config_path } from "@/const/app-config"; +import { config_path } from "@common/const/app-config"; import * as AkHistory from "./ak-history"; diff --git a/src/renderer/components/services/ak-history.ts b/src/renderer/components/services/ak-history.ts index 4948c769..624142d4 100644 --- a/src/renderer/components/services/ak-history.ts +++ b/src/renderer/components/services/ak-history.ts @@ -1,7 +1,7 @@ import fs from "fs" import path from "path" -import * as AppConfig from "@/const/app-config"; +import * as AppConfig from "@common/const/app-config"; class AkHistory { isPublicCloud: boolean diff --git a/src/renderer/components/services/audit-log.ts b/src/renderer/components/services/audit-log.ts index 505c7406..f358e78e 100644 --- a/src/renderer/components/services/audit-log.ts +++ b/src/renderer/components/services/audit-log.ts @@ -4,7 +4,7 @@ import path from 'path' import moment from 'moment' import * as AuthInfo from './authinfo' -import * as AppConfig from '@/const/app-config' +import * as AppConfig from '@common/const/app-config' const expirationMonths = 3; diff --git a/src/renderer/components/services/auto-upgrade.js b/src/renderer/components/services/auto-upgrade.js index 18a45fcf..f0efbacf 100755 --- a/src/renderer/components/services/auto-upgrade.js +++ b/src/renderer/components/services/auto-upgrade.js @@ -4,7 +4,7 @@ import path from 'path' import request from 'request' import downloadsFolder from 'downloads-folder' -import * as util from '@/models/job/utils' +import * as util from '@common/models/job/utils' import webModule from '@/app-module/web' import { upgrade } from '@/customize' diff --git a/src/renderer/components/services/bookmark.test.ts b/src/renderer/components/services/bookmark.test.ts index 8e3c61b3..8b11f10b 100644 --- a/src/renderer/components/services/bookmark.test.ts +++ b/src/renderer/components/services/bookmark.test.ts @@ -3,7 +3,7 @@ import { mocked } from "ts-jest/utils"; import mockFs from "mock-fs"; import fs from 'fs' -import { config_path } from "@/const/app-config"; +import { config_path } from "@common/const/app-config"; import * as KodoNav from "@/const/kodo-nav"; import * as AuthInfo from "./authinfo"; diff --git a/src/renderer/components/services/bookmark.ts b/src/renderer/components/services/bookmark.ts index 232cd717..5210eeff 100755 --- a/src/renderer/components/services/bookmark.ts +++ b/src/renderer/components/services/bookmark.ts @@ -3,7 +3,7 @@ import path from 'path' import moment from 'moment' -import * as AppConfig from '@/const/app-config' +import * as AppConfig from '@common/const/app-config' import * as KodoNav from '@/const/kodo-nav' import * as AuthInfo from './authinfo' diff --git a/src/renderer/components/services/download-manager.js b/src/renderer/components/services/download-manager.js index d4e28c25..c2e30519 100755 --- a/src/renderer/components/services/download-manager.js +++ b/src/renderer/components/services/download-manager.js @@ -4,7 +4,7 @@ import path from 'path' import sanitize from 'sanitize-filename' import { KODO_MODE } from 'kodo-s3-adapter-sdk' -import { DownloadJob } from '@/models/job' +import { DownloadJob } from '@common/models/job' import webModule from '@/app-module/web' import NgConfig from '@/ng-config' diff --git a/src/renderer/components/services/index.js b/src/renderer/components/services/index.js index 3d0784a8..26a54018 100644 --- a/src/renderer/components/services/index.js +++ b/src/renderer/components/services/index.js @@ -15,5 +15,4 @@ import './job-util' import './ng-qiniu-client' import './safe-apply' import './settings.ts' -import './upload-manager' import './util' diff --git a/src/renderer/components/services/ipc-upload-manager.ts b/src/renderer/components/services/ipc-upload-manager.ts new file mode 100644 index 00000000..90a96172 --- /dev/null +++ b/src/renderer/components/services/ipc-upload-manager.ts @@ -0,0 +1,6 @@ +import {UploadActionFns} from "@common/ipc-actions/upload"; +import {ipcRenderer} from "electron"; + +const ipcUploadManager = new UploadActionFns(ipcRenderer, "UploaderManager"); + +export default ipcUploadManager; diff --git a/src/renderer/components/services/job-util.js b/src/renderer/components/services/job-util.js index d17e84d5..4cd94a87 100755 --- a/src/renderer/components/services/job-util.js +++ b/src/renderer/components/services/job-util.js @@ -3,12 +3,9 @@ import webModule from '@/app-module/web' const JOB_UTIL_FACTORY_NAME = 'jobUtil' webModule.factory(JOB_UTIL_FACTORY_NAME, [ - "$q", - "$state", - "$timeout", "$translate", - function($q, $state, $timeout, $translate) { - var T = $translate.instant; + function($translate) { + const T = $translate.instant; return { getStatusLabel: getStatusLabel, @@ -26,6 +23,7 @@ webModule.factory(JOB_UTIL_FACTORY_NAME, [ return "danger"; case "finished": return "success"; + case "duplicated": case "stopped": return "warning"; default: @@ -40,6 +38,7 @@ webModule.factory(JOB_UTIL_FACTORY_NAME, [ return isUp ? T("status.running.uploading") : T("status.running.downloading"); //'正在上传':'正在下载'; + case "duplicated": case "failed": return T("status.failed"); //'失败'; case "finished": diff --git a/src/renderer/components/services/qiniu-client/_mock-helpers_/adapter.ts b/src/renderer/components/services/qiniu-client/_mock-helpers_/adapter.ts index 9d547723..76162700 100644 --- a/src/renderer/components/services/qiniu-client/_mock-helpers_/adapter.ts +++ b/src/renderer/components/services/qiniu-client/_mock-helpers_/adapter.ts @@ -1,6 +1,6 @@ import { ListedObjects, ObjectInfo } from "kodo-s3-adapter-sdk/dist/adapter"; import * as qiniuPathConvertor from "qiniu-path/dist/src/convert"; -import Duration from "@/const/duration"; +import Duration from "@common/const/duration"; import * as FileItem from "@/models/file-item"; export function mockAdapterFactory(adapterName: string) { diff --git a/src/renderer/components/services/qiniu-client/_mock-helpers_/config-file.ts b/src/renderer/components/services/qiniu-client/_mock-helpers_/config-file.ts index fcfe14c2..9d0d9c69 100644 --- a/src/renderer/components/services/qiniu-client/_mock-helpers_/config-file.ts +++ b/src/renderer/components/services/qiniu-client/_mock-helpers_/config-file.ts @@ -1,6 +1,6 @@ import mockFs from "mock-fs"; -import { config_path } from "@/const/app-config"; +import { config_path } from "@common/const/app-config"; const CONFIG_MOCK_CONTENT = `{ "uc_url": "https://mocked-uc.qiniu.io", diff --git a/src/renderer/components/services/qiniu-client/common.ts b/src/renderer/components/services/qiniu-client/common.ts index 3d665fd1..3da49212 100644 --- a/src/renderer/components/services/qiniu-client/common.ts +++ b/src/renderer/components/services/qiniu-client/common.ts @@ -5,7 +5,7 @@ import { S3 as S3Adapter } from "kodo-s3-adapter-sdk/dist/s3" import { RegionService } from "kodo-s3-adapter-sdk/dist/region_service"; -import * as AppConfig from "@/const/app-config"; +import * as AppConfig from "@common/const/app-config"; import * as AuthInfo from "@/components/services/authinfo"; import * as Config from "@/config"; import Settings from '@/components/services/settings' diff --git a/src/renderer/components/services/qiniu-client/files.ts b/src/renderer/components/services/qiniu-client/files.ts index 908d8dbc..9f07c149 100644 --- a/src/renderer/components/services/qiniu-client/files.ts +++ b/src/renderer/components/services/qiniu-client/files.ts @@ -1,7 +1,7 @@ import * as qiniuPathConvertor from "qiniu-path/dist/src/convert"; import { Path as QiniuPath } from "qiniu-path/dist/src/path"; import { Adapter, Domain, FrozenInfo, ObjectInfo, PartialObjectError, StorageClass, TransferObject } from 'kodo-s3-adapter-sdk/dist/adapter' -import Duration from "@/const/duration"; +import Duration from "@common/const/duration"; import * as FileItem from "@/models/file-item"; import { GetAdapterOptionParam, getDefaultClient } from "./common" diff --git a/src/renderer/components/services/qiniu-client/storage-class.ts b/src/renderer/components/services/qiniu-client/storage-class.ts index 321d71c5..60735187 100644 --- a/src/renderer/components/services/qiniu-client/storage-class.ts +++ b/src/renderer/components/services/qiniu-client/storage-class.ts @@ -1,16 +1,9 @@ import { S3_MODE } from "kodo-s3-adapter-sdk"; +import StorageClass from "@common/models/storage-class"; import { getRegionsStorageClasses } from './regions'; import { GetAdapterOptionParam } from "./common"; -interface StorageClass { - fileType: number - kodoName: string - s3Name: string - billingI18n: Record - nameI18n: Record -} - const regionsStorageClasses = new Map(); function hyphenLangFields(i18n: Record) { diff --git a/src/renderer/components/services/qiniu-client/utils.ts b/src/renderer/components/services/qiniu-client/utils.ts index e29c0a37..08be6ab9 100644 --- a/src/renderer/components/services/qiniu-client/utils.ts +++ b/src/renderer/components/services/qiniu-client/utils.ts @@ -2,7 +2,7 @@ import { KODO_MODE, Region } from "kodo-s3-adapter-sdk"; import { Domain } from "kodo-s3-adapter-sdk/dist/adapter"; import { Path as QiniuPath } from "qiniu-path/dist/src/path"; -import * as AppConfig from "@/const/app-config"; +import * as AppConfig from "@common/const/app-config"; import * as KodoNav from "@/const/kodo-nav"; import {debugRequest, debugResponse, GetAdapterOptionParam, getDefaultClient} from './common' diff --git a/src/renderer/components/services/settings.test.ts b/src/renderer/components/services/settings.test.ts index 46f8f873..1a49d97e 100644 --- a/src/renderer/components/services/settings.test.ts +++ b/src/renderer/components/services/settings.test.ts @@ -1,4 +1,16 @@ +jest.mock("electron", () => ({ + __esModule: true, + ipcRenderer: { + on: jest.fn(), + send: jest.fn(), + removeListener: jest.fn(), + } +})); + +import { ipcRenderer } from "electron"; import Settings, { SettingKey } from "./settings"; +import {UploadAction} from "@common/ipc-actions/upload"; +import ByteSize from "@common/const/byte-size"; // WARNING: The getter tests in "no data in storage" section is // for testing default value. @@ -93,8 +105,26 @@ describe("test settings.ts", () => { it("isDebug setter", () => { Settings.isDebug = 1; expect(Settings.isDebug).toBe(1); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + isDebug: true, + }, + }, + ); Settings.isDebug = 0; expect(Settings.isDebug).toBe(0); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + isDebug: false, + }, + }, + ); }); // autoUpgrade @@ -115,8 +145,26 @@ describe("test settings.ts", () => { it("resumeUpload setter", () => { Settings.resumeUpload = 1; expect(Settings.resumeUpload).toBe(1); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + resumeUpload: true, + }, + }, + ); Settings.resumeUpload = 0; expect(Settings.resumeUpload).toBe(0); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + resumeUpload: false, + }, + }, + ); }); // maxUploadConcurrency @@ -126,8 +174,26 @@ describe("test settings.ts", () => { it("maxUploadConcurrency setter", () => { Settings.maxUploadConcurrency = 2; expect(Settings.maxUploadConcurrency).toBe(2); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + maxConcurrency: 2, + }, + }, + ); Settings.maxUploadConcurrency = 1; expect(Settings.maxUploadConcurrency).toBe(1); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + maxConcurrency: 1, + }, + }, + ); }); // multipartUploadSize @@ -135,16 +201,34 @@ describe("test settings.ts", () => { expect(Settings.multipartUploadSize).toBe(8); }); it("multipartUploadSize setter valid", () => { - Settings.multipartUploadSize = 4; - expect(Settings.multipartUploadSize).toBe(4); + Settings.multipartUploadSize = 10; + expect(Settings.multipartUploadSize).toBe(10); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + multipartUploadSize: 10 * ByteSize.MB, + }, + }, + ); Settings.multipartUploadSize = 8; expect(Settings.multipartUploadSize).toBe(8); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + multipartUploadSize: 8 * ByteSize.MB, + }, + }, + ); }); it("multipartUploadSize setter invalid", () => { - for (let i = 1; i < 4; i ++) { - Settings.multipartUploadSize = i; - expect(Settings.multipartUploadSize).toBe(8); - } + Settings.multipartUploadSize = 0; + expect(Settings.multipartUploadSize).toBe(8); + Settings.multipartUploadSize = 1025; + expect(Settings.multipartUploadSize).toBe(8); }); // multipartUploadThreshold @@ -154,8 +238,26 @@ describe("test settings.ts", () => { it("multipartUploadThreshold setter", () => { Settings.multipartUploadThreshold = 110; expect(Settings.multipartUploadThreshold).toBe(110); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + multipartUploadThreshold: 110 * ByteSize.MB, + }, + }, + ); Settings.multipartUploadThreshold = 100; expect(Settings.multipartUploadThreshold).toBe(100); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + multipartUploadThreshold: 100 * ByteSize.MB, + }, + }, + ); }); // uploadSpeedLimitEnabled @@ -165,8 +267,26 @@ describe("test settings.ts", () => { it("uploadSpeedLimitEnabled setter", () => { Settings.uploadSpeedLimitEnabled = 1; expect(Settings.uploadSpeedLimitEnabled).toBe(1); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + uploadSpeedLimit: Settings.uploadSpeedLimitKBperSec * ByteSize.KB, + }, + }, + ); Settings.uploadSpeedLimitEnabled = 0; expect(Settings.uploadSpeedLimitEnabled).toBe(0); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + uploadSpeedLimit: 0, + }, + }, + ); }); // uploadSpeedLimitKBperSec @@ -174,10 +294,29 @@ describe("test settings.ts", () => { expect(Settings.uploadSpeedLimitKBperSec).toBe(1024); }); it("uploadSpeedLimitKBperSec setter", () => { + Settings.uploadSpeedLimitEnabled = 1; Settings.uploadSpeedLimitKBperSec = 2048; expect(Settings.uploadSpeedLimitKBperSec).toBe(2048); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + uploadSpeedLimit: 2048 * ByteSize.KB, + }, + }, + ); Settings.uploadSpeedLimitKBperSec = 1024; expect(Settings.uploadSpeedLimitKBperSec).toBe(1024); + expect(ipcRenderer.send).toHaveBeenLastCalledWith( + "UploaderManager", + { + action: UploadAction.UpdateConfig, + data: { + uploadSpeedLimit: 1024 * ByteSize.KB, + }, + }, + ); }); // resumeDownload @@ -373,14 +512,14 @@ describe("test settings.ts", () => { it("multipartUploadSize setter valid", () => { Settings.multipartUploadSize = 8; expect(Settings.multipartUploadSize).toBe(8); - Settings.multipartUploadSize = 4; - expect(Settings.multipartUploadSize).toBe(4); + Settings.multipartUploadSize = 10; + expect(Settings.multipartUploadSize).toBe(10); }); it("multipartUploadSize setter invalid", () => { - for (let i = 1; i < 4; i ++) { - Settings.multipartUploadSize = i; - expect(Settings.multipartUploadSize).toBe(4); - } + Settings.multipartUploadSize = 0; + expect(Settings.multipartUploadSize).toBe(4); + Settings.multipartUploadSize = 1025; + expect(Settings.multipartUploadSize).toBe(4); }); // multipartUploadThreshold diff --git a/src/renderer/components/services/settings.ts b/src/renderer/components/services/settings.ts index e0030789..2a16b024 100755 --- a/src/renderer/components/services/settings.ts +++ b/src/renderer/components/services/settings.ts @@ -1,3 +1,6 @@ +import ByteSize from "@common/const/byte-size"; +import ipcUploadManager from "@/components/services/ipc-upload-manager"; + export enum SettingKey { IsDebug = "isDebug", AutoUpgrade = "autoUpgrade", @@ -31,6 +34,9 @@ class Settings { } set isDebug(v: number) { localStorage.setItem(SettingKey.IsDebug, v.toString()); + ipcUploadManager.updateConfig({ + isDebug: v !== 0, + }); } // autoUpgrade @@ -47,6 +53,9 @@ class Settings { } set resumeUpload(v: number) { localStorage.setItem(SettingKey.ResumeUpload, v.toString()); + ipcUploadManager.updateConfig({ + resumeUpload: v !== 0, + }); } // maxUploadConcurrency @@ -55,6 +64,9 @@ class Settings { } set maxUploadConcurrency(v: number) { localStorage.setItem(SettingKey.MaxUploadConcurrency, v.toString()); + ipcUploadManager.updateConfig({ + maxConcurrency: v, + }); } // multipartUploadSize @@ -62,8 +74,11 @@ class Settings { return parseInt(localStorage.getItem(SettingKey.MultipartUploadSize) || "8"); } set multipartUploadSize(v: number) { - if (v % 4 == 0) { + if (v >= 1 && v <= 1024) { localStorage.setItem(SettingKey.MultipartUploadSize, v.toString()); + ipcUploadManager.updateConfig({ + multipartUploadSize: v * ByteSize.MB, + }); } } @@ -73,6 +88,9 @@ class Settings { } set multipartUploadThreshold(v: number) { localStorage.setItem(SettingKey.MultipartUploadThreshold, v.toString()); + ipcUploadManager.updateConfig({ + multipartUploadThreshold: v * ByteSize.MB, + }); } // uploadSpeedLimitEnabled @@ -81,6 +99,15 @@ class Settings { } set uploadSpeedLimitEnabled(v: number) { localStorage.setItem(SettingKey.UploadSpeedLimitEnabled, v.toString()); + if (v === 0) { + ipcUploadManager.updateConfig({ + uploadSpeedLimit: 0, + }); + } else { + ipcUploadManager.updateConfig({ + uploadSpeedLimit: this.uploadSpeedLimitKBperSec * ByteSize.KB, + }); + } } // uploadSpeedLimitKBperSec @@ -89,6 +116,11 @@ class Settings { } set uploadSpeedLimitKBperSec (v) { localStorage.setItem(SettingKey.UploadSpeedLimit, v.toString()); + if (this.uploadSpeedLimitEnabled) { + ipcUploadManager.updateConfig({ + uploadSpeedLimit: v * ByteSize.KB, + }); + } } // downloadSpeedLimitKBperSec diff --git a/src/renderer/components/services/upload-manager.js b/src/renderer/components/services/upload-manager.js deleted file mode 100755 index 6df5812a..00000000 --- a/src/renderer/components/services/upload-manager.js +++ /dev/null @@ -1,434 +0,0 @@ -import fs from 'fs' -import path from 'path' - -import qiniuPath from "qiniu-path" - -import { UploadJob } from '@/models/job' -import webModule from '@/app-module/web' - -import NgQiniuClient from './ng-qiniu-client' -import * as AuthInfo from './authinfo' -import NgConfig from '@/ng-config' -import Settings from './settings.ts' - -const UPLOAD_MGR_FACTORY_NAME = 'UploadMgr' - -webModule.factory(UPLOAD_MGR_FACTORY_NAME, [ - "$timeout", - "$translate", - NgQiniuClient, - NgConfig, - function ( - $timeout, - $translate, - QiniuClient, - Config, - ) { - const T = $translate.instant; - - var $scope; - var concurrency = 0; - var stopCreatingFlag = false; - - return { - init: init, - createUploadJobs: createUploadJobs, - trySchedJob: trySchedJob, - trySaveProg: trySaveProg, - - stopCreatingJobs: function () { - stopCreatingFlag = true; - } - }; - - function init(scope) { - $scope = scope; - $scope.lists.uploadJobList = []; - - angular.forEach(tryLoadProg(), (prog) => { - const job = createJob(prog); - if (job.status === "waiting" || job.status === "running") { - job.stop(); - } - addEvents(job); - }); - } - - /** - * @param options { object: { region, from, to, progress, checkPoints } } - * @param options.from {name, path} - * @param options.to {bucket, key} - * @return job { start(), stop(), status, progress } - job.events: statuschange, progress - */ - function createJob(options) { - - console.info( - "PUT", - "::", - options.region, - "::", - options.from.path + "/" + options.from.name, - "=>", - options.to.bucket + "/" + options.to.key - ); - - const config = Config.load(); - - options.clientOptions = { - accessKey: AuthInfo.get().id, - secretKey: AuthInfo.get().secret, - ucUrl: config.ucUrl || "", - regions: config.regions || [], - }; - options.resumeUpload = (Settings.resumeUpload === 1); - options.multipartUploadSize = Settings.multipartUploadSize; - options.multipartUploadThreshold = Settings.multipartUploadThreshold; - options.uploadSpeedLimit = (Settings.uploadSpeedLimitEnabled === 1 && Settings.uploadSpeedLimitKBperSec); - options.isDebug = (Settings.isDebug === 1); - options.storageClasses = options.storageClasses || []; - options.userNatureLanguage = localStorage.getItem('lang') || 'zh-CN'; - - return new UploadJob(options); - } - - /** - * upload - * @param filePaths [] {array} 有可能是目录,需要遍历 - * @param bucketInfo {object: {bucketName, regionId, key, qiniuBackendMode}} - * @param uploadOptions {object: {isOverwrite, storageClassName}}, storageClassName is fetched from server - * @param jobsAddingFn {Function} 快速加入列表回调方法, 返回jobs引用,但是该列表长度还在增长。 - * @param jobsAddedFn {Function} 加入列表完成回调方法, jobs列表已经稳定 - */ - function createUploadJobs(filePaths, bucketInfo, uploadOptions, jobsAddingFn) { - stopCreatingFlag = false; - - _kdig(filePaths, () => { - if (jobsAddingFn) { - jobsAddingFn(); - } - }); - return; - - function _kdig(filePaths, fn) { - let t = []; - let c = 0; - const len = filePaths.length; - - function _dig() { - if (stopCreatingFlag) { - return; - } - - const n = filePaths[c]; - const dirPath = n.parentDirectoryPath(); - - dig(filePaths[c].toString(), dirPath, (jobs) => { - t = t.concat(jobs); - c++; - - if (c >= len) { - fn(t); - } else { - _dig(); - } - }); - } - _dig(); - } - - function loop(parentPath, dirPath, arr, callFn) { - var t = []; - var len = arr.length; - var c = 0; - if (len == 0) { - callFn([]); - } else { - inDig(); - } - - //串行 - function inDig() { - dig(path.join(parentPath.toString(), arr[c].toString()), dirPath, (jobs) => { - t = t.concat(jobs); - - c++; - if (c >= len) { - callFn(t); - } else { - if (stopCreatingFlag) { - return; - } - - inDig(); - } - }); - } - } - - function dig(absPath, dirPath, callFn) { - if (stopCreatingFlag) { - return; - } - - const fileName = path.basename(absPath); - let filePath = path.relative(dirPath.toString(), absPath); - - if (bucketInfo.key) { - if (bucketInfo.key.endsWith('/')) { - filePath = bucketInfo.key + filePath; - } else { - filePath = bucketInfo.key + '/' + filePath; - } - } - const fileStat = fs.statSync(absPath); - - if (fileStat.isDirectory()) { - //创建目录 - let subDirPath = filePath + '/'; - if (path.sep == '\\') { - subDirPath = subDirPath.replace(/\\/g, '/'); - } - subDirPath = qiniuPath.fromQiniuPath(subDirPath); - - //递归遍历目录 - fs.readdir(absPath, (err, arr) => { - if (err) { - console.error(err.stack); - } else if (arr.length > 0 || arr.length === 0 && $scope.emptyFolderUploading.enabled) { - QiniuClient - .createFolder(bucketInfo.regionId, bucketInfo.bucketName, subDirPath) - .then(() => { - checkNeedRefreshFileList(bucketInfo.bucketName, subDirPath); - loop(absPath, dirPath, arr, (jobs) => { $timeout(() => { callFn(jobs); }, 1); }); - }); - } else { - $timeout(() => { callFn([]); }, 1); - } - }); - } else { - //文件 - - //修复 window 下 \ 问题 - if (path.sep == '\\') { - filePath = filePath.replace(/\\/g, '/'); - } - - const job = createJob({ - region: bucketInfo.regionId, - from: { - name: fileName, - path: absPath - }, - to: { - bucket: bucketInfo.bucketName, - key: filePath - }, - overwrite: uploadOptions.isOverwrite, - storageClassName: uploadOptions.storageClassName, - storageClasses: $scope.currentInfo.availableStorageClasses, - backendMode: bucketInfo.qiniuBackendMode, - }); - addEvents(job); - $timeout(() => { callFn([job]); }, 1); - } - } - } - - function addEvents(job) { - if (!job.uploadedParts) { - job.uploadedParts = []; - } - - $scope.lists.uploadJobList.push(job); - - trySchedJob(); - trySaveProg(); - - $timeout(() => { - $scope.calcTotalProg(); - }); - - job.on('fileDuplicated', (data) => { - concurrency--; - $timeout(() => { - trySchedJob(); - $scope.calcTotalProg(); - }); - }); - job.on("partcomplete", (data) => { - job.uploadedId = data.uploadId; - job.uploadedParts[data.part.partNumber] = data.part; - - trySaveProg(); - - $timeout($scope.calcTotalProg); - }); - job.on("statuschange", (status) => { - if (status == "stopped") { - concurrency--; - $timeout(trySchedJob); - } - - trySaveProg(); - $timeout($scope.calcTotalProg); - }); - job.on("speedchange", () => { - $timeout($scope.calcTotalProg); - }); - job.on("complete", () => { - concurrency--; - - $timeout(() => { - trySchedJob(); - $scope.calcTotalProg(); - checkNeedRefreshFileList(job.options.to.bucket, qiniuPath.fromLocalPath(job.options.to.key)); - }); - }); - job.on("error", (err) => { - if (err) { - console.error(`upload kodo://${job.options.to.bucket}/${job.options.to.key} error: ${err}`); - } - if (job.message) { - switch (job.message.error) { - case 'Forbidden': - job.message.i18n = T('permission.denied'); - } - } - - concurrency--; - $timeout(() => { - trySchedJob(); - $scope.calcTotalProg(); - }); - }); - } - - function trySchedJob() { - var maxConcurrency = Settings.maxUploadConcurrency; - var isDebug = (Settings.isDebug === 1); - - concurrency = Math.max(0, concurrency); - if (isDebug) { - console.log(`[JOB] upload max: ${maxConcurrency}, cur: ${concurrency}, jobs: ${$scope.lists.uploadJobList.length}`); - } - - if (concurrency < maxConcurrency) { - var jobs = $scope.lists.uploadJobList; - - for (var i = 0; i < jobs.length && concurrency < maxConcurrency; i++) { - var job = jobs[i]; - if (isDebug) { - console.log(`[JOB] sched ${job.status} => ${JSON.stringify(job._config)}`); - } - - if (job.status === 'waiting') { - concurrency++; - - if (job.prog.resumable) { - var progs = tryLoadProg(); - - if (progs && progs[job.id]) { - job.start(true, progs[job.id]); - } else { - job.start(true); - } - } else { - job.start(); - } - } - } - } - } - - function trySaveProg() { - var t = {}; - angular.forEach($scope.lists.uploadJobList, (job) => { - if (job.status === 'finished') return; - - if (!job.uploadedParts) { - job.uploadedParts = []; - } - - if (fs.existsSync(job.options.from.path)) { - const fileStat = fs.statSync(job.options.from.path); - - t[job.id] = job.getInfoForSave({ - from: { - size: fileStat.size, - mtime: fileStat.mtimeMs, - } - }); - } - }); - - fs.writeFileSync(getProgFilePath(), JSON.stringify(t)); - } - - function tryLoadProg() { - let result = {} - let progs = {}; - try { - const data = fs.readFileSync(getProgFilePath()); - progs = JSON.parse(data); - } catch (e) {} - - Object.entries(progs) - .forEach(([jobId, briefJob]) => { - if (!briefJob.from || !briefJob.from.size || !briefJob.from.mtime) { - delete briefJob.prog.loaded; - result[jobId] = { - ...briefJob, - uploadedParts: [], - }; - return; - } - if (!fs.existsSync(briefJob.from.path)) { - delete briefJob.prog.loaded; - result[jobId] = { - ...briefJob, - uploadedParts: [], - }; - return; - } - const fileStat = fs.statSync(briefJob.from.path); - if (fileStat.size !== briefJob.from.size || briefJob.from.mtime !== fileStat.mtimeMs) { - delete briefJob.prog.loaded; - result[jobId] = { - ...briefJob, - uploadedParts: [], - }; - return; - } - result[jobId] = { - ...briefJob, - uploadedParts: (briefJob.uploadedParts || []) - .filter(part => part && part.PartNumber && part.ETag) - .map(part => ({partNumber: part.PartNumber, etag: part.ETag})) - }; - }); - - return result; - } - - function getProgFilePath() { - var folder = Global.config_path; - if (!fs.existsSync(folder)) { - fs.mkdirSync(folder); - } - - const username = AuthInfo.get().id || "kodo-browser"; - return path.join(folder, "upprog_" + username + ".json"); - } - - function checkNeedRefreshFileList(bucket, key) { - if ($scope.currentInfo.bucketName === bucket) { - if ($scope.currentInfo.key === key.parentDirectoryPath().toString()) { - $scope.$emit('refreshFilesList'); - } - } - } - } -]); - -export default UPLOAD_MGR_FACTORY_NAME diff --git a/src/renderer/components/services/util.ts b/src/renderer/components/services/util.ts index d6863579..cb9d2545 100644 --- a/src/renderer/components/services/util.ts +++ b/src/renderer/components/services/util.ts @@ -1,4 +1,4 @@ -import Duration from "@/const/duration"; +import Duration from "@common/const/duration"; export function leftTime(ms: number): string { if (Number.isNaN(ms)) { diff --git a/src/renderer/config.ts b/src/renderer/config.ts index 7b3f31b5..aecb8abf 100644 --- a/src/renderer/config.ts +++ b/src/renderer/config.ts @@ -3,7 +3,7 @@ import path from 'path' import { Region } from "kodo-s3-adapter-sdk"; -import * as AppConfig from "@/const/app-config"; +import * as AppConfig from "@common/const/app-config"; import * as AuthInfo from '@/components/services/authinfo' interface ConfigInner { diff --git a/src/renderer/main/files/files.js b/src/renderer/main/files/files.js index 6d1f3b81..abd49015 100755 --- a/src/renderer/main/files/files.js +++ b/src/renderer/main/files/files.js @@ -1324,7 +1324,7 @@ webModule.controller(FILES_CONTROLLER_NAME, [ okCallback: () => { return ({ files, uploadOptions }) => { $scope.handlers.uploadFilesHandler( - files.map(qiniuPath.fromLocalPath), + files, angular.copy($scope.currentInfo), uploadOptions, ); diff --git a/src/renderer/main/files/modals/preview/media-modal.js b/src/renderer/main/files/modals/preview/media-modal.js index 6a3852ba..4c750fa8 100755 --- a/src/renderer/main/files/modals/preview/media-modal.js +++ b/src/renderer/main/files/modals/preview/media-modal.js @@ -1,6 +1,7 @@ import angular from 'angular' import webModule from '@/app-module/web' +import Duration from "@common/const/duration"; const MEDIA_MODAL_CONTROLLER_NAME = 'mediaModalCtrl' @@ -45,7 +46,7 @@ webModule } function genURL() { - selectedDomain.domain.signatureUrl(objectInfo.path, qiniuClientOpt).then((url) => { + selectedDomain.domain.signatureUrl(objectInfo.path, 12 * Duration.Hour / Duration.Second, qiniuClientOpt).then((url) => { $scope.src_origin = url.toString(); $scope.src = $sce.trustAsResourceUrl(url.toString()); diff --git a/src/renderer/main/files/modals/preview/picture-modal.js b/src/renderer/main/files/modals/preview/picture-modal.js index a0c368e7..f3caf95c 100755 --- a/src/renderer/main/files/modals/preview/picture-modal.js +++ b/src/renderer/main/files/modals/preview/picture-modal.js @@ -1,5 +1,7 @@ import angular from 'angular' +import Duration from "@common/const/duration"; + import webModule from '@/app-module/web' import { SIZE_FORMAT_FILTER_NAME } from "@/components/filters/formater"; @@ -52,7 +54,7 @@ webModule } function getContent() { - selectedDomain.domain.signatureUrl(objectInfo.path, qiniuClientOpt).then((url) => { + selectedDomain.domain.signatureUrl(objectInfo.path, 12 * Duration.Hour / Duration.Second, qiniuClientOpt).then((url) => { $timeout(() => { $scope.imgsrc = url.toString(); }); diff --git a/src/renderer/main/files/transfer/frame.html b/src/renderer/main/files/transfer/frame.html index 33e4ae1f..39fa2e7e 100755 --- a/src/renderer/main/files/transfer/frame.html +++ b/src/renderer/main/files/transfer/frame.html @@ -1,9 +1,9 @@
- {{totalStat.upDone}}/{{lists.uploadJobList.length}} + {{totalStat.upDone}}/{{totalStat.up}} -    + {{totalStat.downDone}}/{{lists.downloadJobList.length}} @@ -15,7 +15,7 @@
  • {{'upload'|translate}} - +
  • diff --git a/src/renderer/main/files/transfer/frame.js b/src/renderer/main/files/transfer/frame.js index bdfa6954..74af0ee0 100755 --- a/src/renderer/main/files/transfer/frame.js +++ b/src/renderer/main/files/transfer/frame.js @@ -1,8 +1,19 @@ +import fs from "fs"; +import path from "path"; +import { ipcRenderer } from "electron"; import angular from "angular" +import ByteSize from "@common/const/byte-size"; +import { UploadAction } from "@common/ipc-actions/upload"; + import webModule from '@/app-module/web' -import UploadMgr from '@/components/services/upload-manager' +import * as AuthInfo from '@/components/services/authinfo'; +import Settings from '@/components/services/settings'; +import safeApply from '@/components/services/safe-apply'; +import ipcUploadManager from '@/components/services/ipc-upload-manager'; + +import NgConfig from '@/ng-config' import DownloadMgr from '@/components/services/download-manager' import { TOAST_FACTORY_NAME as Toast } from '@/components/directives/toast-list' import { @@ -16,41 +27,54 @@ import './downloads' import './uploads' import './frame.css' +import {Status} from "@common/models/job/types"; const TRANSFER_FRAME_CONTROLLER_NAME = 'transferFrameCtrl' webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ "$scope", "$translate", - UploadMgr, + safeApply, + NgConfig, DownloadMgr, Toast, function ( $scope, $translate, - UploadMgr, + safeApply, + ngConfig, DownloadMgr, Toast, ) { const T = $translate.instant; + let uploaderTimer; angular.extend($scope, { transTab: 1, + transSearch: { + uploadJob: "", + }, + lists: { uploadJobList: [], - downloadJobList: [] + uploadJobListLimit: 100, + downloadJobList: [], }, emptyFolderUploading: { - enabled: localStorage.getItem(EMPTY_FOLDER_UPLOADING) || true, + enabled: localStorage.getItem(EMPTY_FOLDER_UPLOADING) !== null + ? localStorage.getItem(EMPTY_FOLDER_UPLOADING) === "true" + : true, }, overwriteDownloading: { - enabled: localStorage.getItem(OVERWRITE_DOWNLOADING) || false, + enabled: localStorage.getItem(OVERWRITE_DOWNLOADING) === "true" || false, }, totalStat: { running: 0, total: 0, + up: 0, + upRunning: 0, upDone: 0, upStopped: 0, upFailed: 0, @@ -66,9 +90,125 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ $scope.handlers.uploadFilesHandler = uploadFilesHandler; $scope.handlers.downloadFilesHandler = downloadFilesHandler; - UploadMgr.init($scope); + initUploaderIpc(); DownloadMgr.init($scope); + $scope.$on('$destroy', () => { + clearInterval(uploaderTimer); + }); + + // init Uploader IPC + function initUploaderIpc() { + ipcRenderer.on("UploaderManager-reply", (_event, message) => { + safeApply($scope, () => { + switch (message.action) { + case UploadAction.UpdateUiData: { + $scope.lists.uploadJobList = message.data.list; + $scope.totalStat.up = message.data.total; + $scope.totalStat.upDone = message.data.finished; + $scope.totalStat.upFailed = message.data.failed; + $scope.totalStat.upStopped = message.data.stopped; + break; + } + case UploadAction.AddedJobs: { + Toast.info(T("upload.addtolist.success")); + $scope.transTab = 1; + $scope.toggleTransVisible(true); + AuditLog.log( + AuditLog.Action.UploadFilesStart, + { + regionId: message.data.destInfo.regionId, + bucket: message.data.destInfo.bucketName, + to: message.data.destInfo.key, + from: message.data.filePathnameList, + }, + ); + break; + } + case UploadAction.JobCompleted: { + const parentDirectoryKey = message.data.jobUiData.to.key + .split("/") + .slice(0, -1) + .reduce((r, p) => r + p + "/", ""); + if ( + $scope.currentInfo.bucketName === message.data.jobUiData.to.bucket && + $scope.currentInfo.key === parentDirectoryKey + ) { + $scope.$emit('refreshFilesList'); + } + break; + } + case UploadAction.CreatedDirectory: { + const parentDirectoryKey = message.data.directoryKey + .split("/") + .slice(0, -2) + .reduce((r, p) => r + p + "/", ""); + if ( + $scope.currentInfo.bucketName === message.data.bucket && + $scope.currentInfo.key === parentDirectoryKey + ) { + $scope.$emit('refreshFilesList'); + } + break; + } + default: { + console.warn("renderer received unknown/unhandled action, message:", message); + } + } + }); + }); + ipcUploadManager.updateConfig({ + resumeUpload: Settings.resumeUpload !== 0, + maxConcurrency: Settings.maxUploadConcurrency, + multipartUploadSize: Settings.multipartUploadSize * ByteSize.MB, + multipartUploadThreshold: Settings.multipartUploadThreshold * ByteSize.MB, + uploadSpeedLimit: Settings.uploadSpeedLimitEnabled * ByteSize.KB, + isDebug: Settings.isDebug !== 0, + isSkipEmptyDirectory: $scope.emptyFolderUploading.enabled, + persistPath: getProgFilePath(), + }); + ipcUploadManager.loadPersistJobs({ + clientOptions: { + accessKey: AuthInfo.get().id, + secretKey: AuthInfo.get().secret, + ucUrl: NgConfig.ucUrl || "", + regions: NgConfig.regions || [], + }, + uploadOptions: { + userNatureLanguage: localStorage.getItem("lang") || "zh-CN", + }, + }); + uploaderTimer = setInterval(() => { + let query; + if ($scope.transSearch.uploadJob) { + if (Object.values(Status).includes($scope.transSearch.uploadJob.trim())) { + query = { + status: $scope.transSearch.uploadJob.trim(), + }; + } else { + query = { + name: $scope.transSearch.uploadJob.trim(), + }; + } + } + ipcUploadManager.updateUiData({ + pageNum: 0, + count: $scope.lists.uploadJobListLimit, + query: query, + }); + }, 1000); + + function getProgFilePath() { + const folder = Global.config_path; + if (!fs.existsSync(folder)) { + fs.mkdirSync(folder); + } + + const username = AuthInfo.get().id || "kodo-browser"; + return path.join(folder, "upprog_" + username + ".json"); + } + } + /** * upload * @param filePaths [] {array}, iter for folder @@ -77,21 +217,26 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ */ function uploadFilesHandler(filePaths, bucketInfo,uploadOptions) { Toast.info(T("upload.addtolist.on")); - UploadMgr.createUploadJobs(filePaths, bucketInfo, uploadOptions, function (isCancelled) { - Toast.info(T("upload.addtolist.success")); - - $scope.transTab = 1; - $scope.toggleTransVisible(true); - - AuditLog.log( - AuditLog.Action.UploadFilesStart, - { - regionId: bucketInfo.region, - bucket: bucketInfo.bucketName, - to: bucketInfo.key, - from: filePaths, - }, - ); + ipcUploadManager.addJobs({ + filePathnameList: filePaths, + destInfo: { + bucketName: bucketInfo.bucketName, + key: bucketInfo.key, + regionId: bucketInfo.regionId, + }, + uploadOptions: { + isOverwrite: uploadOptions.isOverwrite, + storageClassName: uploadOptions.storageClassName, + storageClasses: bucketInfo.availableStorageClasses, + userNatureLanguage: localStorage.getItem('lang') || 'zh-CN', + }, + clientOptions: { + accessKey: AuthInfo.get().id, + secretKey: AuthInfo.get().secret, + ucUrl: ngConfig.ucUrl || "", + regions: ngConfig.regionId || [], + backendMode: bucketInfo.qiniuBackendMode, + }, }); } @@ -123,24 +268,6 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ function calcTotalProg() { let c = 0, c2 = 0, cf = 0, cf2 = 0, cs = 0, cs2 = 0; - angular.forEach($scope.lists.uploadJobList, function (n) { - if (n.status === 'running') { - c++; - } - if (n.status === 'waiting') { - c++; - } - if (n.status === 'verifying') { - c++; - } - if (n.status === 'failed') { - cf++; - } - if (n.status === 'stopped') { - c++; - cs++; - } - }); angular.forEach($scope.lists.downloadJobList, function (n) { if (n.status === 'running') { c2++; @@ -157,11 +284,8 @@ webModule.controller(TRANSFER_FRAME_CONTROLLER_NAME, [ } }); - $scope.totalStat.running = c + c2; - $scope.totalStat.total = $scope.lists.uploadJobList.length + $scope.lists.downloadJobList.length; - $scope.totalStat.upDone = $scope.lists.uploadJobList.length - c; - $scope.totalStat.upStopped = cs; - $scope.totalStat.upFailed = cf; + $scope.totalStat.running = $scope.totalStat.upRunning + c2; + $scope.totalStat.total = $scope.totalStat.up + $scope.lists.downloadJobList.length; $scope.totalStat.downDone = $scope.lists.downloadJobList.length - c2; $scope.totalStat.downStopped = cs2; $scope.totalStat.downFailed = cf2; diff --git a/src/renderer/main/files/transfer/uploads.html b/src/renderer/main/files/transfer/uploads.html index 2232c334..655a0914 100755 --- a/src/renderer/main/files/transfer/uploads.html +++ b/src/renderer/main/files/transfer/uploads.html @@ -3,18 +3,18 @@
    + class="form-control input-sm" style="width:200px;" ng-model="transSearch.uploadJob">
    @@ -96,11 +96,11 @@
  • - + {{item.prog.loaded|sizeFormat}}/{{item.prog.total|sizeFormat}} + ng-if="item.progress.loaded!=item.progress.total">{{item.progress.loaded|sizeFormat}}/{{item.progress.total|sizeFormat}} - , {{item.predictLeftTime|leftTimeFormat}} + , {{item.estimatedTime|leftTimeFormat}} {{'upload.duplicated'|translate}} @@ -108,8 +108,8 @@
    - @@ -136,7 +136,7 @@
    - + {{'loading'|translate}}
    @@ -146,7 +146,7 @@ diff --git a/src/renderer/main/files/transfer/uploads.js b/src/renderer/main/files/transfer/uploads.js index 68120874..031cc7a2 100755 --- a/src/renderer/main/files/transfer/uploads.js +++ b/src/renderer/main/files/transfer/uploads.js @@ -1,10 +1,9 @@ import angular from "angular" import webModule from '@/app-module/web' - +import ipcUploadManager from "@/components/services/ipc-upload-manager" import jobUtil from '@/components/services/job-util' import DelayDone from '@/components/services/delay-done' -import UploadMgr from '@/components/services/upload-manager' import { TOAST_FACTORY_NAME as Toast } from '@/components/directives/toast-list' import { EMPTY_FOLDER_UPLOADING, @@ -19,7 +18,6 @@ webModule.controller(TRANSFER_UPLOAD_CONTROLLER_NAME, [ "$translate", jobUtil, DelayDone, - UploadMgr, Toast, Dialog, function ( @@ -28,14 +26,14 @@ webModule.controller(TRANSFER_UPLOAD_CONTROLLER_NAME, [ $translate, jobUtil, DelayDone, - UploadMgr, Toast, Dialog ) { - var T = $translate.instant; + const T = $translate.instant; angular.extend($scope, { triggerEmptyFolder: triggerEmptyFolder, + stopItem: stopItem, showRemoveItem: showRemoveItem, clearAllCompleted: clearAllCompleted, clearAll: clearAll, @@ -43,62 +41,54 @@ webModule.controller(TRANSFER_UPLOAD_CONTROLLER_NAME, [ startAll: startAll, checkStartJob: checkStartJob, - sch: { - upname: null - }, - schKeyFn: function (item) { - return ( - item.options.from.name + - " " + - item.status + - " " + - jobUtil.getStatusLabel(item.status) - ); - }, - limitToNum: 100, loadMoreUploadItems: loadMoreItems }); function loadMoreItems() { - var len = $scope.lists.uploadJobList.length; - if ($scope.limitToNum < len) { - $scope.limitToNum += Math.min(100, len - $scope.limitToNum); + const len = $scope.totalStat.up; + if ($scope.lists.uploadJobListLimit < len) { + $scope.lists.uploadJobListLimit += Math.min(100, len - $scope.lists.uploadJobListLimit); } } function triggerEmptyFolder() { $scope.emptyFolderUploading.enabled = !$scope.emptyFolderUploading.enabled; localStorage.setItem(EMPTY_FOLDER_UPLOADING, $scope.emptyFolderUploading.enabled); + ipcUploadManager.updateConfig({ + isSkipEmptyDirectory: !$scope.emptyFolderUploading.enabled, + }); } function checkStartJob(item, force) { if (force) { - item.start(true); + ipcUploadManager.startJob({ + jobId: item.id, + forceOverwrite: force, + }); } else { - item.wait(); + ipcUploadManager.waitJob({ + jobId: item.id, + }); } + } - UploadMgr.trySchedJob(); + function stopItem(item) { + ipcUploadManager.stopJob({ + jobId: item.id, + }); } function showRemoveItem(item) { - if (item.status == "finished") { + if (item.status === "finished") { doRemove(item); } else { - var title = T("remove.from.list.title"); //'从列表中移除' - var message = T("remove.from.list.message"); //'确定移除该上传任务?' + const title = T("remove.from.list.title"); //'从列表中移除' + const message = T("remove.from.list.message"); //'确定移除该上传任务?' Dialog.confirm( title, message, (btn) => { if (btn) { - if (item.status == "running" || - item.status == "waiting" || - item.status == "verifying" || - item.status == "duplicated") { - item.stop(); - } - doRemove(item); } }, @@ -108,126 +98,56 @@ webModule.controller(TRANSFER_UPLOAD_CONTROLLER_NAME, [ } function doRemove(item) { - var jobs = $scope.lists.uploadJobList; - for (var i = 0; i < jobs.length; i++) { - if (item === jobs[i]) { - jobs.splice(i, 1); - break; - } - } - - $timeout(() => { - UploadMgr.trySaveProg(); - $scope.calcTotalProg(); + ipcUploadManager.removeJob({ + jobId: item.id, }); } function clearAllCompleted() { - var jobs = $scope.lists.uploadJobList; - for (var i = 0; i < jobs.length; i++) { - if ("finished" == jobs[i].status) { - jobs.splice(i, 1); - i--; - } - } - - $timeout(() => { - $scope.calcTotalProg(); - }); + ipcUploadManager.cleanUpJobs(); } function clearAll() { if (!$scope.lists.uploadJobList || - $scope.lists.uploadJobList.length == 0) { + $scope.lists.uploadJobList.length === 0) { return; } - var title = T("clear.all.title"); //清空所有 - var message = T("clear.all.upload.message"); //确定清空所有上传任务? + const title = T("clear.all.title"); //清空所有 + const message = T("clear.all.upload.message"); //确定清空所有上传任务? Dialog.confirm( title, message, (btn) => { if (btn) { - var jobs = $scope.lists.uploadJobList; - for (var i = 0; i < jobs.length; i++) { - var job = jobs[i]; - if (job.status == "running" || - job.status == "waiting" || - job.status == "verifying" || - job.status == "duplicated") { - job.stop(); - } - - jobs.splice(i, 1); - i--; - } - - $timeout(() => { - UploadMgr.trySaveProg(); - $scope.calcTotalProg(); - }); + ipcUploadManager.removeAllJobs(); } }, 1 ); } - var stopFlag = false; - function stopAll() { - var arr = $scope.lists.uploadJobList; - if (arr && arr.length > 0) { - stopFlag = true; - - UploadMgr.stopCreatingJobs(); - - Toast.info(T("pause.on")); //'正在暂停...' - $scope.allActionBtnDisabled = true; - - angular.forEach(arr, function (n) { - if (item.resumable && ( - n.status == "running" || - n.status == "waiting" || - n.status == "verifying" - )) - n.stop(); - }); - Toast.info(T("pause.success")); + Toast.info(T("pause.on")); //'正在暂停...' + $scope.allActionBtnDisabled = true; - $timeout(function () { - UploadMgr.trySaveProg(); - $scope.allActionBtnDisabled = false; - }, 100); - } + ipcUploadManager.stopAllJobs() + + Toast.info(T("pause.success")); + + $timeout(function () { + $scope.allActionBtnDisabled = false; + }, 100); } function startAll() { - var arr = $scope.lists.uploadJobList; - stopFlag = false; - //串行 - if (arr && arr.length > 0) { - $scope.allActionBtnDisabled = true; - DelayDone.seriesRun( - arr, - function (n, fn) { - if (stopFlag) { - return; - } + $scope.allActionBtnDisabled = true; - if (n && (n.status == "stopped" || n.status == "failed")) { - n.wait(); - } - - UploadMgr.trySchedJob(); + ipcUploadManager.startAllJobs() - fn(); - }, - function doneFn() { - $scope.allActionBtnDisabled = false; - } - ); - } + $timeout(function () { + $scope.allActionBtnDisabled = false; + }, 100); } } ]); diff --git a/src/renderer/main/modals/settings.html b/src/renderer/main/modals/settings.html index 5a722d75..4161cd7a 100755 --- a/src/renderer/main/modals/settings.html +++ b/src/renderer/main/modals/settings.html @@ -44,7 +44,7 @@
    {{'settings.WhetherResumeUpload'|translate}}
    {{'settings.ResumeUploadSize'|translate}}:
    - +
    diff --git a/src/renderer/models/file-item.ts b/src/renderer/models/file-item.ts index 884efcb4..ed4635d8 100644 --- a/src/renderer/models/file-item.ts +++ b/src/renderer/models/file-item.ts @@ -1,7 +1,7 @@ import { Path as QiniuPath } from "qiniu-path/dist/src/path"; import { ObjectInfo } from "kodo-s3-adapter-sdk/dist/adapter"; import * as qiniuPathConvertor from "qiniu-path/dist/src/convert"; -import Duration from "@/const/duration"; +import Duration from "@common/const/duration"; export enum ItemType { Directory = "folder", diff --git a/src/renderer/models/job/upload-job.test.ts b/src/renderer/models/job/upload-job.test.ts deleted file mode 100644 index a55d0f6b..00000000 --- a/src/renderer/models/job/upload-job.test.ts +++ /dev/null @@ -1,180 +0,0 @@ -jest.mock("electron", () => ({ - __esModule: true, - ipcRenderer: { - on: jest.fn(), - send: jest.fn(), - removeListener: jest.fn(), - } -})); - -import { ipcRenderer } from "electron"; -import * as AppConfig from "@/const/app-config" - -import { EventKey, IpcJobEvent, Status } from "./types"; -import { uploadOptionsFromNewJob } from "./_mock-helpers_/data"; - -import UploadJob from "./upload-job"; - -describe("test models/job/upload-job.ts", () => { - describe("test stop", () => { - it("stop", () => { - const uploadJob = new UploadJob(uploadOptionsFromNewJob); - const spiedEmit = jest.spyOn(uploadJob, "emit"); - spiedEmit.mockImplementation((_eventName: string, ..._args: any[]) => uploadJob); - expect(uploadJob.stop()).toBe(uploadJob); - expect(uploadJob.speed).toBe(0); - expect(uploadJob.predictLeftTime).toBe(0); - expect(uploadJob.status).toBe(Status.Stopped); - expect(uploadJob.emit).toBeCalledWith("stop"); - expect(ipcRenderer.send).toBeCalledWith( - "asynchronous-job", - { - job: uploadJob.id, - key: IpcJobEvent.Stop, - } - ); - expect(ipcRenderer.removeListener).toBeCalledWith( - uploadJob.id, - uploadJob.startUpload, - ); - }); - }); - - describe("test start", () => { - it("start()", () => { - const uploadJob = new UploadJob(uploadOptionsFromNewJob); - const spiedEmit = jest.spyOn(uploadJob, "emit"); - spiedEmit.mockImplementation((_eventName: string, ..._args: any[]) => uploadJob); - expect(uploadJob.start()).toBe(uploadJob); - expect(uploadJob.message).toBe(""); - - // private status flow - expect(uploadJob.emit).toBeCalledWith("statuschange", Status.Running); - expect(uploadJob.status).toBe(Status.Running); - - // ipcRenderer flow - expect(ipcRenderer.on).toBeCalledWith(uploadJob.id, uploadJob.startUpload); - expect(ipcRenderer.send).toBeCalledWith( - "asynchronous-job", - { - clientOptions: { - accessKey: "NgKd0BmebvsFERFEBfKVVZGeGn7VsZQe_H_AunOC", - backendMode: "kodo", - regions: [], - secretKey: "lp4Zv3Gi_7CHtxNTcJx2Pum5hUJB3gHROcg4vp0i", - ucUrl: undefined, - userNatureLanguage: "zh-CN", - }, - job: uploadJob.id, - key: "job-upload", - options: { - kodoBrowserVersion: AppConfig.app.version, - maxConcurrency: 10, - multipartUploadSize: 16777216, - multipartUploadThreshold: 104857600, - resumeUpload: false, - uploadSpeedLimit: 0, - }, - params: { - bucket: "kodo-browser-dev", - key: "remote/path/to/out.gif", - localFile: "/local/path/to/out.gif", - overwriteDup: false, - region: "cn-east-1", - storageClassName: "Standard", - storageClasses: [], - isDebug: false, - }, - }, - ); - - // startSpeedCounter flow - expect(uploadJob.speed).toBe(0); - expect(uploadJob.predictLeftTime).toBe(0); - uploadJob.stop(); - }); - }); - - describe("test resume upload job", () => { - it("getInfoForSave()", () => { - const uploadJob = new UploadJob(uploadOptionsFromNewJob); - uploadJob.on('partcomplete', (data) => { - uploadJob.uploadedId = data.uploadId; - uploadJob.uploadedParts[data.part.partNumber] = data.part; - return false; - }) - - // stat - const fakeProgressTotal = 1024; - const fakeProgressResumable = true; - uploadJob.startUpload(null, { - key: EventKey.Stat, - data: { - progressTotal: fakeProgressTotal, - progressResumable: fakeProgressResumable, - }, - }); - expect(uploadJob.prog.total).toBe(fakeProgressTotal); - expect(uploadJob.prog.resumable).toBe(fakeProgressResumable); - - // progress - const fakeProgressLoaded = 512; - uploadJob.startUpload(null, { - key: EventKey.Progress, - data: { - progressLoaded: fakeProgressLoaded, - progressResumable: fakeProgressResumable, - }, - }); - expect(uploadJob.prog.loaded).toBe(fakeProgressLoaded); - expect(uploadJob.prog.resumable).toBe(fakeProgressResumable); - - // part uploaded - const fakeUploadedId = 'fakeUploadId'; - const fakeUploadedPart = { - partNumber: 0, - etag: 'fakeETag', - }; - uploadJob.startUpload(null, { - key: EventKey.PartUploaded, - data: { - uploadId: fakeUploadedId, - part: fakeUploadedPart, - }, - }); - expect(uploadJob.uploadedParts.length).toBe(1); - expect(uploadJob.uploadedId).toBe(fakeUploadedId); - expect(uploadJob.uploadedParts).toEqual([ - fakeUploadedPart, - ]); - - // info should in disk - expect(uploadJob.getInfoForSave({})) - .toEqual({ - from: uploadOptionsFromNewJob.from, - - backendMode: uploadOptionsFromNewJob.backendMode, - overwrite: uploadOptionsFromNewJob.overwrite, - to: uploadOptionsFromNewJob.to, - region: uploadOptionsFromNewJob.region, - storageClassName: uploadOptionsFromNewJob.storageClassName, - storageClasses: uploadOptionsFromNewJob.storageClasses, - - prog: { - loaded: fakeProgressLoaded, - total: fakeProgressTotal, - resumable: fakeProgressResumable, - }, - status: Status.Waiting, - uploadedId: fakeUploadedId, - uploadedParts: [ - { - PartNumber: fakeUploadedPart.partNumber, - ETag: fakeUploadedPart.etag, - }, - ], - message: "", - }); - }); - }); -}); diff --git a/src/renderer/models/job/upload-job.ts b/src/renderer/models/job/upload-job.ts deleted file mode 100644 index 46c0a630..00000000 --- a/src/renderer/models/job/upload-job.ts +++ /dev/null @@ -1,392 +0,0 @@ -import { ipcRenderer } from "electron"; -import { Region } from "kodo-s3-adapter-sdk"; -import { StorageClass } from "kodo-s3-adapter-sdk/dist/adapter"; -import { NatureLanguage } from "kodo-s3-adapter-sdk/dist/uplog"; - -import Duration from "@/const/duration"; -import * as AppConfig from "@/const/app-config"; - -import { BackendMode, EventKey, IpcUploadJob, IpcJobEvent, Status, UploadedPart } from "./types"; -import Base from "./base" -import * as Utils from "./utils"; - -// if change options, remember to check toJsonString() -interface RequiredOptions { - clientOptions: { - accessKey: string, - secretKey: string, - ucUrl: string, - regions: Region[], - }, - - from: Utils.LocalPath, - to: Utils.RemotePath, - region: string, - backendMode: BackendMode, - - overwrite: boolean, - storageClassName: StorageClass["kodoName"], - storageClasses: StorageClass[], - - userNatureLanguage: NatureLanguage, -} - -interface OptionalOptions { - maxConcurrency: number, - resumeUpload: boolean, - multipartUploadThreshold: number, - multipartUploadSize: number, - uploadSpeedLimit: number, - uploadedId: string, - uploadedParts: UploadedPart[], - - status: Status, - - prog: { - total: number, - loaded: number, - resumable?: boolean, - }, - - message: string, - isDebug: boolean, -} - -export type Options = RequiredOptions & Partial - -const DEFAULT_OPTIONS: OptionalOptions = { - maxConcurrency: 10, - resumeUpload: false, - multipartUploadThreshold: 100, - multipartUploadSize: 8, - uploadSpeedLimit: 0, // 0 means no limit - uploadedId: "", - uploadedParts: [], - - status: Status.Waiting, - - prog: { - total: 0, - loaded: 0, - }, - - message: "", - isDebug: false, -}; - -export default class UploadJob extends Base { - // - create options - - private readonly options: RequiredOptions & OptionalOptions - - // - for job save and log - - readonly id: string - readonly kodoBrowserVersion: string - - // - for UI - - private __status: Status - // speed - speedTimerId?: number = undefined - speed: number = 0 - predictLeftTime: number = 0 - // message - message: string - - // - for resume from break point - - prog: OptionalOptions["prog"] - uploadedId: string - uploadedParts: UploadedPart[] - - constructor(config: Options) { - super(); - this.id = `uj-${new Date().getTime()}-${Math.random().toString().substring(2)}` - this.kodoBrowserVersion = AppConfig.app.version; - - this.options = { - ...DEFAULT_OPTIONS, - ...config, - } - - this.__status = this.options.status; - - this.prog = { - ...this.options.prog, - } - this.uploadedId = this.options.uploadedId; - this.uploadedParts = [ - ...this.options.uploadedParts, - ]; - - this.message = this.options.message; - - this.startUpload = this.startUpload.bind(this); - } - - // TypeScript specification (8.4.3) says... - // > Accessors for the same member name must specify the same accessibility - private set _status(value: Status) { - this.__status = value; - this.emit("statuschange", this.status); - - if ( - this.status === Status.Failed - || this.status === Status.Stopped - || this.status === Status.Finished - || this.status === Status.Duplicated - ) { - clearInterval(this.speedTimerId); - - this.speed = 0; - this.predictLeftTime = 0; - } - } - - get status(): Status { - return this.__status - } - - get isStopped(): boolean { - return this.status !== Status.Running; - } - - private get ipcUploadJob(): IpcUploadJob { - return { - job: this.id, - key: IpcJobEvent.Upload, - clientOptions: { - ...this.options.clientOptions, - // if ucUrl is not undefined, downloader will use it generator url - ucUrl: this.options.clientOptions.ucUrl === '' - ? undefined - : this.options.clientOptions.ucUrl, - backendMode: this.options.backendMode, - - userNatureLanguage: this.options.userNatureLanguage, - }, - options: { - resumeUpload: this.options.resumeUpload, - maxConcurrency: this.options.maxConcurrency, - multipartUploadThreshold: this.options.multipartUploadThreshold * 1024 * 1024, - multipartUploadSize: this.options.multipartUploadSize * 1024 * 1024, - uploadSpeedLimit: this.options.uploadSpeedLimit, - kodoBrowserVersion: this.kodoBrowserVersion, - }, - params: { - region: this.options.region, - bucket: this.options.to.bucket, - key: this.options.to.key, - localFile: this.options.from.path, - overwriteDup: this.options.overwrite, - storageClassName: this.options.storageClassName, - storageClasses: this.options.storageClasses, - isDebug: this.options.isDebug, - } - } - } - - start( - forceOverwrite: boolean = false, - prog?: { // not same as Options["prog"] - uploadedId: string, - uploadedParts: UploadedPart[] - }, - ): this { - if (this.status === Status.Running || this.status === Status.Finished) { - return this; - } - - if (this.options.isDebug) { - console.log(`Try uploading ${this.options.from.path} to kodo://${this.options.to.bucket}/${this.options.to.key}`); - } - - this.message = "" - - this._status = Status.Running; - - const job = this.ipcUploadJob; - if (forceOverwrite) { - job.params.overwriteDup = true; - } - if (prog) { - job.params.uploadedId = prog.uploadedId; - job.params.uploadedParts = prog.uploadedParts; - } - - if (this.options.isDebug) { - console.log(`[JOB] sched starting => ${JSON.stringify(job)}`) - } - - ipcRenderer.on(this.id, this.startUpload); - ipcRenderer.send("asynchronous-job", job); - - this.startSpeedCounter(); - - return this; - } - - stop(): this { - if (this.status === Status.Stopped) { - return this; - } - - if (this.options.isDebug) { - console.log(`Pausing ${this.options.from.path}`); - } - - clearInterval(this.speedTimerId); - - this.speed = 0; - this.predictLeftTime = 0; - - this._status = Status.Stopped; - this.emit("stop"); - - ipcRenderer.send("asynchronous-job", { - job: this.id, - key: IpcJobEvent.Stop, - }); - ipcRenderer.removeListener(this.id, this.startUpload); - - return this; - } - - wait(): this { - if (this.status === Status.Waiting) { - return this; - } - - if (this.options.isDebug) { - console.log(`Pending ${this.options.from.path}`); - } - - this._status = Status.Waiting; - this.emit("pause"); - - return this; - } - - startUpload(_: any, data: any) { - if (this.options.isDebug) { - console.log("[IPC MAIN]", data); - } - - switch (data.key) { - case EventKey.Duplicated: - ipcRenderer.removeListener(this.id, this.startUpload); - this._status = Status.Duplicated; - this.emit("fileDuplicated", data); - return; - case EventKey.Stat: - this.prog.total = data.data.progressTotal; - this.prog.resumable = data.data.progressResumable; - this.emit("progress", this.prog); - return; - case EventKey.Progress: - this.prog.loaded = data.data.progressLoaded; - this.prog.resumable = data.data.progressResumable; - this.emit("progress", this.prog); - return; - case EventKey.PartUploaded: - this.emit("partcomplete", data.data); - return; - case EventKey.Uploaded: - ipcRenderer.removeListener(this.id, this.startUpload); - - this._status = Status.Finished; - this.emit("complete"); - return; - case EventKey.Error: - console.error("upload object error:", data); - ipcRenderer.removeListener(this.id, this.startUpload); - - this.message = data; - this._status = Status.Failed; - this.emit("error", data.error); - return; - case EventKey.Debug: - if (!this.options.isDebug) { - console.log("Debug", data); - } - return; - default: - console.warn("Unknown", data); - return; - } - } - - private startSpeedCounter() { - const startedAt = new Date().getTime(); - - let lastLoaded = this.prog.loaded; - let lastSpeed = 0; - - clearInterval(this.speedTimerId); - const intervalDuration = Duration.Second; - this.speedTimerId = setInterval(() => { - if (this.isStopped) { - this.speed = 0; - this.predictLeftTime = 0; - return; - } - - const avgSpeed = this.prog.loaded / (new Date().getTime() - startedAt) * Duration.Second; - this.speed = this.prog.loaded - lastLoaded; - if (this.speed <= 0 || (lastSpeed / this.speed) > 1.1) { - this.speed = lastSpeed * 0.95; - } - if (this.speed < avgSpeed) { - this.speed = avgSpeed; - } - - lastLoaded = this.prog.loaded; - lastSpeed = this.speed; - - - if (this.options.uploadSpeedLimit && this.speed > this.options.uploadSpeedLimit * 1024) { - this.speed = this.options.uploadSpeedLimit * 1024; - } - this.emit('speedchange', this.speed * 1.2); - - this.predictLeftTime = this.speed <= 0 - ? 0 - : Math.floor((this.prog.total - this.prog.loaded) / this.speed * 1000); - }, intervalDuration) as unknown as number; // hack type problem of nodejs and browser - } - - getInfoForSave({ - from - }: { - from?: { - size?: number, - mtime?: number, - } - }) { - return { - from: { - ...this.options.from, - ...from, - }, - - // read-only info - storageClasses: this.options.storageClasses, - region: this.options.region, - to: this.options.to, - overwrite: this.options.overwrite, - storageClassName: this.options.storageClassName, - backendMode: this.options.backendMode, - - // real-time info - prog: { - loaded: this.prog.loaded, - total: this.prog.total, - resumable: this.prog.resumable - }, - status: this.status, - message: this.message, - uploadedId: this.uploadedId, - uploadedParts: this.uploadedParts.map((part) => { - return { PartNumber: part.partNumber, ETag: part.etag }; - }), - }; - } -} - diff --git a/tsconfig.json b/tsconfig.json index 199997b8..36ef8ba7 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,6 +24,9 @@ "paths": { "@/*": [ "renderer/*" + ], + "@common/*": [ + "common/*" ] }, "esModuleInterop": true, diff --git a/webpack/.gitkeep b/webpack/.gitkeep deleted file mode 100644 index 341adb83..00000000 --- a/webpack/.gitkeep +++ /dev/null @@ -1,3 +0,0 @@ -webpack 相关改动在本文件 commit 之前 ---- -ts 相关改动在本文件 commit 之后 diff --git a/webpack/paths.js b/webpack/paths.js index 1fe7b301..bd916ded 100644 --- a/webpack/paths.js +++ b/webpack/paths.js @@ -22,13 +22,14 @@ module.exports = { appPages: pages.map(resolveApp), appPackageJson: resolveApp('package.json'), appSrc: resolveApp('src'), + appCommon: resolveApp('src/common'), appNodeModules: resolveApp('node_modules'), appWebpackCache: resolveApp('node_modules/.cache'), appMain: resolveApp('src/main'), appMainIndex: resolveApp('src/main/index.js'), appMainDownloadWorker: resolveApp('src/main/download-worker.js'), - appMainUploadWorker: resolveApp('src/main/upload-worker.js'), + appMainUploader: resolveApp('src/main/uploader/index.ts'), appBuildMain: resolveApp('dist/main'), appRenderer: resolveApp('src/renderer'), diff --git a/webpack/webpack-main.config.js b/webpack/webpack-main.config.js index 874b2b4f..62c4c32f 100644 --- a/webpack/webpack-main.config.js +++ b/webpack/webpack-main.config.js @@ -8,10 +8,16 @@ module.exports = function(webpackEnv) { return { target: 'electron-main', mode: isEnvProduction ? 'production' : isEnvDevelopment && 'development', + resolve: { + alias: { + '@common': paths.appCommon, + }, + extensions: ['.ts', '.js'], + }, entry: { main: paths.appMainIndex, 'download-worker': paths.appMainDownloadWorker, - 'upload-worker': paths.appMainUploadWorker, + 'uploader': paths.appMainUploader, }, output: { filename: '[name]-bundle.js', @@ -21,6 +27,11 @@ module.exports = function(webpackEnv) { path: paths.appBuildMain, clean: true, }, + module: { + rules: [ + { test: /\.ts$/, loader: "ts-loader" }, + ], + }, optimization: { splitChunks: { chunks: "all", diff --git a/webpack/webpack-renderer.config.js b/webpack/webpack-renderer.config.js index fd1da749..22b6a88a 100644 --- a/webpack/webpack-renderer.config.js +++ b/webpack/webpack-renderer.config.js @@ -22,6 +22,7 @@ module.exports = function(webpackEnv) { resolve: { alias: { '@': paths.appRenderer, + '@common': paths.appCommon, '@template-mappings': paths.appRendererTemplateMappings, }, extensions: ['.ts', '.js'], diff --git a/yarn.lock b/yarn.lock index e92d80ac..73739207 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1299,6 +1299,11 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" +"@root/walk@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@root/walk/-/walk-1.1.0.tgz#784d81e8b8d3fb3bedbb0e18952e4ea577e3796a" + integrity sha512-FfXPAta9u2dBuaXhPRawBcijNC9rmKVApmbi6lIZyg36VR/7L02ytxoY5K/14PJlHqiBUoYII73cTlekdKTUOw== + "@sindresorhus/is@^0.14.0": version "0.14.0" resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" @@ -1448,6 +1453,11 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.9.tgz#97edc9037ea0c38585320b28964dde3b39e4660d" integrity sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ== +"@types/lodash@^4.14.182": + version "4.14.182" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.182.tgz#05301a4d5e62963227eaafe0ce04dd77c54ea5c2" + integrity sha512-/THyiqyQAP9AfARo4pF+aCGcyiQ94tX/Is2I7HofNRqoYLgN1PBoOWu2/zTA5zMxzP5EFutMtWtGAFRKUe961Q== + "@types/minimatch@*": version "3.0.3" resolved "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" @@ -6925,10 +6935,10 @@ kleur@^3.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== -kodo-s3-adapter-sdk@0.2.29: - version "0.2.29" - resolved "https://registry.yarnpkg.com/kodo-s3-adapter-sdk/-/kodo-s3-adapter-sdk-0.2.29.tgz#28e518127cbb1daf6ef70877bc3a827e63c98dd5" - integrity sha512-SVIqTGbF2g33UzS769in81sD5fSi8XkXcp/lzdkRfl2qnNH91ECg0QgPcVUztvQ2KkjgiTpTRW6iNPnu8TVO6g== +kodo-s3-adapter-sdk@0.2.30: + version "0.2.30" + resolved "https://registry.yarnpkg.com/kodo-s3-adapter-sdk/-/kodo-s3-adapter-sdk-0.2.30.tgz#6f9ff034eb75e33979aa638fbd2c921df2a903b4" + integrity sha512-E1BpNCSSwSG40wJAPT3yAz04jLB4g/FG6qB6uYQNNNXtsZ6JNjlZui1KJq8vuf9b7I+i6zYkpQW860V8CEQySg== dependencies: async-lock "^1.2.4" aws-sdk "^2.800.0"