Skip to content

Commit

Permalink
feature: send download-progress information when downloading diff pac…
Browse files Browse the repository at this point in the history
…kages #2521 #4919 #5341...and so many other duplicate tickets (#5515)
  • Loading branch information
mastergberry committed Jan 15, 2021
1 parent 53270cf commit d05a62b
Show file tree
Hide file tree
Showing 5 changed files with 196 additions and 18 deletions.
15 changes: 12 additions & 3 deletions packages/electron-updater/src/AppImageUpdater.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import { unlinkSync } from "fs"
import * as path from "path"
import { DownloadUpdateOptions } from "./AppUpdater"
import { BaseUpdater, InstallOptions } from "./BaseUpdater"
import { DifferentialDownloaderOptions } from "./differentialDownloader/DifferentialDownloader"
import { FileWithEmbeddedBlockMapDifferentialDownloader } from "./differentialDownloader/FileWithEmbeddedBlockMapDifferentialDownloader"
import { DOWNLOAD_PROGRESS } from "./main"
import { findFile } from "./providers/Provider"

export class AppImageUpdater extends BaseUpdater {
Expand Down Expand Up @@ -42,14 +44,21 @@ export class AppImageUpdater extends BaseUpdater {

let isDownloadFull = false
try {
await new FileWithEmbeddedBlockMapDifferentialDownloader(fileInfo.info, this.httpExecutor, {
const downloadOptions: DifferentialDownloaderOptions = {
newUrl: fileInfo.url,
oldFile,
logger: this._logger,
newFile: updateFile,
isUseMultipleRangeRequest: provider.isUseMultipleRangeRequest,
requestHeaders: downloadUpdateOptions.requestHeaders,
})
cancellationToken: downloadUpdateOptions.cancellationToken,
}

if (this.listenerCount(DOWNLOAD_PROGRESS) > 0) {
downloadOptions.onProgress = it => this.emit(DOWNLOAD_PROGRESS, it)
}

await new FileWithEmbeddedBlockMapDifferentialDownloader(fileInfo.info, this.httpExecutor, downloadOptions)
.download()
}
catch (e) {
Expand Down Expand Up @@ -109,4 +118,4 @@ export class AppImageUpdater extends BaseUpdater {
}
return true
}
}
}
31 changes: 23 additions & 8 deletions packages/electron-updater/src/NsisUpdater.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ import * as path from "path"
import { AppAdapter } from "./AppAdapter"
import { DownloadUpdateOptions } from "./AppUpdater"
import { BaseUpdater, InstallOptions } from "./BaseUpdater"
import { DifferentialDownloaderOptions } from "./differentialDownloader/DifferentialDownloader"
import { FileWithEmbeddedBlockMapDifferentialDownloader } from "./differentialDownloader/FileWithEmbeddedBlockMapDifferentialDownloader"
import { GenericDifferentialDownloader } from "./differentialDownloader/GenericDifferentialDownloader"
import { newUrlFromBase, ResolvedUpdateFileInfo } from "./main"
import { DOWNLOAD_PROGRESS, newUrlFromBase, ResolvedUpdateFileInfo } from "./main"
import { findFile, Provider } from "./providers/Provider"
import { unlink } from "fs-extra"
import { verifySignature } from "./windowsExecutableCodeSignatureVerifier"
Expand Down Expand Up @@ -41,7 +42,7 @@ export class NsisUpdater extends BaseUpdater {
}

if (isWebInstaller) {
if (await this.differentialDownloadWebPackage(packageInfo!!, packageFile!!, provider)) {
if (await this.differentialDownloadWebPackage(downloadUpdateOptions, packageInfo!!, packageFile!!, provider)) {
try {
await this.httpExecutor.download(new URL(packageInfo!!.path), packageFile!!, {
headers: downloadUpdateOptions.requestHeaders,
Expand Down Expand Up @@ -157,15 +158,22 @@ export class NsisUpdater extends BaseUpdater {
}
}

const blockMapDataList = await Promise.all([downloadBlockMap(oldBlockMapUrl), downloadBlockMap(newBlockMapUrl)])
await new GenericDifferentialDownloader(fileInfo.info, this.httpExecutor, {
const downloadOptions: DifferentialDownloaderOptions = {
newUrl: fileInfo.url,
oldFile: path.join(this.downloadedUpdateHelper!!.cacheDir, CURRENT_APP_INSTALLER_FILE_NAME),
logger: this._logger,
newFile: installerPath,
isUseMultipleRangeRequest: provider.isUseMultipleRangeRequest,
requestHeaders: downloadUpdateOptions.requestHeaders,
})
cancellationToken: downloadUpdateOptions.cancellationToken,
}

if (this.listenerCount(DOWNLOAD_PROGRESS) > 0) {
downloadOptions.onProgress = it => this.emit(DOWNLOAD_PROGRESS, it)
}

const blockMapDataList = await Promise.all([downloadBlockMap(oldBlockMapUrl), downloadBlockMap(newBlockMapUrl)])
await new GenericDifferentialDownloader(fileInfo.info, this.httpExecutor, downloadOptions)
.download(blockMapDataList[0], blockMapDataList[1])
return false
}
Expand All @@ -179,20 +187,27 @@ export class NsisUpdater extends BaseUpdater {
}
}

private async differentialDownloadWebPackage(packageInfo: PackageFileInfo, packagePath: string, provider: Provider<any>): Promise<boolean> {
private async differentialDownloadWebPackage(downloadUpdateOptions: DownloadUpdateOptions, packageInfo: PackageFileInfo, packagePath: string, provider: Provider<any>): Promise<boolean> {
if (packageInfo.blockMapSize == null) {
return true
}

try {
await new FileWithEmbeddedBlockMapDifferentialDownloader(packageInfo, this.httpExecutor, {
const downloadOptions: DifferentialDownloaderOptions = {
newUrl: new URL(packageInfo.path),
oldFile: path.join(this.downloadedUpdateHelper!!.cacheDir, CURRENT_APP_PACKAGE_FILE_NAME),
logger: this._logger,
newFile: packagePath,
requestHeaders: this.requestHeaders,
isUseMultipleRangeRequest: provider.isUseMultipleRangeRequest,
})
cancellationToken: downloadUpdateOptions.cancellationToken,
}

if (this.listenerCount(DOWNLOAD_PROGRESS) > 0) {
downloadOptions.onProgress = it => this.emit(DOWNLOAD_PROGRESS, it)
}

await new FileWithEmbeddedBlockMapDifferentialDownloader(packageInfo, this.httpExecutor, downloadOptions)
.download()
}
catch (e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@ import { BlockMap } from "builder-util-runtime/out/blockMapApi"
import { close, open } from "fs-extra"
import { createWriteStream } from "fs"
import { OutgoingHttpHeaders, RequestOptions } from "http"
import { ProgressInfo, CancellationToken } from "builder-util-runtime"
import { Logger } from "../main"
import { copyData } from "./DataSplitter"
import { URL } from "url"
import { computeOperations, Operation, OperationKind } from "./downloadPlanBuilder"
import { checkIsRangesSupported, executeTasksUsingMultipleRangeRequests } from "./multipleRangeDownloader"
import { ProgressDifferentialDownloadCallbackTransform, ProgressDifferentialDownloadInfo } from "./ProgressDifferentialDownloadCallbackTransform"

export interface DifferentialDownloaderOptions {
readonly oldFile: string
Expand All @@ -18,6 +20,9 @@ export interface DifferentialDownloaderOptions {
readonly requestHeaders: OutgoingHttpHeaders | null

readonly isUseMultipleRangeRequest?: boolean

readonly cancellationToken: CancellationToken
onProgress?(progress: ProgressInfo): void
}

export abstract class DifferentialDownloader {
Expand Down Expand Up @@ -74,10 +79,10 @@ export abstract class DifferentialDownloader {

logger.info(`Full: ${formatBytes(newSize)}, To download: ${formatBytes(downloadSize)} (${Math.round(downloadSize / (newSize / 100))}%)`)

return this.downloadFile(operations)
return this.downloadFile(operations, downloadSize)
}

private downloadFile(tasks: Array<Operation>): Promise<any> {
private downloadFile(tasks: Array<Operation>, downloadSize: number): Promise<any> {
const fdList: Array<OpenedFile> = []
const closeFiles = (): Promise<Array<void>> => {
return Promise.all(fdList.map(openedFile => {
Expand All @@ -87,7 +92,7 @@ export abstract class DifferentialDownloader {
})
}))
}
return this.doDownloadFile(tasks, fdList)
return this.doDownloadFile(tasks, fdList, downloadSize)
.then(closeFiles)
.catch(e => {
// then must be after catch here (since then always throws error)
Expand All @@ -113,14 +118,37 @@ export abstract class DifferentialDownloader {
})
}

private async doDownloadFile(tasks: Array<Operation>, fdList: Array<OpenedFile>): Promise<any> {
private async doDownloadFile(tasks: Array<Operation>, fdList: Array<OpenedFile>, downloadSize: number): Promise<any> {
const oldFileFd = await open(this.options.oldFile, "r")
fdList.push({descriptor: oldFileFd, path: this.options.oldFile})
const newFileFd = await open(this.options.newFile, "w")
fdList.push({descriptor: newFileFd, path: this.options.newFile})
const fileOut = createWriteStream(this.options.newFile, {fd: newFileFd})
await new Promise((resolve, reject) => {
const streams: Array<any> = []

// Create our download info transformer if we have one
let downloadInfoTransform: ProgressDifferentialDownloadCallbackTransform | undefined = undefined
if (!this.options.isUseMultipleRangeRequest && this.options.onProgress) { // TODO: Does not support multiple ranges (someone feel free to PR this!)
const expectedByteCounts: Array<number> = []
let grandTotalBytes = 0

for (const task of tasks) {
if (task.kind === OperationKind.DOWNLOAD) {
expectedByteCounts.push(task.end - task.start)
grandTotalBytes += task.end - task.start
}
}

const progressDifferentialDownloadInfo: ProgressDifferentialDownloadInfo = {
expectedByteCounts: expectedByteCounts,
grandTotal: grandTotalBytes
}

downloadInfoTransform = new ProgressDifferentialDownloadCallbackTransform(progressDifferentialDownloadInfo, this.options.cancellationToken, this.options.onProgress)
streams.push(downloadInfoTransform)
}

const digestTransform = new DigestTransform(this.blockAwareFileInfo.sha512)
// to simply debug, do manual validation to allow file to be fully written
digestTransform.isValidateOnEnd = false
Expand Down Expand Up @@ -183,6 +211,11 @@ export abstract class DifferentialDownloader {

const operation = tasks[index++]
if (operation.kind === OperationKind.COPY) {
// We are copying, let's not send status updates to the UI
if (downloadInfoTransform) {
downloadInfoTransform.beginFileCopy()
}

copyData(operation, firstStream, oldFileFd, reject, () => w(index))
return
}
Expand All @@ -195,6 +228,11 @@ export abstract class DifferentialDownloader {
debug(`download range: ${range}`)
}

// We are starting to download
if (downloadInfoTransform) {
downloadInfoTransform.beginRangeDownload()
}

const request = this.httpExecutor.createRequest(requestOptions, response => {
// Electron net handles redirects automatically, our NodeJS test server doesn't use redirects - so, we don't check 3xx codes.
if (response.statusCode >= 400) {
Expand All @@ -205,6 +243,11 @@ export abstract class DifferentialDownloader {
end: false
})
response.once("end", () => {
// Pass on that we are downloading a segment
if (downloadInfoTransform) {
downloadInfoTransform.endRangeDownload()
}

if (++downloadOperationCount === 100) {
downloadOperationCount = 0
setTimeout(() => w(index), 1000)
Expand Down Expand Up @@ -273,4 +316,4 @@ function removeQuery(url: string): string {
interface OpenedFile {
readonly descriptor: number
readonly path: string
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
import { Transform } from "stream"
import { CancellationToken } from "builder-util-runtime"

enum OperationKind {
COPY, DOWNLOAD
}

export interface ProgressInfo {
total: number
delta: number
transferred: number
percent: number
bytesPerSecond: number
}

export interface ProgressDifferentialDownloadInfo {
expectedByteCounts: Array<number>,
grandTotal: number
}

export class ProgressDifferentialDownloadCallbackTransform extends Transform {
private start = Date.now()
private transferred = 0
private delta = 0
private expectedBytes = 0
private index = 0
private operationType = OperationKind.COPY

private nextUpdate = this.start + 1000

constructor(private readonly progressDifferentialDownloadInfo: ProgressDifferentialDownloadInfo, private readonly cancellationToken: CancellationToken, private readonly onProgress: (info: ProgressInfo) => any) {
super()
}

_transform(chunk: any, encoding: string, callback: any) {
if (this.cancellationToken.cancelled) {
callback(new Error("cancelled"), null)
return
}

// Don't send progress update when copying from disk
if (this.operationType == OperationKind.COPY) {
callback(null, chunk)
return
}

this.transferred += chunk.length
this.delta += chunk.length

const now = Date.now()
if (now >= this.nextUpdate
&& this.transferred !== this.expectedBytes /* will be emitted by endRangeDownload() */
&& this.transferred !== this.progressDifferentialDownloadInfo.grandTotal /* will be emitted on _flush */) {
this.nextUpdate = now + 1000

this.onProgress({
total: this.progressDifferentialDownloadInfo.grandTotal,
delta: this.delta,
transferred: this.transferred,
percent: (this.transferred / this.progressDifferentialDownloadInfo.grandTotal) * 100,
bytesPerSecond: Math.round(this.transferred / ((now - this.start) / 1000))
})
this.delta = 0
}

callback(null, chunk)
}

beginFileCopy(): void {
this.operationType = OperationKind.COPY
}

beginRangeDownload(): void {
this.operationType = OperationKind.DOWNLOAD

this.expectedBytes += this.progressDifferentialDownloadInfo.expectedByteCounts[this.index++]
}

endRangeDownload(): void {
// _flush() will doour final 100%
if (this.transferred !== this.progressDifferentialDownloadInfo.grandTotal) {
this.onProgress({
total: this.progressDifferentialDownloadInfo.grandTotal,
delta: this.delta,
transferred: this.transferred,
percent: (this.transferred / this.progressDifferentialDownloadInfo.grandTotal) * 100,
bytesPerSecond: Math.round(this.transferred / ((Date.now() - this.start) / 1000))
})
}
}

// Called when we are 100% done with the connection/download
_flush(callback: any): void {
if (this.cancellationToken.cancelled) {
callback(new Error("cancelled"))
return
}

this.onProgress({
total: this.progressDifferentialDownloadInfo.grandTotal,
delta: this.delta,
transferred: this.transferred,
percent: 100,
bytesPerSecond: Math.round(this.transferred / ((Date.now() - this.start) / 1000))
})
this.delta = 0
this.transferred = 0


callback(null)
}
}
3 changes: 1 addition & 2 deletions packages/electron-updater/src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import { LoginCallback } from "./electronHttpExecutor"

export { AppUpdater, NoOpLogger } from "./AppUpdater"
export { UpdateInfo }
export { CancellationToken } from "builder-util-runtime"
export { Provider } from "./providers/Provider"
export { AppImageUpdater } from "./AppImageUpdater"
export { MacUpdater } from "./MacUpdater"
Expand Down Expand Up @@ -143,4 +142,4 @@ export function newUrlFromBase(pathname: string, baseUrl: URL, addRandomQueryToA
result.search = `noCache=${Date.now().toString(32)}`
}
return result
}
}

0 comments on commit d05a62b

Please sign in to comment.