feat: use go multi-part downloader to download big files (much more faster and reliable compared to node due to obvious reasons)

Author: develar

.circleci/config.yml

@@ -3,12 +3,12 @@ version: 2
 jobs:
     build:
       docker:
-        - image: circleci/node:9.3
+        - image: circleci/node:9.5
       steps:
         - checkout
         - restore_cache:
             keys:
-              - dependencies-{{ checksum "yarn.lock" }}
+              - deps-{{ checksum "yarn.lock" }}
         - restore_cache:
             keys:
               - v-1.7.11-electron
@@ -17,7 +17,7 @@ jobs:
         - run:
             command: yarn pretest
         - save_cache:
-            key: dependencies-{{ checksum "yarn.lock" }}
+            key: deps-{{ checksum "yarn.lock" }}
             paths:
               - node_modules
         - run:
@@ -38,7 +38,7 @@ jobs:
         - checkout
         - restore_cache:
             keys:
-              - dependencies-{{ checksum "yarn.lock" }}
+              - deps-{{ checksum "yarn.lock" }}
         - restore_cache:
             keys:
               - v-1.7.11-electron

package.json

@@ -10,7 +10,7 @@
     "///": "Please see https://github.com/electron-userland/electron-builder/blob/master/CONTRIBUTING.md#run-test-using-cli how to run particular test instead full (and very slow) run",
     "test": "node ./test/out/helpers/runTests.js skipArtifactPublisher ALL_TESTS=isCi",
     "test-all": "yarn pretest && node ./test/out/helpers/runTests.js",
-    "test-linux": "docker run --rm -ti -v ${PWD}:/project -v ${PWD##*/}-node-modules:/project/node_modules -v ~/Library/Caches/electron:/root/.cache/electron  -v ~/Library/Caches/electron-builder:/root/.cache/electron-builder electronuserland/builder:wine /bin/bash -c \"yarn && TEST_FILES=debTest node ./test/out/helpers/runTests.js\"",
+    "test-linux": "docker run --rm -ti -v ${PWD}:/project -v ${PWD##*/}-node-modules:/project/node_modules -v ~/Library/Caches/electron:/root/.cache/electron  -v ~/Library/Caches/electron-builder:/root/.cache/electron-builder electronuserland/builder:wine /bin/bash -c \"yarn && TEST_FILES=snapTest node ./test/out/helpers/runTests.js\"",
     "whitespace": "whitespace 'src/**/*.ts'",
     "docker-images": "docker/build.sh",
     "update-deps": "npm-check-updates -a -x gitbook-plugin-github,chalk && node ./scripts/update-deps.js",
@@ -28,7 +28,8 @@
   "///": "All dependencies for all packages (hoisted)",
   "////": "All typings are added into root `package.json` to avoid duplication errors in the IDE compiler (several `node.d.ts` files).",
   "dependencies": {
-    "7zip-bin": "~3.0.0",
+    "7zip-bin": "~3.1.0",
+    "app-builder-bin": "1.0.5",
     "archiver": "^2.1.1",
     "async-exit-hook": "^2.0.1",
     "aws-sdk": "^2.188.0",
@@ -55,7 +56,7 @@
     "normalize-package-data": "^2.4.0",
     "parse-color": "^1.0.0",
     "plist": "^2.1.0",
-    "read-config-file": "2.1.1",
+    "read-config-file": "3.0.0",
     "sanitize-filename": "^1.6.1",
     "sax": "^1.2.4",
     "semver": "^5.5.0",
@@ -85,7 +86,7 @@
     "convert-source-map": "^1.5.1",
     "decompress-zip": "^0.3.0",
     "depcheck": "^0.6.8",
-    "develar-typescript-json-schema": "0.19.0",
+    "develar-typescript-json-schema": "0.20.0",
     "electron-builder-tslint-config": "^1.1.0",
     "env-paths": "^1.0.0",
     "gitbook-plugin-analytics": "^0.2.1",
@@ -95,7 +96,7 @@
     "gitbook-plugin-github-buttons": "^3.0.0",
     "globby": "^7.1.1",
     "jest-cli": "^22.1.4",
-    "jest-junit": "^3.4.1",
+    "jest-junit": "^3.5.0",
     "jsdoc-to-markdown": "^4.0.1",
     "path-sort": "^0.1.0",
     "ts-babel": "^4.1.8",

packages/builder-util/package.json

@@ -11,6 +11,7 @@
     "out"
   ],
   "dependencies": {
+    "app-builder-bin": "1.0.5",
     "temp-file": "^3.1.1",
     "fs-extra-p": "^4.5.0",
     "is-ci": "^1.1.0",
@@ -20,9 +21,7 @@
     "debug": "^3.1.0",
     "builder-util-runtime": "^0.0.0-semantic-release",
     "source-map-support": "^0.5.3",
-    "7zip-bin": "~3.0.0",
-    "ini": "^1.3.5",
-    "tunnel-agent": "^0.6.0",
+    "7zip-bin": "~3.1.0",
     "semver": "^5.5.0",
     "lazy-val": "^1.0.3",
     "js-yaml": "^3.10.0"

packages/builder-util/src/binDownload.ts

@@ -1,19 +1,12 @@
 import { path7za } from "7zip-bin"
-import BluebirdPromise from "bluebird-lst"
-import { CancellationToken, DownloadOptions } from "builder-util-runtime"
+import { appBuilderPath } from "app-builder-bin"
 import { emptyDir, rename, unlink } from "fs-extra-p"
 import * as path from "path"
 import { getTempName } from "temp-file"
 import { statOrNull } from "./fs"
-import { httpExecutor } from "./nodeHttpExecutor"
 import { debug7zArgs, getCacheDirectory, log, spawn } from "./util"
 
-const versionToPromise = new Map<string, BluebirdPromise<string>>()
-
-export function getBinFromBintray(name: string, version: string, sha2: string): Promise<string> {
-  const dirName = `${name}-${version}`
-  return getBin(name, dirName, `https://dl.bintray.com/electron-userland/bin/${dirName}.7z`, sha2)
-}
+const versionToPromise = new Map<string, Promise<string>>()
 
 export function getBinFromGithub(name: string, version: string, checksum: string): Promise<string> {
   const dirName = `${name}-${version}`
@@ -23,15 +16,23 @@ export function getBinFromGithub(name: string, version: string, checksum: string
 export function getBin(name: string, dirName: string, url: string, checksum: string): Promise<string> {
   let promise = versionToPromise.get(dirName)
   // if rejected, we will try to download again
-  if (promise != null && !promise.isRejected()) {
+  if (promise != null) {
     return promise
   }
 
-  promise = doGetBin(name, dirName, url, checksum) as BluebirdPromise<string>
+  promise = doGetBin(name, dirName, url, checksum)
   versionToPromise.set(dirName, promise)
   return promise
 }
 
+export function download(url: string, output: string, checksum?: string | null): Promise<void> {
+  const args = ["download", "--url", url, "--output", output]
+  if (checksum != null) {
+    args.push("--sha512", checksum)
+  }
+  return spawn(appBuilderPath, args)
+}
+
 // we cache in the global location - in the home dir, not in the node_modules/.cache (https://www.npmjs.com/package/find-cache-dir) because
 // * don't need to find node_modules
 // * don't pollute user project dir (important in case of 1-package.json project structure)
@@ -55,42 +56,12 @@ async function doGetBin(name: string, dirName: string, url: string, checksum: st
   const archiveName = `${tempUnpackDir}.7z`
   // 7z doesn't create out dir, so, we don't create dir in parallel to download - dir creation will create parent dirs for archive file also
   await emptyDir(tempUnpackDir)
-  const options: DownloadOptions = {
-    skipDirCreation: true,
-    cancellationToken: new CancellationToken(),
-  }
-
-  if (checksum.length === 64 && !checksum.includes("+") && !checksum.includes("Z") && !checksum.includes("=")) {
-    (options as any).sha2 = checksum
-  }
-  else {
-    (options as any).sha512 = checksum
-  }
-
-  for (let attemptNumber = 1; attemptNumber < 4; attemptNumber++) {
-    try {
-      await httpExecutor.download(url, archiveName, options)
-    }
-    catch (e) {
-      if (attemptNumber >= 3) {
-        throw e
-      }
-
-      log.warn({...logFlags, attempt: attemptNumber}, `cannot download: ${e}`)
-      await new BluebirdPromise((resolve, reject) => {
-        setTimeout(() =>
-          httpExecutor
-            .download(url, archiveName, options)
-            .then(resolve).catch(reject), 1000 * attemptNumber)
-      })
-    }
-  }
-
+  await download(url, archiveName, checksum)
   await spawn(path7za, debug7zArgs("x").concat(archiveName, `-o${tempUnpackDir}`), {
     cwd: cachePath,
   })
 
-  await BluebirdPromise.all([
+  await Promise.all([
     rename(tempUnpackDir, dirPath)
       .catch(e => log.debug({...logFlags, tempUnpackDir, e}, `cannot move downloaded into final location (another process downloaded faster?)`)),
     unlink(archiveName),

packages/builder-util/src/nodeHttpExecutor.ts

@@ -1,90 +1,18 @@
-import { CancellationToken, configureRequestOptionsFromUrl, DownloadOptions, HttpExecutor } from "builder-util-runtime"
-import { ensureDir, readFile } from "fs-extra-p"
-import { Agent, ClientRequest, request as httpRequest } from "http"
+import { HttpExecutor, DownloadOptions } from "builder-util-runtime"
+import { download as _download } from "./binDownload"
+import { ClientRequest, request as httpRequest } from "http"
 import * as https from "https"
-import { parse as parseIni } from "ini"
-import { homedir } from "os"
-import * as path from "path"
-import { parse as parseUrl } from "url"
 
 export class NodeHttpExecutor extends HttpExecutor<ClientRequest> {
-  private httpsAgentPromise: Promise<Agent> | null = null
-
-  async download(url: string, destination: string, options: DownloadOptions = {cancellationToken: new CancellationToken()}): Promise<string> {
-    if (!options.skipDirCreation) {
-      await ensureDir(path.dirname(destination))
-    }
-
-    if (this.httpsAgentPromise == null) {
-      this.httpsAgentPromise = createAgent()
-    }
-
-    const agent = await this.httpsAgentPromise
-    return await options.cancellationToken.createPromise<string>((resolve, reject, onCancel) => {
-      this.doDownload(configureRequestOptionsFromUrl(url, {
-        headers: options.headers || undefined,
-        agent,
-      }), destination, 0, options, error => {
-        if (error == null) {
-          resolve(destination)
-        }
-        else {
-          reject(error)
-        }
-      }, onCancel)
-    })
+  // used only in tests of electron-updater
+  download(url: string, destination: string, options: DownloadOptions): Promise<string> {
+    return _download(url, destination, options == null ? null : options.sha512)
+      .then(() => destination)
   }
 
   doRequest(options: any, callback: (response: any) => void): any {
     return (options.protocol === "http:" ? httpRequest : https.request)(options, callback)
   }
 }
 
-export const httpExecutor: NodeHttpExecutor = new NodeHttpExecutor()
-
-// only https proxy
-async function proxyFromNpm() {
-  let data = ""
-  try {
-    data = await readFile(path.join(homedir(), ".npmrc"), "utf-8")
-  }
-  catch (ignored) {
-    return null
-  }
-
-  if (!data) {
-    return null
-  }
-
-  try {
-    const config = parseIni(data)
-    return config["https-proxy"] || config.proxy
-  }
-  catch (e) {
-    // used in nsis auto-updater, do not use .util.warn here
-    console.warn(e)
-    return null
-  }
-}
-
-// only https url
-async function createAgent() {
-  let proxyString = process.env.npm_config_https_proxy || process.env.HTTPS_PROXY || process.env.https_proxy || process.env.npm_config_proxy
-  if (!proxyString) {
-    proxyString = await proxyFromNpm()
-    if (!proxyString) {
-      return null
-    }
-  }
-
-  const proxy = parseUrl(proxyString)
-
-  const proxyProtocol = proxy.protocol === "https:" ? "Https" : "Http"
-  return require("tunnel-agent")[`httpsOver${proxyProtocol}`]({
-    proxy: {
-      port: proxy.port || (proxyProtocol === "Https" ? 443 : 80),
-      host: proxy.hostname,
-      proxyAuth: proxy.auth
-    }
-  })
-}
+export const httpExecutor: NodeHttpExecutor = new NodeHttpExecutor()

packages/electron-builder-lib/package.json

@@ -41,7 +41,8 @@
   "bugs": "https://github.com/electron-userland/electron-builder/issues",
   "homepage": "https://github.com/electron-userland/electron-builder",
   "dependencies": {
-    "7zip-bin": "~3.0.0",
+    "7zip-bin": "~3.1.0",
+    "app-builder-bin": "1.0.5",
     "async-exit-hook": "^2.0.1",
     "bluebird-lst": "^1.0.5",
     "chromium-pickle-js": "^0.2.0",
@@ -54,7 +55,7 @@
     "is-ci": "^1.1.0",
     "isbinaryfile": "^3.0.2",
     "js-yaml": "^3.10.0",
-    "read-config-file": "2.1.1",
+    "read-config-file": "3.0.0",
     "minimatch": "^3.0.4",
     "normalize-package-data": "^2.4.0",
     "plist": "^2.1.0",

packages/electron-builder-lib/src/codeSign.ts

@@ -1,14 +1,15 @@
+import { appBuilderPath } from "app-builder-bin"
 import BluebirdPromise from "bluebird-lst"
 import { exec, getCacheDirectory, InvalidConfigurationError, isEmptyOrSpaces, isEnvTrue, isMacOsSierra, isPullRequest, log, TmpDir } from "builder-util"
 import { copyFile, statOrNull, unlinkIfExists } from "builder-util/out/fs"
 import { Fields, Logger } from "builder-util/out/log"
-import { httpExecutor } from "builder-util/out/nodeHttpExecutor"
 import { randomBytes } from "crypto"
 import { outputFile, rename } from "fs-extra-p"
 import { Lazy } from "lazy-val"
 import { homedir } from "os"
 import * as path from "path"
 import { getTempName } from "temp-file"
+import { download } from "builder-util/out/binDownload"
 import { isAutoDiscoveryCodeSignIdentity } from "./util/flags"
 
 export const appleCertificatePrefixes = ["Developer ID Application:", "Developer ID Installer:", "3rd Party Mac Developer Application:", "3rd Party Mac Developer Installer:"]
@@ -106,7 +107,7 @@ export async function downloadCertificate(urlOrBase64: string, tmpDir: TmpDir, c
     if (isUrl || urlOrBase64.length > 2048 || urlOrBase64.endsWith("=")) {
       const tempFile = await tmpDir.getTempFile({suffix: ".p12"})
       if (isUrl) {
-        await httpExecutor.download(urlOrBase64, tempFile)
+        await download(appBuilderPath, urlOrBase64, tempFile)
       }
       else {
         await outputFile(tempFile, Buffer.from(urlOrBase64, "base64"))

packages/electron-builder-lib/src/platformPackager.ts

@@ -1,3 +1,4 @@
+import { appBuilderPath } from "app-builder-bin"
 import { computeData, AsarIntegrity } from "asar-integrity"
 import BluebirdPromise from "bluebird-lst"
 import { Arch, asArray, AsyncTaskManager, debug, DebugLogger, exec, getArchSuffix, InvalidConfigurationError, isEmptyOrSpaces, log, deepAssign } from "builder-util"
@@ -18,7 +19,6 @@ import { AfterPackContext, AsarOptions, Configuration, FileAssociation, Platform
 import { Packager } from "./packager"
 import { unpackElectron, unpackMuon } from "./packager/dirPackager"
 import { PackagerOptions } from "./packagerApi"
-import { getAppBuilderTool } from "./targets/tools"
 import { copyAppFiles } from "./util/appFileCopier"
 import { computeFileSets, ELECTRON_COMPILE_SHIM_FILENAME } from "./util/AppFileCopierHelper"
 
@@ -586,7 +586,7 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
       arg.push("--input", source)
     }
 
-    const rawResult = await exec(await getAppBuilderTool(), arg, {
+    const rawResult = await exec(appBuilderPath, arg, {
       cwd: this.projectDir,
       env: {
         ...process.env,

packages/electron-builder-lib/src/targets/differentialUpdateInfoBuilder.ts

@@ -1,10 +1,10 @@
+import { appBuilderPath } from "app-builder-bin"
 import { exec, log } from "builder-util"
 import { BlockMapDataHolder, PackageFileInfo } from "builder-util-runtime"
 import * as path from "path"
 import { Target } from "../core"
 import { PlatformPackager } from "../platformPackager"
 import { ArchiveOptions } from "./archive"
-import { getAppBuilderTool } from "./tools"
 
 export const BLOCK_MAP_FILE_SUFFIX = ".blockmap"
 
@@ -61,13 +61,13 @@ export function configureDifferentialAwareArchiveOptions(archiveOptions: Archive
 
 export async function appendBlockmap(file: string): Promise<BlockMapDataHolder> {
   log.info({file: log.filePath(file)}, "building embedded block map")
-  return JSON.parse(await exec(await getAppBuilderTool(), ["blockmap", "--input", file, "--compression", "deflate"]))
+  return JSON.parse(await exec(appBuilderPath, ["blockmap", "--input", file, "--compression", "deflate"]))
 }
 
 export async function createBlockmap(file: string, target: Target, packager: PlatformPackager<any>, safeArtifactName: string | null): Promise<BlockMapDataHolder> {
   const blockMapFile = `${file}${BLOCK_MAP_FILE_SUFFIX}`
   log.info({blockMapFile: log.filePath(blockMapFile)}, "building block map")
-  const updateInfo: BlockMapDataHolder = JSON.parse(await exec(await getAppBuilderTool(), ["blockmap", "--input", file, "--output", blockMapFile]))
+  const updateInfo: BlockMapDataHolder = JSON.parse(await exec(appBuilderPath, ["blockmap", "--input", file, "--output", blockMapFile]))
   packager.info.dispatchArtifactCreated({
     file: blockMapFile,
     safeArtifactName: `${safeArtifactName}${BLOCK_MAP_FILE_SUFFIX}`,