feat: add proxy support (thanks @ChrisSG)

* Adds proxy support

* adds tests for downloadOptions and proxy

* sets tabsize to 2 spaces

* added https-proxy-agent dependency

* fixes linter errors

* adds types to new functions

* refactors setting https agent

* fixes bug for setting agent

* fixes linter errors

* revert: changes for MongoBinaryDownloadUrl

This class only provide URL where can be obtained binaries for current 
platform.

* refactor: MongoBinaryDownload now determine proxy settings from ENV vars

This class handle downloading, so better place for proxy setup is here.

* fix: small other fixes

* chore: update dependencies

.eslintrc

@@ -24,7 +24,9 @@
       "trailingComma": "es5",
     }],
     "no-prototype-builtins": 0,
-    "prefer-destructuring": 0
+    "prefer-destructuring": 0,
+    "no-else-return": 0,
+    "lines-between-class-members": ["error", "always", { exceptAfterSingleLine: true }],
   },
   "env": {
     "jasmine": true,

package.json

@@ -33,15 +33,15 @@
     "babel-plugin-transform-runtime": "^6.23.0",
     "babel-preset-env": "^1.7.0",
     "cz-conventional-changelog": "^2.1.0",
-    "eslint": "^4.19.1",
-    "eslint-config-airbnb-base": "^12.1.0",
+    "eslint": "^5.0.1",
+    "eslint-config-airbnb-base": "^13.0.0",
     "eslint-config-prettier": "^2.9.0",
     "eslint-plugin-flowtype": "^2.49.3",
     "eslint-plugin-import": "^2.12.0",
     "eslint-plugin-prettier": "^2.6.0",
-    "flow-bin": "^0.74.0",
+    "flow-bin": "^0.75.0",
     "jest": "^23.1.0",
-    "mongodb": "3.1.0-beta4",
+    "mongodb": "3.1.0",
     "npm-run-all": "^4.1.3",
     "prettier": "^1.13.5",
     "rimraf": "^2.6.2",
@@ -54,11 +54,10 @@
     "fs-extra": "^6.0.1",
     "get-port": "^3.2.0",
     "getos": "^3.1.0",
+    "https-proxy-agent": "^2.2.1",
     "lockfile": "^1.0.4",
     "md5-file": "^4.0.0",
     "mkdirp": "^0.5.1",
-    "request": "^2.87.0",
-    "request-promise": "^4.2.2",
     "tmp": "^0.0.33",
     "uuid": "^3.2.1"
   },

src/util/MongoBinaryDownload.js

@@ -2,11 +2,12 @@
 /* eslint-disable class-methods-use-this */
 
 import os from 'os';
+import url from 'url';
 import path from 'path';
 import fs from 'fs-extra';
-import request from 'request-promise';
 import md5File from 'md5-file';
 import https from 'https';
+import HttpsProxyAgent from 'https-proxy-agent';
 import decompress from 'decompress';
 import MongoBinaryDownloadUrl from './MongoBinaryDownloadUrl';
 
@@ -64,7 +65,7 @@ export default class MongoBinaryDownload {
       return mongodPath;
     }
 
-    const mongoDBArchive = await this.download();
+    const mongoDBArchive = await this.startDownload();
     await this.extract(mongoDBArchive);
     fs.unlinkSync(mongoDBArchive);
 
@@ -75,34 +76,66 @@ export default class MongoBinaryDownload {
     throw new Error(`Cannot find downloaded mongod binary by path ${mongodPath}`);
   }
 
-  async download(): Promise<string> {
+  async startDownload(): Promise<string> {
     const mbdUrl = new MongoBinaryDownloadUrl({
       platform: this.platform,
       arch: this.arch,
       version: this.version,
     });
 
     await fs.ensureDir(this.downloadDir);
-    const url = await mbdUrl.getDownloadUrl();
-    const archName = await mbdUrl.getArchiveName();
-    const downloadLocation = path.resolve(this.downloadDir, archName);
-    console.log('Downloading MongoDB:', url);
-    const tempDownloadLocation = path.resolve(this.downloadDir, `${archName}.downloading`);
-    const mongoDBArchive = await this.httpDownload(url, downloadLocation, tempDownloadLocation);
-    const md5Remote = await this.downloadMD5(`${url}.md5`);
+
+    const downloadUrl = await mbdUrl.getDownloadUrl();
+    const mongoDBArchive = await this.download(downloadUrl);
+
+    const mongoDBArchiveMd5 = await this.download(`${downloadUrl}.md5`);
+    await this.checkMd5(mongoDBArchiveMd5, mongoDBArchive);
+
+    return mongoDBArchive;
+  }
+
+  async checkMd5(mongoDBArchiveMd5: string, mongoDBArchive: string) {
+    const signatureContent = (await fs.readFile(mongoDBArchiveMd5)).toString('UTF-8');
+    const md5Remote = signatureContent.match(/(.*?)\s/)[1];
     const md5Local = md5File.sync(mongoDBArchive);
     if (md5Remote !== md5Local) {
       throw new Error('MongoBinaryDownload: md5 check is failed');
     }
-    return mongoDBArchive;
   }
 
-  async downloadMD5(md5url: string): Promise<string> {
-    const signatureContent = await request(md5url);
-    this.debug(`getDownloadMD5Hash content: ${signatureContent}`);
-    const signature = signatureContent.match(/(.*?)\s/)[1];
-    this.debug(`getDownloadMD5Hash extracted signature: ${signature}`);
-    return signature;
+  async download(downloadUrl: string) {
+    const proxy =
+      process.env['yarn_https-proxy'] ||
+      process.env.yarn_proxy ||
+      process.env['npm_config_https-proxy'] ||
+      process.env.npm_config_proxy ||
+      process.env.https_proxy ||
+      process.env.http_proxy;
+
+    const urlObject = url.parse(downloadUrl);
+
+    const downloadOptions = {
+      hostname: urlObject.hostname,
+      port: urlObject.port || 443,
+      path: urlObject.path,
+      method: 'GET',
+      agent: proxy ? new HttpsProxyAgent(proxy) : undefined,
+    };
+
+    const filename = (urlObject.pathname || '').split('/').pop();
+    if (!filename) {
+      throw new Error(`MongoBinaryDownload: missing filename for url ${downloadUrl}`);
+    }
+
+    const downloadLocation = path.resolve(this.downloadDir, filename);
+    const tempDownloadLocation = path.resolve(this.downloadDir, `${filename}.downloading`);
+    console.log(`Downloading${proxy ? ` via proxy ${proxy}` : ''}:`, downloadUrl);
+    const downloadedFile = await this.httpDownload(
+      downloadOptions,
+      downloadLocation,
+      tempDownloadLocation
+    );
+    return downloadedFile;
   }
 
   async extract(mongoDBArchive: string): Promise<string> {

src/util/MongoBinaryDownloadUrl.js

@@ -8,8 +8,6 @@ type OS = {
   release?: string,
 };
 
-const DOWNLOAD_URI = 'https://fastdl.mongodb.org';
-
 export type MongoBinaryDownloadUrlOpts = {
   version: string,
   platform: string,
@@ -32,7 +30,7 @@ export default class MongoBinaryDownloadUrl {
 
   async getDownloadUrl(): Promise<string> {
     const archive = await this.getArchiveName();
-    return `${DOWNLOAD_URI}/${this.platform}/${archive}`;
+    return `https://fastdl.mongodb.org/${this.platform}/${archive}`;
   }
 
   async getArchiveName(): Promise<string> {

src/util/__tests__/MongoBinary-test.js

@@ -4,7 +4,7 @@ import tmp from 'tmp';
 import MongoBinary from '../MongoBinary';
 
 tmp.setGracefulCleanup();
-jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000;
+jasmine.DEFAULT_TIMEOUT_INTERVAL = 160000;
 
 describe('MongoBinary', () => {
   it('should download binary and keep it in cache', async () => {

src/util/__tests__/MongoBinaryDownload-test.js

@@ -0,0 +1,37 @@
+/* @flow */
+
+import MongoBinaryDownload from '../MongoBinaryDownload';
+
+describe('MongoBinaryDownload', () => {
+  it('should use direct download', async () => {
+    process.env['yarn_https-proxy'] = '';
+    process.env.yarn_proxy = '';
+    process.env['npm_config_https-proxy'] = '';
+    process.env.npm_config_proxy = '';
+    process.env.https_proxy = '';
+    process.env.http_proxy = '';
+
+    const du = new MongoBinaryDownload({});
+    // $FlowFixMe
+    du.httpDownload = jest.fn();
+
+    await du.download('https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.6.3.tgz');
+    expect(du.httpDownload).toHaveBeenCalledTimes(1);
+    const callArg1 = du.httpDownload.mock.calls[0][0];
+    expect(callArg1.agent).toBeUndefined();
+  });
+
+  it('should pick up proxy from env vars', async () => {
+    process.env['yarn_https-proxy'] = 'http://user:pass@proxy:8080';
+
+    const du = new MongoBinaryDownload({});
+    // $FlowFixMe
+    du.httpDownload = jest.fn();
+
+    await du.download('https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.6.3.tgz');
+    expect(du.httpDownload).toHaveBeenCalledTimes(1);
+    const callArg1 = du.httpDownload.mock.calls[0][0];
+    expect(callArg1.agent).toBeDefined();
+    expect(callArg1.agent.options.href).toBe('http://user:pass@proxy:8080/');
+  });
+});