From 130b1bdfaab40637cc0e913fb3f97871ce99613c Mon Sep 17 00:00:00 2001 From: Rick Waldron Date: Thu, 20 Oct 2016 15:39:04 -0400 Subject: [PATCH] Check if remote hosts are reachable before making requests. Fixes gh-1051 Fails with error that indicates the issue --- lib/crash-reporter.js | 47 +++++----- lib/install/rust.js | 50 +++++----- lib/remote.js | 24 +++++ lib/tessel/deployment/javascript.js | 29 +++--- lib/tessel/deployment/rust.js | 138 ++++++++++++++-------------- lib/update-fetch.js | 98 ++++++++++---------- test/.jshintrc | 4 +- test/common/bootstrap.js | 4 +- test/unit/deployment/javascript.js | 2 + test/unit/deployment/rust.js | 1 + test/unit/remote.js | 65 +++++++++++++ test/unit/update.js | 3 +- 12 files changed, 289 insertions(+), 176 deletions(-) create mode 100644 lib/remote.js create mode 100644 test/unit/remote.js diff --git a/lib/crash-reporter.js b/lib/crash-reporter.js index dc076973..7fc92705 100644 --- a/lib/crash-reporter.js +++ b/lib/crash-reporter.js @@ -10,6 +10,7 @@ var log = require('./log'); var Menu = require('./menu'); var packageJson = require('../package.json'); var Preferences = require('./preferences'); +var remote = require('./remote'); // the value of the crash reporter preference // the value has to be one of 'on' or 'off' @@ -18,7 +19,7 @@ var CRASH_REPORTER_PROMPT = 'crash.reporter.prompt'; var CRASH_PROMPT_MESSAGE = `\nSubmit Crash Report to help Tessel Developers improve the CLI ? If yes(y), subsequent crashes will be submitted automatically.`; -var CRASH_REPORTER_BASE_URL = 'http://crash-reporter.tessel.io'; +var CRASH_REPORTER_BASE_URL = `http://${remote.CRASH_REPORTER_HOSTNAME}`; // override for testing if (process.env.DEV_MODE === 'true') { @@ -149,30 +150,32 @@ CrashReporter.post = function(labels, crash, argv) { var url = SUBMIT_CRASH_URL; return new Promise((resolve, reject) => { - request.post({ - url, - form: { - argv, - crash, - labels, - f - } - }, (error, httpResponse, body) => { - try { - if (error) { - reject(error); - } else { - var json = JSON.parse(body); - if (json.error) { - reject(json.error); + remote.ifReachable(remote.CRASH_REPORTER_HOSTNAME).then(() => { + request.post({ + url, + form: { + argv, + crash, + labels, + f + } + }, (error, httpResponse, body) => { + try { + if (error) { + reject(error); } else { - resolve(json.crash_report.fingerprint); + var json = JSON.parse(body); + if (json.error) { + reject(json.error); + } else { + resolve(json.crash_report.fingerprint); + } } + } catch (exception) { + reject(exception); } - } catch (exception) { - reject(exception); - } - }); + }); + }).catch(reject); }); }; diff --git a/lib/install/rust.js b/lib/install/rust.js index 0cc3c19b..0c09084d 100644 --- a/lib/install/rust.js +++ b/lib/install/rust.js @@ -20,19 +20,17 @@ var tar = require('tar-fs'); // Internal var log = require('../log'); +var remote = require('../remote'); -var SDK_PATHS = { +const SDK_PATHS = { sdk: path.join(osenv.home(), '.tessel/sdk'), rustlib: path.join(osenv.home(), '.tessel/rust'), }; - -var SDK_URLS = { - macos: 'https://builds.tessel.io/t2/sdk/t2-sdk-macos-x86_64.tar.bz2', - linux: 'https://builds.tessel.io/t2/sdk/t2-sdk-linux-x86_64.tar.bz2', +const SDK_URLS = { + macos: `https://${remote.BUILDS_HOSTNAME}/t2/sdk/t2-sdk-macos-x86_64.tar.bz2`, + linux: `https://${remote.BUILDS_HOSTNAME}/t2/sdk/t2-sdk-linux-x86_64.tar.bz2`, }; -var RUST_LIB_TGZ_URL = 'https://builds.tessel.io/t2/sdk/t2-rustlib-VERSION.tar.gz'; - // Get the platform identifier. This actually conforms to the list of OSes // Rust supports, not the value of process.platform, so we need to convert it. // See: https://doc.rust-lang.org/std/env/consts/constant.OS.html @@ -186,31 +184,33 @@ module.exports.installTools = () => { var url = SDK_URLS[getPlatform()]; var checksumVerify = null; - return downloadString(`${url}.sha256`) - .then((checksum) => { - checksumVerify = checksum; - return exports.checkTools(checksumVerify); - }) - .then((check) => { - if (check.exists && check.isVerified) { - log.info(`Latest ${pkgname} already installed.`); - return; - } else if (!check.exists) { - log.info(`Installing ${pkgname}...`); - } else { - log.info(`Updating ${pkgname}...`); - } + return remote.ifReachable(remote.BUILDS_HOSTNAME).then(() => { + return downloadString(`${url}.sha256`) + .then((checksum) => { + checksumVerify = checksum; + return exports.checkTools(checksumVerify); + }) + .then((check) => { + if (check.exists && check.isVerified) { + log.info(`Latest ${pkgname} already installed.`); + return; + } else if (!check.exists) { + log.info(`Installing ${pkgname}...`); + } else { + log.info(`Updating ${pkgname}...`); + } - fs.mkdirpSync(path.join(osenv.home(), '.tessel/sdk')); - return extractTools(checksumVerify, path.basename(url), download(url)); - }); + fs.mkdirpSync(path.join(osenv.home(), '.tessel/sdk')); + return extractTools(checksumVerify, path.basename(url), download(url)); + }); + }); }; module.exports.installRustlib = () => { return exports.rustVersion() .then((rustv) => { var pkgname = `MIPS libstd v${rustv}`; - var url = RUST_LIB_TGZ_URL.replace('VERSION', rustv); + var url = `https://${remote.BUILDS_HOSTNAME}/t2/sdk/t2-rustlib-${rustv}.tar.gz`; var checksumVerify; return downloadString(url + '.sha256') diff --git a/lib/remote.js b/lib/remote.js new file mode 100644 index 00000000..70f45bc8 --- /dev/null +++ b/lib/remote.js @@ -0,0 +1,24 @@ +var dns = require('dns'); + +const remote = { + CRASH_REPORTER_HOSTNAME: 'crash-reporter.tessel.io', + BUILDS_HOSTNAME: 'builds.tessel.io', + PACKAGES_HOSTNAME: 'packages.tessel.io', + RUSTCC_HOSTNAME: 'rustcc.tessel.io', + + ifReachable(url) { + return new Promise((resolve, reject) => { + dns.lookup(url, error => { + if (error) { + reject(new Error('This operation requires an internet connection')); + } else { + resolve(); + } + }); + }); + }, +}; + + + +module.exports = remote; diff --git a/lib/tessel/deployment/javascript.js b/lib/tessel/deployment/javascript.js index 32d5e14b..c391a7fa 100644 --- a/lib/tessel/deployment/javascript.js +++ b/lib/tessel/deployment/javascript.js @@ -25,12 +25,13 @@ var lists = require('./lists/javascript'); var log = require('../../log'); // Necessary to ensure that the next line has had the LOCAL_AUTH_PATH descriptor added. var provision = require('../provision'); // jshint ignore:line +var remote = require('../../remote'); var Tessel = require('../tessel'); var binaryModulesUsed = new Map(); var isWindows = process.platform.startsWith('win'); -const BINARY_SERVER_ROOT = 'http://packages.tessel.io/npm/'; +const BINARY_SERVER_ROOT = `http://${remote.PACKAGES_HOSTNAME}/npm/`; const BINARY_CACHE_PATH = path.join(Tessel.LOCAL_AUTH_PATH, 'binaries'); var exportables = { @@ -295,18 +296,20 @@ exportables.resolveBinaryModules = function(opts) { } }); - request({ - url: url, - gzip: true, - }) - .pipe(gunzip) - .pipe(extract) - .on('error', reject) - .on('end', () => { - // Once complete, the locally cached binary can - // be found in ~/.tessel/binaries - resolve(); - }); + remote.ifReachable(remote.PACKAGES_HOSTNAME).then(() => { + request({ + url: url, + gzip: true, + }) + .pipe(gunzip) + .pipe(extract) + .on('error', reject) + .on('end', () => { + // Once complete, the locally cached binary can + // be found in ~/.tessel/binaries + resolve(); + }); + }).catch(reject); }); } }); diff --git a/lib/tessel/deployment/rust.js b/lib/tessel/deployment/rust.js index cb1daf9c..4c4bf0b3 100644 --- a/lib/tessel/deployment/rust.js +++ b/lib/tessel/deployment/rust.js @@ -16,6 +16,7 @@ var Reader = fstream.Reader; var commands = require('../commands'); var lists = require('./lists/rust'); var log = require('../../log'); +var remote = require('../../remote'); var rust = require('../../install/rust'); var Tessel = require('../tessel'); @@ -101,7 +102,7 @@ exportables.preRun = function(tessel, opts) { // jshint ignore:line exportables.remoteRustCompilation = (opts) => { log.info('Compiling Rust code remotely (--rustcc)...'); - return new Promise(function(resolve, reject) { + return new Promise((resolve, reject) => { // Save our incoming compiled buffer to this array var buffers = []; @@ -110,7 +111,7 @@ exportables.remoteRustCompilation = (opts) => { if (!opts.rustcc.startsWith('http')) { // Without an HTTP protocol definition, url.parse will fail; // assume http:// was implied - opts.rustcc = 'http://' + opts.rustcc; + opts.rustcc = `http://${opts.rustcc}`; } var destination = url.parse(opts.rustcc); @@ -125,74 +126,77 @@ exportables.remoteRustCompilation = (opts) => { } }; - // Set up the request - var req = http.request(options, (res) => { - // When we get incoming binary data, save to our buffers - res.on('data', function(chunk) { - // Write this incoming data to our buffer collection - buffers.push(chunk); + remote.ifReachable(destination.hostname).then(() => { + // Set up the request + var req = http.request(options, (res) => { + // When we get incoming binary data, save to our buffers + res.on('data', function(chunk) { + // Write this incoming data to our buffer collection + buffers.push(chunk); + }) + // Reject on failure + .on('error', function(postReqError) { + return reject(postReqError); + }) + // When the post completes, resolve with the executable + .on('end', function() { + // Parse the incoming data as JSON + var result; + try { + result = JSON.parse(Buffer.concat(buffers)); + } catch (e) { + return reject(tags.stripIndent ` + Please file an issue on https://github.com/tessel/t2-cli with the following: + You received an invalid JSON response from the cross-compilation server. + ${Buffer.concat(buffers).toString()}`); + } + + // Check if there was an error message written + if (result.error !== undefined && result.error !== '') { + // If there was, abort with the provided message + return reject(new Error(result.error)); + } + + // Print out any stderr output + else if (result.stderr !== null && result.stderr !== '') { + log.info(result.stderr); + } + + // Print out any stdout output + /* istanbul ignore else */ + if (result.stdout !== null) { + log.info(result.stdout); + } + + // If the binary was not provided + if (result.binary === null) { + // Reject with an error + return reject(new Error('Neither binary nor error returned by cross compilation server.')); + } + // If the binary was provided + else { + // All was successful and we can return + return resolve(new Buffer(result.binary, 'base64')); + } + }); + }); + + // Create an outgoing tar packer for our project + var outgoingPacker = tar.Pack({ + noProprietary: true }) - // Reject on failure - .on('error', function(postReqError) { - return reject(postReqError); + .on('error', reject); + + // Send the project directory through the tar packer and into the post request + Reader({ + path: opts.target, + type: 'Directory' }) - // When the post completes, resolve with the executable - .on('end', function() { - // Parse the incoming data as JSON - var result; - try { - result = JSON.parse(Buffer.concat(buffers)); - } catch (e) { - return reject(tags.stripIndent ` - Please file an issue on https://github.com/tessel/t2-cli with the following: - You received an invalid JSON response from the cross-compilation server. - ${Buffer.concat(buffers).toString()}`); - } - - // Check if there was an error message written - if (result.error !== undefined && result.error !== '') { - // If there was, abort with the provided message - return reject(new Error(result.error)); - } - - // Print out any stderr output - else if (result.stderr !== null && result.stderr !== '') { - log.info(result.stderr); - } - - // Print out any stdout output - /* istanbul ignore else */ - if (result.stdout !== null) { - log.info(result.stdout); - } - - // If the binary was not provided - if (result.binary === null) { - // Reject with an error - return reject(new Error('Neither binary nor error returned by cross compilation server.')); - } - // If the binary was provided - else { - // All was successful and we can return - return resolve(new Buffer(result.binary, 'base64')); - } - }); - }); - - // Create an outgoing tar packer for our project - var outgoingPacker = tar.Pack({ - noProprietary: true - }) - .on('error', reject); + .on('error', reject) + .pipe(outgoingPacker) + .pipe(req); - // Send the project directory through the tar packer and into the post request - Reader({ - path: opts.target, - type: 'Directory' - }) - .on('error', reject) - .pipe(outgoingPacker) - .pipe(req); + }).catch(reject); }); }; diff --git a/lib/update-fetch.js b/lib/update-fetch.js index 9e7b71bc..2806fc60 100644 --- a/lib/update-fetch.js +++ b/lib/update-fetch.js @@ -13,8 +13,9 @@ var semver = require('semver'); // Internal var log = require('./log'); +var remote = require('./remote'); -const BUILD_SERVER_ROOT = 'https://builds.tessel.io/t2'; +const BUILD_SERVER_ROOT = `https://${remote.BUILDS_HOSTNAME}/t2`; const FIRMWARE_PATH = urljoin(BUILD_SERVER_ROOT, 'firmware'); const BUILDS_JSON_FILE = urljoin(FIRMWARE_PATH, 'builds.json'); const OPENWRT_BINARY_FILE = 'openwrt.bin'; @@ -38,33 +39,35 @@ var exportables = { */ exportables.requestBuildList = function() { return new Promise((resolve, reject) => { - // Fetch the list of available builds - request.get(BUILDS_JSON_FILE, (err, response, body) => { - if (err) { - return reject(err); - } - - var outcome = exportables.reviewResponse(response); - var builds; - // If there wasn't an issue with the request - if (outcome.success) { - // Resolve with the parsed data - try { - builds = JSON.parse(body); + return remote.ifReachable(remote.BUILDS_HOSTNAME).then(() => { + // Fetch the list of available builds + request.get(BUILDS_JSON_FILE, (err, response, body) => { + if (err) { + return reject(err); } - // If the parse failed, reject - catch (err) { - reject(err); - } - - // Sort the builds by semver version in chronological order - builds.sort((a, b) => semver.compare(a.version, b.version)); - return resolve(builds); - } else { - reject(outcome.reason); - } - }); + var outcome = exportables.reviewResponse(response); + var builds; + // If there wasn't an issue with the request + if (outcome.success) { + // Resolve with the parsed data + try { + builds = JSON.parse(body); + } + // If the parse failed, reject + catch (err) { + reject(err); + } + + // Sort the builds by semver version in chronological order + builds.sort((a, b) => semver.compare(a.version, b.version)); + + return resolve(builds); + } else { + reject(outcome.reason); + } + }); + }).catch(reject); }); }; @@ -160,31 +163,34 @@ exportables.downloadTgz = function(tgzUrl, fileMap) { return resolve(files); }); - var req = request.get(tgzUrl); - // When we receive the response - req.on('response', (res) => { + remote.ifReachable(remote.BUILDS_HOSTNAME).then(() => { + var req = request.get(tgzUrl); - // Parse out the length of the incoming bundle - var contentLength = parseInt(res.headers['content-length'], 10); + // When we receive the response + req.on('response', (res) => { - // Create a new progress bar - var bar = new Progress(' [:bar] :percent :etas remaining', { - clear: true, - complete: '=', - incomplete: ' ', - width: 20, - total: contentLength - }); + // Parse out the length of the incoming bundle + var contentLength = parseInt(res.headers['content-length'], 10); - // When we get incoming data, update the progress bar - res.on('data', (chunk) => { - bar.tick(chunk.length); - }); + // Create a new progress bar + var bar = new Progress(' [:bar] :percent :etas remaining', { + clear: true, + complete: '=', + incomplete: ' ', + width: 20, + total: contentLength + }); - // unzip and extract the binary tarball - res.pipe(gunzip).pipe(extract); - }); + // When we get incoming data, update the progress bar + res.on('data', (chunk) => { + bar.tick(chunk.length); + }); + + // unzip and extract the binary tarball + res.pipe(gunzip).pipe(extract); + }); + }).catch(reject); }); }; diff --git a/test/.jshintrc b/test/.jshintrc index 4cbc56ba..e63a65a7 100644 --- a/test/.jshintrc +++ b/test/.jshintrc @@ -42,6 +42,7 @@ "deployment": true, "deployTestCode": true, "discover": true, + "dns": true, "document": true, "Duplex": true, "Emitter": true, @@ -79,11 +80,12 @@ "Promise": true, "processVersions": true, "provision": true, - "restore": true, "reference": true, + "remote": true, "RemoteProcessSimulator": true, "request": true, "Request": true, + "restore": true, "RSA": true, "rust": true, "sandbox": true, diff --git a/test/common/bootstrap.js b/test/common/bootstrap.js index bc5c42f5..4f0d2e19 100644 --- a/test/common/bootstrap.js +++ b/test/common/bootstrap.js @@ -2,6 +2,7 @@ global.IS_TEST_ENV = true; // System Objects global.cp = require('child_process'); +global.dns = require('dns'); global.events = require('events'); global.http = require('http'); global.os = require('os'); @@ -63,8 +64,9 @@ global.discover = require('../../lib/discover'); global.init = require('../../lib/init'); global.installer = require('../../lib/installer'); global.log = require('../../lib/log'); -global.updates = require('../../lib/update-fetch'); global.lan = require('../../lib/lan-connection'); +global.remote = require('../../lib/remote'); +global.updates = require('../../lib/update-fetch'); global.usb = require('../../lib/usb-connection'); // ./lib/install/* diff --git a/test/unit/deployment/javascript.js b/test/unit/deployment/javascript.js index 97b65519..2a87165d 100644 --- a/test/unit/deployment/javascript.js +++ b/test/unit/deployment/javascript.js @@ -1976,6 +1976,8 @@ exports['deployment.js.resolveBinaryModules'] = { return path.normalize(`node_modules/${pathPart}/`); }); + this.ifReachable = sandbox.stub(remote, 'ifReachable', () => Promise.resolve()); + done(); }, diff --git a/test/unit/deployment/rust.js b/test/unit/deployment/rust.js index 68875106..8f98c988 100644 --- a/test/unit/deployment/rust.js +++ b/test/unit/deployment/rust.js @@ -14,6 +14,7 @@ exports['deploy.rust'] = { this.outgoingResponse._read = () => {}; + this.ifReachable = sandbox.stub(remote, 'ifReachable').returns(Promise.resolve()); this.httpRequest = sandbox.stub(http, 'request', (options, cb) => { setImmediate(() => cb(this.outgoingResponse)); return this.incomingRequest; diff --git a/test/unit/remote.js b/test/unit/remote.js new file mode 100644 index 00000000..76f34cc1 --- /dev/null +++ b/test/unit/remote.js @@ -0,0 +1,65 @@ +// Test dependencies are required and exposed in common/bootstrap.js +require('../common/bootstrap'); + + +exports['remote.* consts'] = { + setUp(done) { + done(); + }, + + tearDown(done) { + done(); + }, + + all(test) { + test.expect(4); + test.equal(remote.CRASH_REPORTER_HOSTNAME, 'crash-reporter.tessel.io'); + test.equal(remote.BUILDS_HOSTNAME, 'builds.tessel.io'); + test.equal(remote.PACKAGES_HOSTNAME, 'packages.tessel.io'); + test.equal(remote.RUSTCC_HOSTNAME, 'rustcc.tessel.io'); + test.done(); + }, +}; + +exports['remote.ifReachable(...)'] = { + setUp(done) { + this.sandbox = sinon.sandbox.create(); + this.logWarn = this.sandbox.stub(log, 'warn', () => {}); + this.logInfo = this.sandbox.stub(log, 'info', () => {}); + this.logBasic = this.sandbox.stub(log, 'basic', () => {}); + + this.error = null; + this.lookup = this.sandbox.stub(dns, 'lookup', (hostname, handler) => { + handler(this.error); + }); + done(); + }, + + tearDown(done) { + this.sandbox.restore(); + done(); + }, + + success(test) { + test.expect(2); + remote.ifReachable('foo.bar.baz').then(() => { + test.equal(this.lookup.callCount, 1); + test.equal(this.lookup.lastCall.args[0], 'foo.bar.baz'); + test.done(); + }); + }, + + failure(test) { + test.expect(3); + this.error = new Error('ENOTFOUND'); + remote.ifReachable('foo.bar.baz').then(() => { + test.ok(false, 'This should not have been reached.'); + test.done(); + }).catch(error => { + test.equal(error.message, 'This operation requires an internet connection'); + test.equal(this.lookup.callCount, 1); + test.equal(this.lookup.lastCall.args[0], 'foo.bar.baz'); + test.done(); + }); + }, +}; diff --git a/test/unit/update.js b/test/unit/update.js index e290af81..8453f0c3 100644 --- a/test/unit/update.js +++ b/test/unit/update.js @@ -38,7 +38,6 @@ exports['controller.update'] = { this.updateTesselWithVersion = this.sandbox.spy(controller, 'updateTesselWithVersion'); this.closeTesselConnections = this.sandbox.spy(controller, 'closeTesselConnections'); - done(); }, @@ -784,6 +783,8 @@ exports['update-fetch'] = { statusCode: 200 }, JSON.stringify(mixedBuilds)); }); + + this.ifReachable = this.sandbox.stub(remote, 'ifReachable', () => Promise.resolve()); done(); }, tearDown: function(done) {