From 02f26723394e158d7328d6ea39cb5592eddca183 Mon Sep 17 00:00:00 2001 From: Lucas Hrabovsky Date: Thu, 21 Jul 2016 15:59:18 -0400 Subject: [PATCH 1/6] INT-1650 :bug: :memo: Update tunnel-ssh config for their latest release --- lib/model.js | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/lib/model.js b/lib/model.js index e9133388..77230979 100644 --- a/lib/model.js +++ b/lib/model.js @@ -442,42 +442,42 @@ assign(props, { default: SSH_TUNNEL_DEFAULT }, /** - * The hostname to open the ssh tunnel to. + * The hostname of the SSH remote host. */ ssh_tunnel_hostname: { type: 'string', default: undefined }, /** - * The local port for the ssh tunnel. + * The SSH port of the remote host. */ ssh_tunnel_port: { type: 'number', default: 22 }, /** - * The ssh username. + * The optional SSH username for the remote host. */ ssh_tunnel_username: { type: 'string', default: undefined }, /** - * The ssh password. + * The optional SSH password for the remote host. */ ssh_tunnel_password: { type: 'string', default: undefined }, /** - * The path to the ssh identity file. + * The optional path to the SSH identity file for the remote host. */ ssh_tunnel_identity_file: { type: 'any', default: undefined }, /** - * The passphrase for the identity file. + * The optional passphrase for `ssh_tunnel_identity_file`. */ ssh_tunnel_passphrase: { type: 'string', @@ -506,7 +506,7 @@ var DRIVER_OPTIONS_DEFAULT = { assign(derived, { /** - * Get the URL which can be passed to `MongoClient.connect`. + * Get the URL which can be passed to `MongoClient.connect(url)`. * @see http://bit.ly/mongoclient-connect * @return {String} */ @@ -535,11 +535,6 @@ assign(derived, { } }; - if (this.ssh_tunnel !== 'NONE') { - req.hostname = 'localhost'; - req.port = this.ssh_tunnel_port; - } - if (this.ns) { req.pathname = format('/%s', this.ns); } @@ -622,6 +617,9 @@ assign(derived, { return opts; } }, + /** + * @return {Object} The options passed to http://npm.im/tunnel-ssh + */ ssh_tunnel_options: { deps: [ 'ssh_tunnel', @@ -636,16 +634,21 @@ assign(derived, { if (this.ssh_tunnel === 'NONE') { return {}; } + var opts = { - dstHost: this.hostname, + dstHost: this.ssh_tunnel_hostname, dstPort: this.port, - username: this.ssh_tunnel_username, + localPort: this.port, + localHost: this.hostname, host: this.ssh_tunnel_hostname, - sshPort: this.ssh_tunnel_port + port: this.ssh_tunnel_port, + username: this.ssh_tunnel_username }; + if (this.ssh_tunnel === 'USER_PASSWORD') { assign(opts, { password: this.ssh_tunnel_password }); } else if (this.ssh_tunnel === 'IDENTITY_FILE') { + /* eslint no-sync: 0 */ assign(opts, { privateKey: fs.readFileSync(this.ssh_tunnel_identity_file[0]) }); @@ -662,7 +665,7 @@ assign(derived, { * An ampersand.js model to represent a connection to a MongoDB database. * It does not actually talk to MongoDB. It is just a higher-level * abstraction that prepares arguments for `MongoClient.connect`. -**/ + */ Connection = AmpersandModel.extend({ namespace: 'Connection', idAttribute: 'instance_id', From 601e9cc7ccd94f850dfb6f89e43777c754e1f357 Mon Sep 17 00:00:00 2001 From: Lucas Hrabovsky Date: Mon, 25 Jul 2016 09:52:04 -0400 Subject: [PATCH 2/6] :green_heart: Fix ssh tunnel tests --- test/index.test.js | 46 ++++++++++------------------------------------ 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/test/index.test.js b/test/index.test.js index cb60771b..f0ed596f 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -538,20 +538,6 @@ describe('mongodb-connection-model', function() { }); describe('ssh tunnel', function() { - describe('#driver_url', function() { - var c = new Connection({ - hostname: '127.0.0.1', - ssh_tunnel: 'USER_PASSWORD', - ssh_tunnel_hostname: '127.0.0.1', - ssh_tunnel_port: 5000, - ssh_tunnel_username: 'username', - ssh_tunnel_password: 'password' - }); - it('replaces the host and port with localhost and the ssh tunnel port', function() { - assert.equal(c.driver_url, 'mongodb://localhost:5000/?slaveOk=true'); - }); - }); - describe('#ssh_tunnel_options', function() { context('when ssh_tunnel is NONE', function() { var c = new Connection({ @@ -575,24 +561,20 @@ describe('mongodb-connection-model', function() { ssh_tunnel_password: 'password' }).ssh_tunnel_options; - it('maps ssh_tunnel_hostname -> host', function() { - assert.equal(options.host, 'my.ssh-server.com'); - }); - it('maps ssh_tunnel_username -> username', function() { assert.equal(options.username, 'my-user'); }); - it('maps hostname -> dstHost', function() { - assert.equal(options.dstHost, 'mongodb.my-internal-host.com'); + it('maps ssh_tunnel_hostname -> dstHost', function() { + assert.equal(options.dstHost, 'my.ssh-server.com'); }); it('maps port -> dstPort', function() { assert.equal(options.dstPort, 27000); }); - it('maps ssh_tunnel_port -> sshPort', function() { - assert.equal(options.sshPort, 3000); + it('maps ssh_tunnel_port -> port', function() { + assert.equal(options.port, 3000); }); it('maps ssh_tunnel_password -> password', function() { @@ -614,10 +596,6 @@ describe('mongodb-connection-model', function() { ssh_tunnel_passphrase: 'password' }).ssh_tunnel_options; - it('maps ssh_tunnel_hostname -> host', function() { - assert.equal(options.host, 'my.ssh-server.com'); - }); - it('maps ssh_tunnel_username -> username', function() { assert.equal(options.username, 'my-user'); }); @@ -627,15 +605,15 @@ describe('mongodb-connection-model', function() { }); it('maps hostname -> dstHost', function() { - assert.equal(options.dstHost, 'mongodb.my-internal-host.com'); + assert.equal(options.dstHost, 'my.ssh-server.com'); }); it('maps port -> dstPort', function() { assert.equal(options.dstPort, 27000); }); - it('maps ssh_tunnel_port -> sshPort', function() { - assert.equal(options.sshPort, 3000); + it('maps ssh_tunnel_port -> port', function() { + assert.equal(options.port, 3000); }); it('maps ssh_tunnel_passphrase -> password', function() { @@ -655,10 +633,6 @@ describe('mongodb-connection-model', function() { ssh_tunnel_port: 3000 }).ssh_tunnel_options; - it('maps ssh_tunnel_hostname -> host', function() { - assert.equal(options.host, 'my.ssh-server.com'); - }); - it('maps ssh_tunnel_username -> username', function() { assert.equal(options.username, 'my-user'); }); @@ -667,8 +641,8 @@ describe('mongodb-connection-model', function() { assert.equal(options.privateKey.toString(), fs.readFileSync(fileName).toString()); }); - it('maps hostname -> dstHost', function() { - assert.equal(options.dstHost, 'mongodb.my-internal-host.com'); + it('maps ssh_tunnel_hostname -> dstHost', function() { + assert.equal(options.dstHost, 'my.ssh-server.com'); }); it('maps port -> dstPort', function() { @@ -676,7 +650,7 @@ describe('mongodb-connection-model', function() { }); it('maps ssh_tunnel_port -> sshPort', function() { - assert.equal(options.sshPort, 3000); + assert.equal(options.port, 3000); }); }); }); From 317e3ed6fcea89a51ca36832ba8b5cd8da5f1b96 Mon Sep 17 00:00:00 2001 From: Lucas Hrabovsky Date: Mon, 25 Jul 2016 11:55:25 -0400 Subject: [PATCH 3/6] INT-1650: Add `port` dataType - Move ssh_tunnel tests to their own module - Add `port` dataType that is much easier to use with form inputs and handles all casting & validation in a single code path - Add more tests --- lib/data-types.js | 26 ++++ lib/model.js | 14 ++- package.json | 2 +- test/index.test.js | 247 -------------------------------------- test/ssh-tunnel.test.js | 258 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 295 insertions(+), 252 deletions(-) create mode 100644 lib/data-types.js create mode 100644 test/ssh-tunnel.test.js diff --git a/lib/data-types.js b/lib/data-types.js new file mode 100644 index 00000000..1f8c5c91 --- /dev/null +++ b/lib/data-types.js @@ -0,0 +1,26 @@ +module.exports = { + port: { + set: function(newVal) { + var port = parseInt(newVal, 10); + if (newVal === '' || isNaN(port)) { + return { + type: 'undefined', + val: undefined + }; + } + + if (port < 0) { + throw new TypeError('port number must be positive.'); + } + + if (port >= 65536) { + throw new TypeError('port number must be below 65536'); + } + + return { + type: 'port', + val: newVal + }; + } + } +}; diff --git a/lib/model.js b/lib/model.js index 77230979..10ada984 100644 --- a/lib/model.js +++ b/lib/model.js @@ -7,6 +7,7 @@ var defaults = require('lodash.defaults'); var contains = require('lodash.contains'); var clone = require('lodash.clone'); var parse = require('mongodb-url'); +var dataTypes = require('./data-types'); var fs = require('fs'); // var debug = require('debug')('mongodb-connection-model'); @@ -36,7 +37,7 @@ assign(props, { default: 'localhost' }, port: { - type: 'number', + type: 'port', default: 27017 }, ns: { @@ -452,7 +453,7 @@ assign(props, { * The SSH port of the remote host. */ ssh_tunnel_port: { - type: 'number', + type: 'port', default: 22 }, /** @@ -646,14 +647,18 @@ assign(derived, { }; if (this.ssh_tunnel === 'USER_PASSWORD') { - assign(opts, { password: this.ssh_tunnel_password }); + assign(opts, { + password: this.ssh_tunnel_password + }); } else if (this.ssh_tunnel === 'IDENTITY_FILE') { /* eslint no-sync: 0 */ assign(opts, { privateKey: fs.readFileSync(this.ssh_tunnel_identity_file[0]) }); if (this.ssh_tunnel_passphrase) { - assign(opts, { password: this.ssh_tunnel_passphrase }); + assign(opts, { + password: this.ssh_tunnel_passphrase + }); } } return opts; @@ -671,6 +676,7 @@ Connection = AmpersandModel.extend({ idAttribute: 'instance_id', props: props, derived: derived, + dataTypes: dataTypes, initialize: function(attrs) { if (attrs) { if (typeof attrs === 'string') { diff --git a/package.json b/package.json index 6345399c..6cd9045b 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "scripts": { "ci": "npm run check && npm test", "test": "mocha", - "fmt": "mongodb-js-fmt ./*.js ./test/*.js", + "fmt": "mongodb-js-fmt ./*.js lib/*.js ./test/*.js", "check": "mongodb-js-precommit" }, "precommit": [ diff --git a/test/index.test.js b/test/index.test.js index f0ed596f..aed232c8 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -6,8 +6,6 @@ var driverParse = require('mongodb/lib/url_parser'); var fixture = require('mongodb-connection-fixture'); var clone = require('lodash.clone'); var format = require('util').format; -var fs = require('fs'); -var path = require('path'); function isNotValidAndHasMessage(model, msg) { assert.equal(model.isValid(), false, @@ -536,249 +534,4 @@ describe('mongodb-connection-model', function() { }); }); }); - - describe('ssh tunnel', function() { - describe('#ssh_tunnel_options', function() { - context('when ssh_tunnel is NONE', function() { - var c = new Connection({ - hostname: '127.0.0.1', - ssh_tunnel: 'NONE' - }); - - it('returns an empty object', function() { - assert.equal(c.ssh_tunnel_options.hostname, null); - }); - }); - - context('when ssh_tunnel is USER_PASSWORD', function() { - var options = new Connection({ - ssh_tunnel: 'USER_PASSWORD', - ssh_tunnel_hostname: 'my.ssh-server.com', - ssh_tunnel_username: 'my-user', - hostname: 'mongodb.my-internal-host.com', - port: 27000, - ssh_tunnel_port: 3000, - ssh_tunnel_password: 'password' - }).ssh_tunnel_options; - - it('maps ssh_tunnel_username -> username', function() { - assert.equal(options.username, 'my-user'); - }); - - it('maps ssh_tunnel_hostname -> dstHost', function() { - assert.equal(options.dstHost, 'my.ssh-server.com'); - }); - - it('maps port -> dstPort', function() { - assert.equal(options.dstPort, 27000); - }); - - it('maps ssh_tunnel_port -> port', function() { - assert.equal(options.port, 3000); - }); - - it('maps ssh_tunnel_password -> password', function() { - assert.equal(options.password, 'password'); - }); - }); - - context('when ssh_tunnel is IDENTITY_FILE', function() { - context('when a passphrase exists', function() { - var fileName = path.join(__dirname, 'fake-identity-file.txt'); - var options = new Connection({ - ssh_tunnel: 'IDENTITY_FILE', - ssh_tunnel_hostname: 'my.ssh-server.com', - ssh_tunnel_username: 'my-user', - ssh_tunnel_identity_file: [fileName], - hostname: 'mongodb.my-internal-host.com', - port: 27000, - ssh_tunnel_port: 3000, - ssh_tunnel_passphrase: 'password' - }).ssh_tunnel_options; - - it('maps ssh_tunnel_username -> username', function() { - assert.equal(options.username, 'my-user'); - }); - - it('maps ssh_tunnel_identity_file -> privateKey', function() { - assert.equal(options.privateKey.toString(), fs.readFileSync(fileName).toString()); - }); - - it('maps hostname -> dstHost', function() { - assert.equal(options.dstHost, 'my.ssh-server.com'); - }); - - it('maps port -> dstPort', function() { - assert.equal(options.dstPort, 27000); - }); - - it('maps ssh_tunnel_port -> port', function() { - assert.equal(options.port, 3000); - }); - - it('maps ssh_tunnel_passphrase -> password', function() { - assert.equal(options.password, 'password'); - }); - }); - - context('when a passphrase does not exist', function() { - var fileName = path.join(__dirname, 'fake-identity-file.txt'); - var options = new Connection({ - ssh_tunnel: 'IDENTITY_FILE', - ssh_tunnel_hostname: 'my.ssh-server.com', - ssh_tunnel_username: 'my-user', - ssh_tunnel_identity_file: [fileName], - hostname: 'mongodb.my-internal-host.com', - port: 27000, - ssh_tunnel_port: 3000 - }).ssh_tunnel_options; - - it('maps ssh_tunnel_username -> username', function() { - assert.equal(options.username, 'my-user'); - }); - - it('maps ssh_tunnel_identity_file -> privateKey', function() { - assert.equal(options.privateKey.toString(), fs.readFileSync(fileName).toString()); - }); - - it('maps ssh_tunnel_hostname -> dstHost', function() { - assert.equal(options.dstHost, 'my.ssh-server.com'); - }); - - it('maps port -> dstPort', function() { - assert.equal(options.dstPort, 27000); - }); - - it('maps ssh_tunnel_port -> sshPort', function() { - assert.equal(options.port, 3000); - }); - }); - }); - }); - - describe('#validate', function() { - context('when ssh_tunnel is NONE', function() { - var c = new Connection({ - ssh_tunnel: 'NONE' - }); - - it('does not fail validation', function() { - assert(c.isValid()); - }); - }); - - context('when ssh_tunnel is USER_PASSWORD', function() { - context('when hostname is missing', function() { - var c = new Connection({ - ssh_tunnel: 'USER_PASSWORD', - ssh_tunnel_port: 5000, - ssh_tunnel_username: 'username', - ssh_tunnel_password: 'password' - }); - - it('fails validation', function() { - assert(!c.isValid()); - }); - }); - - context('when username is missing', function() { - var c = new Connection({ - ssh_tunnel: 'USER_PASSWORD', - ssh_tunnel_hostname: '127.0.0.1', - ssh_tunnel_port: 5000, - ssh_tunnel_password: 'password' - }); - - it('fails validation', function() { - assert(!c.isValid()); - }); - }); - - context('when password is missing', function() { - var c = new Connection({ - ssh_tunnel: 'USER_PASSWORD', - ssh_tunnel_hostname: '127.0.0.1', - ssh_tunnel_port: 5000, - ssh_tunnel_username: 'username' - }); - - it('fails validation', function() { - assert(!c.isValid()); - }); - }); - - context('when the connection is valid', function() { - var c = new Connection({ - ssh_tunnel: 'USER_PASSWORD', - ssh_tunnel_hostname: '127.0.0.1', - ssh_tunnel_port: 5000, - ssh_tunnel_username: 'username', - ssh_tunnel_password: 'password' - }); - - it('does not fail validation', function() { - assert(c.isValid()); - }); - }); - }); - - context('when ssh_tunnel is IDENTITY_FILE', function() { - context('when hostname is missing', function() { - var c = new Connection({ - ssh_tunnel: 'IDENTITY_FILE', - ssh_tunnel_identity_file: '/path/to/.ssh/me.pub', - ssh_tunnel_port: 5000, - ssh_tunnel_username: 'username' - }); - - it('fails validation', function() { - assert(!c.isValid()); - }); - }); - - context('when username is missing', function() { - var c = new Connection({ - ssh_tunnel: 'IDENTITY_FILE', - ssh_tunnel_identity_file: '/path/to/.ssh/me.pub', - ssh_tunnel_hostname: '127.0.0.1', - ssh_tunnel_port: 5000, - ssh_tunnel_passphrase: 'password' - }); - - it('fails validation', function() { - assert(!c.isValid()); - }); - }); - - context('when identity file is missing', function() { - var c = new Connection({ - ssh_tunnel: 'IDENTITY_FILE', - ssh_tunnel_username: 'username', - ssh_tunnel_hostname: '127.0.0.1', - ssh_tunnel_port: 5000, - ssh_tunnel_passphrase: 'password' - }); - - it('fails validation', function() { - assert(!c.isValid()); - }); - }); - - context('when the connection is valid', function() { - var c = new Connection({ - ssh_tunnel: 'IDENTITY_FILE', - ssh_tunnel_identity_file: '/path/to/.ssh/me.pub', - ssh_tunnel_hostname: '127.0.0.1', - ssh_tunnel_port: 5000, - ssh_tunnel_username: 'username', - ssh_tunnel_passphrase: 'password' - }); - - it('does not fail validation', function() { - assert(c.isValid()); - }); - }); - }); - }); - }); }); diff --git a/test/ssh-tunnel.test.js b/test/ssh-tunnel.test.js new file mode 100644 index 00000000..6f6aac74 --- /dev/null +++ b/test/ssh-tunnel.test.js @@ -0,0 +1,258 @@ +var assert = require('assert'); +var Connection = require('../'); +var fs = require('fs'); +var path = require('path'); + +describe('ssh_tunnel', function() { + describe('ssh_tunnel_port', function() { + it('should have the default value', function() { + var c = new Connection(); + assert.equal(c.ssh_tunnel_port, 22); + }); + + it('should cast an empty string to the default', function() { + var c = new Connection({ + ssh_tunnel_port: '' + }); + + assert.equal(c.ssh_tunnel_port, 22); + }); + + it('should cast an empty string to the default when updating', function() { + var c = new Connection({}); + c.set({ + ssh_tunnel_port: '' + }); + assert.equal(c.ssh_tunnel_port, 22); + }); + + it('should not allow negative numbers', function() { + assert.throws(function() { + /* eslint no-new:0 */ + new Connection({ + ssh_tunnel_port: -22 + }); + }, TypeError, /must be positive/); + }); + + it('should not allow values above the max port number value', function() { + assert.throws(function() { + /* eslint no-new:0 */ + new Connection({ + ssh_tunnel_port: 27017 * 10000 + }); + }, TypeError, /must be below/); + }); + }); + + describe('NONE', function() { + var c = new Connection({ + hostname: '127.0.0.1', + ssh_tunnel: 'NONE' + }); + + it('should be valid', function() { + assert(c.isValid()); + }); + + describe('ssh_tunnel_options', function() { + it('should return an empty object', function() { + assert.equal(c.ssh_tunnel_options.hostname, null); + }); + }); + }); + + describe('USER_PASSWORD', function() { + it('should require `ssh_tunnel_hostname`', function() { + var c = new Connection({ + ssh_tunnel: 'USER_PASSWORD', + ssh_tunnel_port: 5000, + ssh_tunnel_username: 'username', + ssh_tunnel_password: 'password' + }); + assert(!c.isValid()); + }); + + it('should require `ssh_tunnel_username`', function() { + var c = new Connection({ + ssh_tunnel: 'USER_PASSWORD', + ssh_tunnel_hostname: '127.0.0.1', + ssh_tunnel_port: 5000, + ssh_tunnel_password: 'password' + }); + assert(!c.isValid()); + }); + + it('should require `ssh_tunnel_password`', function() { + var c = new Connection({ + ssh_tunnel: 'USER_PASSWORD', + ssh_tunnel_hostname: '127.0.0.1', + ssh_tunnel_port: 5000, + ssh_tunnel_username: 'username' + }); + assert(!c.isValid()); + }); + + var connection = new Connection({ + ssh_tunnel: 'USER_PASSWORD', + ssh_tunnel_hostname: 'my.ssh-server.com', + ssh_tunnel_username: 'my-user', + hostname: 'mongodb.my-internal-host.com', + port: 27000, + ssh_tunnel_port: 3000, + ssh_tunnel_password: 'password' + }); + + it('should be valid', function() { + assert(connection.isValid()); + }); + + describe('ssh_tunnel_options', function() { + var options = connection.ssh_tunnel_options; + + it('maps ssh_tunnel_username -> username', function() { + assert.equal(options.username, 'my-user'); + }); + + it('maps ssh_tunnel_hostname -> dstHost', function() { + assert.equal(options.dstHost, 'my.ssh-server.com'); + }); + + it('maps port -> dstPort', function() { + assert.equal(options.dstPort, 27000); + }); + + it('maps ssh_tunnel_port -> port', function() { + assert.equal(options.port, 3000); + }); + + it('maps ssh_tunnel_password -> password', function() { + assert.equal(options.password, 'password'); + }); + }); + }); + + describe('IDENTITY_FILE', function() { + it('should require `ssh_tunnel_hostname`', function() { + var c = new Connection({ + ssh_tunnel: 'IDENTITY_FILE', + ssh_tunnel_identity_file: '/path/to/.ssh/me.pub', + ssh_tunnel_port: 5000, + ssh_tunnel_username: 'username' + }); + + assert(!c.isValid()); + }); + + it('should require `ssh_tunnel_username`', function() { + var c = new Connection({ + ssh_tunnel: 'IDENTITY_FILE', + ssh_tunnel_identity_file: '/path/to/.ssh/me.pub', + ssh_tunnel_hostname: '127.0.0.1', + ssh_tunnel_port: 5000, + ssh_tunnel_passphrase: 'password' + }); + assert(!c.isValid()); + }); + + it('should require `ssh_tunnel_identity_file`', function() { + var c = new Connection({ + ssh_tunnel: 'IDENTITY_FILE', + ssh_tunnel_username: 'username', + ssh_tunnel_hostname: '127.0.0.1', + ssh_tunnel_port: 5000, + ssh_tunnel_passphrase: 'password' + }); + assert(!c.isValid()); + }); + + describe('When `ssh_tunnel_passphrase` is provided', function() { + var fileName = path.join(__dirname, 'fake-identity-file.txt'); + var c = new Connection({ + ssh_tunnel: 'IDENTITY_FILE', + ssh_tunnel_hostname: 'my.ssh-server.com', + ssh_tunnel_username: 'my-user', + ssh_tunnel_identity_file: [fileName], + hostname: 'mongodb.my-internal-host.com', + port: 27000, + ssh_tunnel_port: 3000, + ssh_tunnel_passphrase: 'password' + }); + + it('should be valid', function() { + assert(c.isValid()); + }); + + describe('ssh_tunnel_options', function() { + var options = c.ssh_tunnel_options; + + it('maps ssh_tunnel_username -> username', function() { + assert.equal(options.username, 'my-user'); + }); + + it('maps ssh_tunnel_identity_file -> privateKey', function() { + /* eslint no-sync: 0 */ + assert.equal(options.privateKey.toString(), fs.readFileSync(fileName).toString()); + }); + + it('maps hostname -> dstHost', function() { + assert.equal(options.dstHost, 'my.ssh-server.com'); + }); + + it('maps port -> dstPort', function() { + assert.equal(options.dstPort, 27000); + }); + + it('maps ssh_tunnel_port -> port', function() { + assert.equal(options.port, 3000); + }); + + it('maps ssh_tunnel_passphrase -> password', function() { + assert.equal(options.password, 'password'); + }); + }); + }); + + describe('When `ssh_tunnel_passphrase` is NOT provided', function() { + var fileName = path.join(__dirname, 'fake-identity-file.txt'); + var c = new Connection({ + ssh_tunnel: 'IDENTITY_FILE', + ssh_tunnel_hostname: 'my.ssh-server.com', + ssh_tunnel_username: 'my-user', + ssh_tunnel_identity_file: [fileName], + hostname: 'mongodb.my-internal-host.com', + port: 27000, + ssh_tunnel_port: 3000 + }); + + it('should be valid', function() { + assert(c.isValid()); + }); + + describe('ssh_tunnel_options', function() { + var options = c.ssh_tunnel_options; + + it('maps ssh_tunnel_username -> username', function() { + assert.equal(options.username, 'my-user'); + }); + + it('maps ssh_tunnel_identity_file -> privateKey', function() { + /* eslint no-sync: 0 */ + assert.equal(options.privateKey.toString(), fs.readFileSync(fileName).toString()); + }); + + it('maps ssh_tunnel_hostname -> dstHost', function() { + assert.equal(options.dstHost, 'my.ssh-server.com'); + }); + + it('maps port -> dstPort', function() { + assert.equal(options.dstPort, 27000); + }); + + it('maps ssh_tunnel_port -> sshPort', function() { + assert.equal(options.port, 3000); + }); + }); + }); + }); +}); From 3401a9d783d95784f86d4bc968c7baa9a862abca Mon Sep 17 00:00:00 2001 From: Lucas Hrabovsky Date: Mon, 25 Jul 2016 15:37:47 -0400 Subject: [PATCH 4/6] Make connectWithBackoff its own module --- lib/connect-with-backoff.js | 35 +++++++++++++++++++++++++++++++++++ lib/connect.js | 37 ------------------------------------- 2 files changed, 35 insertions(+), 37 deletions(-) create mode 100644 lib/connect-with-backoff.js diff --git a/lib/connect-with-backoff.js b/lib/connect-with-backoff.js new file mode 100644 index 00000000..fd4d748e --- /dev/null +++ b/lib/connect-with-backoff.js @@ -0,0 +1,35 @@ +var backoff = require('backoff'); +var Connection = require('./model'); +var MongoClient = require('mongodb').MongoClient; +var debug = require('debug')('mongodb-connection-model:connect-with-backoff'); + +module.exports = function(model, done) { + if (!(model instanceof Connection)) { + model = new Connection(model); + } + var url = model.driver_url; + var opts = model.driver_options; + var call; + + var onConnected = function(err, db) { + if (err) { + debug('%s connection attempts to %s failed. giving up.', url, call.getNumRetries()); + return done(err); + } + debug('Successfully connected to %s after %s attempts!', url, call.getNumRetries()); + done(null, db); + }; + + call = backoff.call(MongoClient.connect, url, opts, onConnected); + call.setStrategy(new backoff.ExponentialStrategy({ + randomisationFactor: 0, + initialDelay: 500, + maxDelay: 10000 + })); + + call.on('backoff', function(number, delay) { + debug('connect attempt #%s failed. retrying in %sms...', number, delay); + }); + call.failAfter(25); + call.start(); +}; diff --git a/lib/connect.js b/lib/connect.js index 3bf72e91..6a8bef9a 100644 --- a/lib/connect.js +++ b/lib/connect.js @@ -3,7 +3,6 @@ var parallel = require('run-parallel'); var series = require('run-series'); var clone = require('lodash.clone'); var MongoClient = require('mongodb').MongoClient; -var backoff = require('backoff'); var Connection = require('./model'); var parseURL = require('mongodb/lib/url_parser'); var debug = require('debug')('mongodb-connection-model:connect'); @@ -77,36 +76,6 @@ function validateURL(model, done) { } } -function connectWithBackoff(model, done) { - if (!(model instanceof Connection)) { - model = new Connection(model); - } - var url = model.driver_url; - var opts = model.driver_options; - var call; - - var onConnected = function(err, db) { - if (err) { - debug('%s connection attempts to %s failed. giving up.', url, call.getNumRetries()); - return done(err); - } - debug('Successfully connected to %s after %s attempts!', url, call.getNumRetries()); - done(null, db); - }; - - call = backoff.call(MongoClient.connect, url, opts, onConnected); - call.setStrategy(new backoff.ExponentialStrategy({ - randomisationFactor: 0, - initialDelay: 500, - maxDelay: 10000 - })); - - call.on('backoff', function(number, delay) { - debug('connect attempt #%s failed. retrying in %sms...', number, delay); - }); - call.failAfter(25); - call.start(); -} function connect(model, done) { if (!(model instanceof Connection)) { @@ -128,16 +97,10 @@ function connect(model, done) { }); } -if (process.env.MONGODB_BACKOFF) { - exports = connectWithBackoff; -} else { - exports = connect; } exports.loadOptions = loadOptions; exports.validateURL = validateURL; -exports.connectWithBackoff = connectWithBackoff; exports.connect = connect; - module.exports = exports; From 2e1750862b8df043edefc12be65eb86d1ac6234a Mon Sep 17 00:00:00 2001 From: Lucas Hrabovsky Date: Mon, 25 Jul 2016 15:39:25 -0400 Subject: [PATCH 5/6] Migrate to async and lodash@4 --- lib/connect.js | 13 +++++------ lib/model.js | 58 +++++++++++++++++++--------------------------- package.json | 9 +++---- test/index.test.js | 8 +++---- 4 files changed, 37 insertions(+), 51 deletions(-) diff --git a/lib/connect.js b/lib/connect.js index 6a8bef9a..7433d4bd 100644 --- a/lib/connect.js +++ b/lib/connect.js @@ -1,7 +1,6 @@ var fs = require('fs'); -var parallel = require('run-parallel'); -var series = require('run-series'); -var clone = require('lodash.clone'); +var async = require('async'); +var _ = require('lodash'); var MongoClient = require('mongodb').MongoClient; var Connection = require('./model'); var parseURL = require('mongodb/lib/url_parser'); @@ -16,7 +15,7 @@ function loadOptions(model, done) { } var tasks = {}; - var opts = clone(model.driver_options, true); + var opts = _.clone(model.driver_options, true); Object.keys(opts.server).map(function(key) { if (key.indexOf('ssl') === -1) { return; @@ -24,7 +23,7 @@ function loadOptions(model, done) { if (Array.isArray(opts.server[key])) { tasks[key] = function(cb) { - parallel(opts.server[key].map(function(k) { + async.parallel(opts.server[key].map(function(k) { return fs.readFile.bind(null, k); }), cb); }; @@ -39,7 +38,7 @@ function loadOptions(model, done) { tasks[key] = fs.readFile.bind(null, opts.server[key]); }); - parallel(tasks, function(err, res) { + async.parallel(tasks, function(err, res) { if (err) { return done(err); } @@ -82,7 +81,7 @@ function connect(model, done) { model = new Connection(model); } debug('preparing model...'); - series([ + async.series([ validateURL.bind(null, model), loadOptions.bind(null, model) ], function(err, args) { diff --git a/lib/model.js b/lib/model.js index 10ada984..d29cd300 100644 --- a/lib/model.js +++ b/lib/model.js @@ -2,16 +2,11 @@ var toURL = require('url').format; var format = require('util').format; var AmpersandModel = require('ampersand-model'); var AmpersandCollection = require('ampersand-rest-collection'); -var assign = require('lodash.assign'); -var defaults = require('lodash.defaults'); -var contains = require('lodash.contains'); -var clone = require('lodash.clone'); +var _ = require('lodash'); var parse = require('mongodb-url'); var dataTypes = require('./data-types'); var fs = require('fs'); -// var debug = require('debug')('mongodb-connection-model'); - var Connection = {}; var props = {}; var derived = {}; @@ -19,7 +14,7 @@ var derived = {}; /** * # Top-Level */ -assign(props, { +_.assign(props, { /** * User specified name for this connection. * @@ -46,7 +41,7 @@ assign(props, { } }); -assign(derived, { +_.assign(derived, { /** * @see http://npm.im/mongodb-instance-model */ @@ -93,7 +88,7 @@ var AUTHENTICATION_VALUES = [ */ var AUTHENTICATION_DEFAULT = 'NONE'; -assign(props, { +_.assign(props, { /** * @property {String} authentication - `auth_mechanism` for humans. */ @@ -115,7 +110,7 @@ var AUTHENICATION_TO_AUTH_MECHANISM = { LDAP: 'PLAIN' }; -assign(derived, { +_.assign(derived, { /** * Converts the value of `authentication` (for humans) * into the `auth_mechanism` value for the driver. @@ -183,7 +178,7 @@ var AUTHENTICATION_TO_FIELD_NAMES = { * db: { readPreference: 'nearest' }, * replSet: { connectWithNoPrimary: true } } */ -assign(props, { +_.assign(props, { mongodb_username: { type: 'string', default: undefined @@ -227,7 +222,7 @@ var MONGODB_DATABASE_NAME_DEFAULT = 'admin'; * @enterprise * @see http://bit.ly/mongodb-node-driver-kerberos */ -assign(props, { +_.assign(props, { /** * Any program or computer you access over a network. Examples of * services include “host” (a host, e.g., when you use telnet and rsh), @@ -289,7 +284,7 @@ var KERBEROS_SERVICE_NAME_DEFAULT = 'mongodb'; * @enterprise * @see http://bit.ly/mongodb-node-driver-ldap */ -assign(props, { +_.assign(props, { /** * @see http://bit.ly/mongodb-node-driver-ldap * @see http://bit.ly/mongodb-ldap @@ -330,7 +325,7 @@ assign(props, { * @see http://bit.ly/mongodb-node-driver-x509 * @see http://bit.ly/mongodb-x509 */ -assign(props, { +_.assign(props, { /** * The x.509 certificate derived user name, e.g. "CN=user,OU=OrgUnit,O=myOrg,..." */ @@ -372,7 +367,7 @@ var SSL_VALUES = [ */ var SSL_DEFAULT = 'NONE'; -assign(props, { +_.assign(props, { ssl: { type: 'string', values: SSL_VALUES, @@ -436,7 +431,7 @@ var SSH_TUNNEL_VALUES = [ */ var SSH_TUNNEL_DEFAULT = 'NONE'; -assign(props, { +_.assign(props, { ssh_tunnel: { type: 'string', values: SSH_TUNNEL_VALUES, @@ -505,7 +500,7 @@ var DRIVER_OPTIONS_DEFAULT = { } }; -assign(derived, { +_.assign(derived, { /** * Get the URL which can be passed to `MongoClient.connect(url)`. * @see http://bit.ly/mongoclient-connect @@ -544,7 +539,7 @@ assign(derived, { req.auth = format('%s:%s', this.mongodb_username, this.mongodb_password); req.query.authSource = this.mongodb_database_name; } else if (this.authentication === 'KERBEROS') { - defaults(req.query, { + _.defaults(req.query, { gssapiServiceName: this.kerberos_service_name, authMechanism: this.driver_auth_mechanism }); @@ -559,19 +554,20 @@ assign(derived, { } } else if (this.authentication === 'X509') { req.auth = encodeURIComponent(this.x509_username); - defaults(req.query, { + _.defaults(req.query, { authMechanism: this.driver_auth_mechanism }); } else if (this.authentication === 'LDAP') { req.auth = format('%s:%s', encodeURIComponent(this.ldap_username), this.ldap_password); - defaults(req.query, { + + _.defaults(req.query, { authMechanism: this.driver_auth_mechanism }); } - if (contains(['UNVALIDATED', 'SERVER', 'ALL'], this.ssl)) { + if (_.includes(['UNVALIDATED', 'SERVER', 'ALL'], this.ssl)) { req.query.ssl = 'true'; } @@ -593,16 +589,16 @@ assign(derived, { 'ssl_private_key_password' ], fn: function() { - var opts = clone(DRIVER_OPTIONS_DEFAULT, true); + var opts = _.clone(DRIVER_OPTIONS_DEFAULT, true); if (this.ssl === 'SERVER') { - assign(opts, { + _.assign(opts, { server: { sslValidate: true, sslCA: this.ssl_ca } }); } else if (this.ssl === 'ALL') { - assign(opts, { + _.assign(opts, { server: { sslValidate: true, sslCA: this.ssl_ca, @@ -647,18 +643,12 @@ assign(derived, { }; if (this.ssh_tunnel === 'USER_PASSWORD') { - assign(opts, { - password: this.ssh_tunnel_password - }); + opts.password = this.ssh_tunnel_password; } else if (this.ssh_tunnel === 'IDENTITY_FILE') { /* eslint no-sync: 0 */ - assign(opts, { - privateKey: fs.readFileSync(this.ssh_tunnel_identity_file[0]) - }); + opts.privateKey = fs.readFileSync(this.ssh_tunnel_identity_file[0]); if (this.ssh_tunnel_passphrase) { - assign(opts, { - password: this.ssh_tunnel_passphrase - }); + opts.password = this.ssh_tunnel_passphrase; } } return opts; @@ -740,7 +730,7 @@ Connection = AmpersandModel.extend({ * @param {Object} attrs - Incoming attributes. */ validate_ssl: function(attrs) { - if (!attrs.ssl || contains(['NONE', 'UNVALIDATED'], attrs.ssl)) { + if (!attrs.ssl || _.includes(['NONE', 'UNVALIDATED'], attrs.ssl)) { return; } if (attrs.ssl === 'SERVER' && !attrs.ssl_ca) { diff --git a/package.json b/package.json index 6cd9045b..a3ecf70a 100644 --- a/package.json +++ b/package.json @@ -26,11 +26,9 @@ "dependencies": { "ampersand-model": "^8.0.0", "ampersand-rest-collection": "^5.0.0", + "async": "^2.0.1", "debug": "^2.2.0", - "lodash.assign": "^4.0.2", - "lodash.clone": "^3.0.3", - "lodash.contains": "^2.4.3", - "lodash.defaults": "^3.1.2", + "lodash": "^4.14.0", "mongodb-url": "^1.0.2" }, "devDependencies": { @@ -49,7 +47,6 @@ "backoff": "^2.4.1", "kerberos": "^0.0.20", "mongodb": "^2.2.4", - "run-parallel": "^1.1.4", - "run-series": "^1.1.4" + "tunnel-ssh": "^2.1.1" } } diff --git a/test/index.test.js b/test/index.test.js index aed232c8..d0fb3421 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -4,7 +4,7 @@ var loadOptions = Connection.connect.loadOptions; var parse = require('mongodb-url'); var driverParse = require('mongodb/lib/url_parser'); var fixture = require('mongodb-connection-fixture'); -var clone = require('lodash.clone'); +var _ = require('lodash'); var format = require('util').format; function isNotValidAndHasMessage(model, msg) { @@ -458,7 +458,7 @@ describe('mongodb-connection-model', function() { 'mongodb://localhost:27017/?slaveOk=true&ssl=true'); }); it('should produce the correct driver options', function() { - var expected = clone(Connection.DRIVER_OPTIONS_DEFAULT); + var expected = _.clone(Connection.DRIVER_OPTIONS_DEFAULT); expected.server = { sslCA: [fixture.ssl.ca], sslValidate: true @@ -492,7 +492,7 @@ describe('mongodb-connection-model', function() { }); it('should produce the correct driver_options', function() { - var expected = clone(Connection.DRIVER_OPTIONS_DEFAULT); + var expected = _.clone(Connection.DRIVER_OPTIONS_DEFAULT); expected.server = { sslCA: [fixture.ssl.ca], sslCert: fixture.ssl.server, @@ -521,7 +521,7 @@ describe('mongodb-connection-model', function() { }); it('should produce the correct driver_options', function() { - var expected = clone(Connection.DRIVER_OPTIONS_DEFAULT); + var expected = _.clone(Connection.DRIVER_OPTIONS_DEFAULT); expected.server = { sslCA: [fixture.ssl.ca], sslCert: fixture.ssl.server, From 906720f29dc3ad4729fa4e2a1786550cd40bca84 Mon Sep 17 00:00:00 2001 From: Lucas Hrabovsky Date: Mon, 25 Jul 2016 17:59:32 -0400 Subject: [PATCH 6/6] INT-1650: Move ssh-tunnel from data-service to connect - All logic in one place for easier DX and release management - Refactor `connect()` so if you specify ssh tunnel options, it all just works. - New `status` events to enable live updating status bars @rueckstiess would like to implement. - This will be an `npm version major` and subsequent major of data-service. --- lib/connect.js | 119 ++++++++++++++++++++++++++++++++++------ lib/ssh-tunnel.js | 113 ++++++++++++++++++++++++++++++++++++++ test/connect.test.js | 21 +++++++ test/ssh-tunnel.test.js | 23 ++++++++ 4 files changed, 258 insertions(+), 18 deletions(-) create mode 100644 lib/ssh-tunnel.js diff --git a/lib/connect.js b/lib/connect.js index 7433d4bd..935e0f1c 100644 --- a/lib/connect.js +++ b/lib/connect.js @@ -2,8 +2,11 @@ var fs = require('fs'); var async = require('async'); var _ = require('lodash'); var MongoClient = require('mongodb').MongoClient; -var Connection = require('./model'); var parseURL = require('mongodb/lib/url_parser'); +var Connection = require('./model'); +var createSSHTunnel = require('./ssh-tunnel'); +var EventEmitter = require('events').EventEmitter; + var debug = require('debug')('mongodb-connection-model:connect'); function loadOptions(model, done) { @@ -75,31 +78,111 @@ function validateURL(model, done) { } } +function getTasks(model) { + var options = {}; + var tunnel; + var db; + var state = new EventEmitter(); + var tasks = {}; + + /** + * TODO (imlucas) If localhost, check if MongoDB installed -> no: click/prompt to download + * TODO (imlucas) If localhost, check if MongoDB running -> no: click/prompt to start + */ + + _.assign(tasks, { + 'Validate driver URL': function(cb) { + validateURL(model, cb); + }, + 'Load driver options': function(cb) { + loadOptions(model, function(err, opts) { + if (err) { + return cb(err); + } + options = opts; + cb(); + }); + } + }); + + if (model.ssh_tunnel !== 'NONE') { + _.assign(tasks, { + 'Create SSH Tunnel': function(cb) { + tunnel = createSSHTunnel(model, cb); + tunnel.on('status', function(evt) { + state.emit('status', evt); + }); + } + }); + + // TODO (imlucas) Figure out how to make this less flakey. + // tasks['Test SSH Tunnel'] = function(cb) { + // tunnel.test(cb); + // }; + } + + _.assign(tasks, { + 'Connect': function(cb) { + MongoClient.connect(model.driver_url, options, function(err, _db) { + if (err) { + return cb(err); + } + db = _db; + cb(); + }); + } + }); + + /** + * TODO (imlucas) Option to check if can run a specific command/read|write to collection. + */ + + Object.defineProperties(tasks, { + driver_options: { + get: function() { + return options; + }, + enumerable: false + }, + db: { + get: function() { + return db; + }, + enumerable: false + }, + tunnel: { + get: function() { + return tunnel; + }, + enumerable: false + }, + state: { + get: function() { + return state; + }, + enumerable: false + } + }); + + return tasks; +} function connect(model, done) { if (!(model instanceof Connection)) { model = new Connection(model); } - debug('preparing model...'); - async.series([ - validateURL.bind(null, model), - loadOptions.bind(null, model) - ], function(err, args) { + + var tasks = getTasks(model); + async.series(tasks, function(err) { if (err) { - debug('error preparing model', err); return done(err); } - var url = args[0]; - var options = args[1]; - debug('model prepared! calling driver.connect...'); - MongoClient.connect(url, options, done); + return done(null, tasks.db); }); + return tasks.state; } -} - -exports.loadOptions = loadOptions; -exports.validateURL = validateURL; -exports.connect = connect; - -module.exports = exports; +module.exports = connect; +module.exports.loadOptions = loadOptions; +module.exports.validateURL = validateURL; +module.exports.getTasks = getTasks; diff --git a/lib/ssh-tunnel.js b/lib/ssh-tunnel.js new file mode 100644 index 00000000..d40ee58c --- /dev/null +++ b/lib/ssh-tunnel.js @@ -0,0 +1,113 @@ +var assert = require('assert'); +var createTunnel = require('tunnel-ssh'); +var EventEmitter = require('events').EventEmitter; +var net = require('net'); +var inherits = require('util').inherits; +var debug = require('debug')('mongodb-connection-model:ssh-tunnel'); + +function SSHTunnel(model) { + assert(model.hostname, 'hostname required'); + assert(model.port, 'port required'); + + this.model = model; + this.on('status', function(evt) { + debug('status', evt); + }); +} +inherits(SSHTunnel, EventEmitter); + +SSHTunnel.prototype.listen = function(done) { + /** + * TODO (imlucas) dns.lookup(model.ssh_tunnel_hostname) to check for typos + */ + + this.emit('status', { + message: 'Create SSH Tunnel', + pending: true + }); + + this._tunnel = createTunnel(this.model.ssh_tunnel_options, function(err) { + if (err) { + debug('error setting up tunnel', err); + this.emit('status', { + message: 'Create SSH Tunnel', + error: err + }); + + return done(err); + } + this.emit('status', { + message: 'Create SSH Tunnel', + complete: true + }); + + done(null, true); + }.bind(this)); + + return this; +}; + +SSHTunnel.prototype.test = function(done) { + this.emit('status', { + message: 'Test SSH Tunnel', + pending: true + }); + + var client = new net.Socket(); + client.on('error', function(err) { + debug('test client got an error', err); + client.end(); + this.emit('status', { + message: 'Test SSH Tunnel', + error: err + }); + + done(new Error('SSH Failed. Please ' + err.message)); + }.bind(this)); + + debug('test client connecting to %s:%s', this.model.hostname, this.model.port); + + client.connect(this.model.hostname, this.model.port, function() { + debug('writing test message'); + try { + client.write('mongodb-connection-model:ssh-tunnel: ping'); + } catch (err) { + debug('write to test client failed with error', err); + return done(err); + } + + client.on('end', function() { + debug('disconnecting test socket'); + this.emit('status', { + message: 'Test SSH Tunnel', + complete: true + }); + done(null, true); + }.bind(this)); + + setTimeout(function() { + client.end(); + }, 300); + }.bind(this)); +}; + +SSHTunnel.prototype.close = function() { + this.emit('status', { + message: 'Closing SSH Tunnel' + }); + + if (this._connected) { + this._tunnel.close(); + } +}; + +module.exports = function(model, done) { + var tunnel = new SSHTunnel(model); + if (model.ssh_tunnel === 'NONE') { + done(); + return tunnel; + } + + tunnel.listen(done); + return tunnel; +}; diff --git a/test/connect.test.js b/test/connect.test.js index abe72ff7..4e0d81d0 100644 --- a/test/connect.test.js +++ b/test/connect.test.js @@ -44,5 +44,26 @@ describe('mongodb-connection#connect', function() { }); }); }); + + var find = function(_db, done) { + _db.db('mongodb').collection('fanclub').find({}, {limit: 10}, function(err, docs) { + if (err) { + return done(err); + } + assert.equal(docs.length, 10); + done(); + }); + }; + + data.SSH_TUNNEL_MATRIX.map(function(d) { + it('connects via the ssh_tunnel to ' + d.ssh_tunnel_hostname, function(done) { + connect(d, function(err, _db) { + if (err) { + return done(err); + } + find(_db, done); + }); + }); + }); }); }); diff --git a/test/ssh-tunnel.test.js b/test/ssh-tunnel.test.js index 6f6aac74..2d12c4a3 100644 --- a/test/ssh-tunnel.test.js +++ b/test/ssh-tunnel.test.js @@ -1,9 +1,32 @@ var assert = require('assert'); var Connection = require('../'); +var createSSHTunnel = require('../lib/ssh-tunnel'); var fs = require('fs'); var path = require('path'); describe('ssh_tunnel', function() { + it.skip('should error when ssh fails', function(done) { + var c = new Connection({ + hostname: '127.0.0.1', + ssh_tunnel: 'USER_PASSWORD', + ssh_tunnel_hostname: 'my.ssh-server.com', + ssh_tunnel_username: 'my-user', + ssh_tunnel_password: 'password' + }); + + var tunnel = createSSHTunnel(c, function(err) { + if (err) { + return done(err); + } + tunnel.test(function(_err) { + if (!_err) { + done(new Error('Should have failed to connect')); + } + done(); + }); + }); + }); + describe('ssh_tunnel_port', function() { it('should have the default value', function() { var c = new Connection();