From 4c2cedb040157648c004d4179753a8f5e2f06c2a Mon Sep 17 00:00:00 2001 From: Christopher Hiller Date: Sat, 3 Jun 2017 16:16:47 -0700 Subject: [PATCH] use @mocha/karma-sauce-launcher - straighten some spaghetti in `karma.conf.js` - used increased concurrency; this *should* improve CI speed - allow Travis-CI to open the SauceLabs tunnel and reuse it - allow ES2015+ in config files - make AWS region and bucket public - make "http" tests more robust via `get-port` package - conditionally run sauce connect - allow PRs to execute sauce tests --- .eslintrc.yaml | 1 - .travis.yml | 38 +---- karma.conf.js | 267 +++++++++++++++++++++++------------ lib/.eslintrc.yaml | 2 + package.json | 3 +- test/.eslintrc.yaml | 2 + test/acceptance/http.spec.js | 25 +++- test/http-meta-2.spec.js | 99 +++++++------ test/http-meta.spec.js | 49 ++++--- 9 files changed, 298 insertions(+), 188 deletions(-) create mode 100644 lib/.eslintrc.yaml diff --git a/.eslintrc.yaml b/.eslintrc.yaml index d1e5b62c7d..c967d40d02 100644 --- a/.eslintrc.yaml +++ b/.eslintrc.yaml @@ -3,7 +3,6 @@ env: node: true browser: true parserOptions: - ecmaVersion: 5 sourceType: script extends: semistandard rules: diff --git a/.travis.yml b/.travis.yml index 8a76015bdd..7b46891888 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,9 @@ env: # phantomjs hosts binaries @ bitbucket, which has fairly restrictive # rate-limiting. pull it from this sketchy site in China instead. - PHANTOMJS_CDNURL='https://cnpmjs.org/downloads' + - SAUCE_CONNECT_DOWNLOAD_ON_INSTALL=1 + - ARTIFACTS_BUCKET=mochajs + - ARTIFACTS_REGION=us-west-2 matrix: fast_finish: true @@ -35,42 +38,13 @@ matrix: env: TARGET=test-node - node_js: '8' env: TARGET=lint - # phantomjs - node_js: '8' - env: TARGET=test-browser - # chrome - - node_js: '8' - env: TARGET=test-browser BROWSER="chrome@latest" PLATFORM="Windows 8" - # edge - - node_js: '8' - env: TARGET=test-browser BROWSER="MicrosoftEdge@latest" PLATFORM="Windows 10" - # ie11 - - node_js: '8' - env: TARGET=test-browser BROWSER="internet explorer@11.0" PLATFORM="Windows 8.1" - # ie10 - - node_js: '8' - env: TARGET=test-browser BROWSER="internet explorer@10.0" PLATFORM="Windows 8" - # ie9 - - node_js: '8' - env: TARGET=test-browser BROWSER="internet explorer@9.0" PLATFORM="Windows 7" - # ie8 - - node_js: '8' - env: TARGET=test-browser BROWSER="internet explorer@8.0" PLATFORM="Windows 7" - # ie7 - - node_js: '8' - env: TARGET=test-browser BROWSER="internet explorer@7.0" PLATFORM="Windows XP" - # firefox - - node_js: '8' - env: TARGET=test-browser BROWSER="firefox@latest" PLATFORM="Windows 8.1" - # safari - - node_js: '8' - env: TARGET=test-browser BROWSER="safari@latest" PLATFORM="OS X 10.11" + env: TRAVIS_SAUCE_CONNECT=1 TARGET=test-browser before_install: scripts/travis-before-install.sh install: - npm install - - cd node_modules && ln -nsf @coderbyheart/karma-sauce-launcher karma-sauce-launcher && cd ../ before_script: scripts/travis-before-script.sh @@ -89,4 +63,6 @@ addons: artifacts: paths: - .karma/ - - ./mocha.js + - ./BUILDTMP/mocha.js + jwt: + secure: uZVTLV6IOz2JfT4GJ5Gr0wfaQ3uYQZTtH919oatETYhd2d4+4blXuypd5u93iOV4mIuf3QNlrGPjI6I7vJ/5aMHogD/sWMpmAzy3zX848ueg9Wvv5ldKGYbMNV7eYBrB73EWiwkVk0dS1uDFm0qG/lFFC3vUhnFqzoVRoX7peKQ= diff --git a/karma.conf.js b/karma.conf.js index 2ba9d4d2e1..0face76099 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -1,19 +1,108 @@ 'use strict'; -var fs = require('fs'); -var path = require('path'); -var mkdirp = require('mkdirp'); -var baseBundleDirpath = path.join(__dirname, '.karma'); -var osName = require('os-name'); +const fs = require('fs'); +const path = require('path'); +const mkdirp = require('mkdirp'); +const baseBundleDirpath = path.join(__dirname, '.karma'); +const osName = require('os-name'); + +/** + * To run tests locally against SauceLabs, use: + * `CI=1 SAUCE_USERNAME= SAUCE_ACCESS_KEY= make test-browser` + * Set `DEBUG=1` for extra debug info. + */ + +/** + * SauceLabs concurrency limit + * @type {number} + */ +const CONCURRENCY = 30; + +/** + * Makes a browser object into a custom launcher string + * @param {Object} browser - Object having `version` and `browserName` props + * @returns {string} Custom launcher name + */ +function browserify (browser) { + return `${browser.browserName}@${browser.version}`; +} + +/** + * Browser objects + * @type {Object[]} + */ +const BROWSERS = [ + { + browserName: 'chrome', + version: 'latest', + platform: 'Windows 8' + }, + { + browserName: 'MicrosoftEdge', + version: 'latest', + platform: 'Windows 10' + }, + { + browserName: 'internet explorer', + version: '11.0', + platform: 'Windows 8.1' + }, + { + browserName: 'internet explorer', + version: '10.0', + platform: 'Windows 8' + }, + { + browserName: 'internet explorer', + version: '9.0', + platform: 'Windows 7' + }, + { + browserName: 'internet explorer', + version: '8.0', + platform: 'Windows 7' + }, + { + browserName: 'internet explorer', + version: '7.0', + platform: 'Windows XP' + }, + { + browserName: 'firefox', + version: 'latest', + platform: 'Windows 8.1' + }, + { + browserName: 'safari', + version: 'latest', + platform: 'OS X 10.12' + } +]; module.exports = function (config) { - var bundleDirpath; - var cfg = { + const env = process.env; + + /** + * This is where the artifacts are output for uploading to S3. + * It's used regardless of whether or not we upload anything. + */ + let bundleDirpath; + + let cfg = { frameworks: [ 'browserify', 'expect', 'mocha' ], + plugins: [ + 'karma-browserify', + 'karma-mocha', + 'karma-expect', + 'karma-chrome-launcher', + 'karma-phantomjs-launcher', + 'karma-spec-reporter', + require.resolve('@mocha/karma-sauce-launcher') + ], files: [ // we use the BDD interface for all of the tests that // aren't interface-specific. @@ -40,16 +129,23 @@ module.exports = function (config) { .on('bundled', function (err, content) { if (!err && bundleDirpath) { // write bundle to directory for debugging - fs.writeFileSync(path.join(bundleDirpath, - 'bundle.' + Date.now() + '.js'), content); + fs.writeFileSync( + path.join(bundleDirpath, 'bundle.' + Date.now() + '.js'), + content); } }); } }, reporters: ['spec'], - colors: true, - browsers: [osName() === 'macOS Sierra' ? 'Chrome' : 'PhantomJS'], // This is the default browser to run, locally - logLevel: config.LOG_INFO, + colors: true, /* This is the default browser to run, locally. + * Sierra is incompatible with PhantomJS 1.x + */ + browsers: [ + osName() === 'macOS Sierra' + ? 'Chrome' + : 'PhantomJS' + ], + logLevel: env.DEBUG ? config.LOG_INFO : config.LOG_DEBUG, client: { mocha: { reporter: 'html' @@ -57,114 +153,109 @@ module.exports = function (config) { } }; - // see https://github.com/saucelabs/karma-sauce-example - - // We define the browser to run on the Saucelabs Infrastructure - // via the environment variables BROWSER and PLATFORM. - // PLATFORM is e.g. "Windows" - // BROWSER is expected to be in the format "@", - // e.g. "MicrosoftEdge@latest" - // See https://wiki.saucelabs.com/display/DOCS/Platform+Configurator#/ - // for available browsers. - - // TO RUN LOCALLY, execute: - // `CI=1 SAUCE_USERNAME= SAUCE_ACCESS_KEY= BROWSER= PLATFORM= make test-browser` - var env = process.env; - var sauceConfig; + let useSauceLabs = false; if (env.CI) { console.error('CI mode enabled'); if (env.TRAVIS) { console.error('Travis-CI detected'); - bundleDirpath = path.join(baseBundleDirpath, process.env.TRAVIS_BUILD_ID); - if (env.BROWSER && env.PLATFORM) { - if (env.SAUCE_USERNAME && env.SAUCE_ACCESS_KEY) { - // correlate build/tunnel with Travis - sauceConfig = { - build: 'TRAVIS #' + env.TRAVIS_BUILD_NUMBER + - ' (' + env.TRAVIS_BUILD_ID + ')', - tunnelIdentifier: env.TRAVIS_JOB_NUMBER - }; - console.error('Configured SauceLabs'); - } else { - console.error('No SauceLabs credentials present'); - } - } + bundleDirpath = path.join(baseBundleDirpath, env.TRAVIS_BUILD_ID); + useSauceLabs = env.SAUCE_USERNAME && env.SAUCE_ACCESS_KEY; } else if (env.APPVEYOR) { console.error('AppVeyor detected'); - bundleDirpath = path.join(baseBundleDirpath, process.env.APPVEYOR_BUILD_ID); + bundleDirpath = path.join(baseBundleDirpath, env.APPVEYOR_BUILD_ID); } else { - console.error('Local/unknown environment detected'); - bundleDirpath = path.join(baseBundleDirpath, 'local'); - // don't need to run sauce from appveyor b/c travis does it. - if (env.SAUCE_USERNAME || env.SAUCE_ACCESS_KEY) { - sauceConfig = { - build: require('os') - .hostname() + ' (' + Date.now() + ')' - }; - console.error('Configured SauceLabs'); - } else { - console.error('No SauceLabs credentials present'); - } + console.error('Developer environment detected'); + useSauceLabs = env.SAUCE_USERNAME && env.SAUCE_ACCESS_KEY; } - mkdirp.sync(bundleDirpath); - } else { - console.error('CI mode disabled'); } - if (sauceConfig) { - cfg.sauceLabs = sauceConfig; - addSauceTests(cfg); + // artifacts go in here + bundleDirpath = bundleDirpath || path.join(baseBundleDirpath, 'development'); + mkdirp.sync(bundleDirpath); + + if (useSauceLabs) { + console.error('Configuring for SauceLabs'); + cfg = configureSauceLabs(cfg); + } else { + console.error('No SauceLabs credentials present'); + if (!env.TRAVIS && env.CI) { + console.error('(add SAUCE_ACCESS_KEY and SAUCE_USERNAME to environment)'); + } } - // the MOCHA_UI env var will determine if we're running interface-specific + // the MOCHA_UI env const will determine if we're running interface-specific // tests. since you can only load one at a time, each must be run separately. // each has its own set of acceptance tests and a fixture. // the "bdd" fixture is used by default. - var ui = env.MOCHA_UI; - if (ui) { - if (cfg.sauceLabs) { - cfg.sauceLabs.testName = 'Interface "' + ui + '" integration tests'; - } + if (env.MOCHA_UI) { cfg.files = [ - 'test/browser-fixtures/' + ui + '.fixture.js', - 'test/acceptance/interfaces/' + ui + '.spec.js' + `test/browser-fixtures/${env.MOCHA_UI}.fixture.js`, + `test/acceptance/interfaces/${env.MOCHA_UI}.spec.js` ]; - } else if (cfg.sauceLabs) { - cfg.sauceLabs.testName = 'Unit Tests'; } config.set(cfg); }; -function addSauceTests (cfg) { - var env = process.env; - cfg.reporters.push('saucelabs'); - cfg.customLaunchers = {}; - cfg.customLaunchers[env.BROWSER] = { - base: 'SauceLabs', - browserName: env.BROWSER.split('@')[0], - version: env.BROWSER.split('@')[1], - platform: env.PLATFORM - }; - cfg.browsers = [env.BROWSER]; - - // See https://github.com/karma-runner/karma-sauce-launcher - // See https://github.com/bermi/sauce-connect-launcher#advanced-usage - cfg.sauceLabs = { +function configureSauceLabs (cfg, isTravis) { + const env = process.env; + // base opts shouldn't change + const sauceConfig = { public: 'public', - startConnect: true, connectOptions: { connectRetries: 10, connectRetryTimeout: 60000 } }; - cfg.concurrency = 5; + if (isTravis) { + // correlate build/tunnel with Travis + Object.assign(sauceConfig, { + build: `Travis-CI #${env.TRAVIS_BUILD_NUMBER} (${env.TRAVIS_BUILD_ID})`, + tunnelIdentifier: env.TRAVIS_JOB_NUMBER, + startConnect: false + }); + } else { + const hostname = require('os') + .hostname(); + Object.assign(sauceConfig, { + build: `${hostname} (${new Date()}))`, + testName: 'browser tests: unit', + tags: [ + 'unit', + 'development', + hostname + ] + }); + } - cfg.retryLimit = 5; + if (env.MOCHA_UI) { + sauceConfig.testName = + `${sauceConfig.testName}: ${env.MOCHA_UI} integration`; + sauceConfig.tags.push(env.MOCHA_UI); + } + + cfg.sauceLabs = sauceConfig; + + // add sauce browser launchers + cfg.customLaunchers = {}; + BROWSERS.forEach(browser => { + cfg.customLaunchers[browserify(browser)] = Object.assign(browser, { + base: 'SauceLabs' + }); + }); + // add the launcher names to list of browsers to launch + cfg.browsers.push(...Object.keys(cfg.customLaunchers)); + + // add saucelabs reporter + cfg.reporters.push('saucelabs'); // for slow browser booting, ostensibly + cfg.concurrency = CONCURRENCY; + cfg.retryLimit = 5; cfg.captureTimeout = 120000; - cfg.browserNoActivityTimeout = 20000; + cfg.browserNoActivityTimeout = 40000; + + return cfg; } diff --git a/lib/.eslintrc.yaml b/lib/.eslintrc.yaml new file mode 100644 index 0000000000..85e1b6da5b --- /dev/null +++ b/lib/.eslintrc.yaml @@ -0,0 +1,2 @@ +parserOptions: + ecmaVersion: 5 diff --git a/package.json b/package.json index 562a182a8c..e583391da4 100644 --- a/package.json +++ b/package.json @@ -320,6 +320,7 @@ "supports-color": "3.1.2" }, "devDependencies": { + "@mocha/karma-sauce-launcher": "^2.0.0", "assert": "^1.4.1", "browserify": "^13.0.0", "coffee-script": "^1.10.0", @@ -330,6 +331,7 @@ "eslint-plugin-promise": "^3.4.0", "eslint-plugin-standard": "2.0.1", "expect.js": "^0.3.1", + "get-port": "^1.0.0", "istanbul-combine": "^0.3.0", "karma": "1.3.0", "karma-browserify": "^5.0.5", @@ -337,7 +339,6 @@ "karma-expect": "^1.1.2", "karma-mocha": "^1.3.0", "karma-phantomjs-launcher": "0.2.3", - "karma-sauce-launcher": "coderbyheart/karma-sauce-launcher", "karma-spec-reporter": "0.0.26", "nyc": "^10.0.0", "os-name": "^2.0.1", diff --git a/test/.eslintrc.yaml b/test/.eslintrc.yaml index a32fd84d48..a601b0b940 100644 --- a/test/.eslintrc.yaml +++ b/test/.eslintrc.yaml @@ -3,3 +3,5 @@ env: globals: expect: false assert: false +parserOptions: + ecmaVersion: 5 diff --git a/test/acceptance/http.spec.js b/test/acceptance/http.spec.js index b8bc2f7e26..2ab6266d07 100644 --- a/test/acceptance/http.spec.js +++ b/test/acceptance/http.spec.js @@ -1,17 +1,34 @@ 'use strict'; var http = require('http'); +var getPort = require('get-port'); var server = http.createServer(function (req, res) { res.end('Hello World\n'); }); -server.listen(8888); - describe('http', function () { + var port; + + before(function (done) { + getPort(function (err, portNo) { + if (err) { + return done(err); + } + port = portNo; + server.listen(port, done); + }); + }); + it('should provide an example', function (done) { - http.get({ path: '/', port: 8888 }, function (res) { - expect(res).to.have.property('statusCode', 200); + http.get({ + path: '/', + port: port + }, function (res) { + expect(res) + .to + .have + .property('statusCode', 200); done(); }); }); diff --git a/test/http-meta-2.spec.js b/test/http-meta-2.spec.js index 18b4158467..3b45e1ac95 100644 --- a/test/http-meta-2.spec.js +++ b/test/http-meta-2.spec.js @@ -1,8 +1,7 @@ 'use strict'; var http = require('http'); - -var PORT = 8899; +var getPort = require('get-port'); var server = http.createServer(function (req, res) { var accept = req.headers.accept || ''; @@ -22,55 +21,71 @@ var server = http.createServer(function (req, res) { } }); -function get (url) { - var fields; - var expected; - var header = {}; +describe('http server', function () { + var port; + + function get (url) { + var fields; + var expected; + var header = {}; - function request (done) { - http.get({ path: url, port: PORT, headers: header }, function (res) { - var buf = ''; - res.should.have.property('statusCode', 200); - res.setEncoding('utf8'); - res.on('data', function (chunk) { buf += chunk; }); - res.on('end', function () { - buf.should.equal(expected); - done(); + function request (done) { + http.get({ + path: url, + port: port, + headers: header + }, function (res) { + var buf = ''; + res.should.have.property('statusCode', 200); + res.setEncoding('utf8'); + res.on('data', function (chunk) { + buf += chunk; + }); + res.on('end', function () { + buf.should.equal(expected); + done(); + }); }); - }); - } + } - return { - set: function (field, val) { - header[field] = val; - return this; - }, + return { + set: function (field, val) { + header[field] = val; + return this; + }, - should: { - respond: function (body) { - fields = Object.keys(header).map(function (field) { - return field + ': ' + header[field]; - }).join(', '); + should: { + respond: function (body) { + fields = Object.keys(header) + .map(function (field) { + return field + ': ' + header[field]; + }) + .join(', '); - expected = body; - describe('GET ' + url, function () { - this.timeout(500); - if (fields) { - describe('when given ' + fields, function () { + expected = body; + describe('GET ' + url, function () { + this.timeout(500); + if (fields) { + describe('when given ' + fields, function () { + it('should respond with "' + body + '"', request); + }); + } else { it('should respond with "' + body + '"', request); - }); - } else { - it('should respond with "' + body + '"', request); - } - }); + } + }); + } } - } - }; -} + }; + } -describe('http server', function () { before(function (done) { - server.listen(PORT, done); + getPort(function (err, portNo) { + if (err) { + return done(err); + } + port = portNo; + server.listen(port, done); + }); }); beforeEach(function () { diff --git a/test/http-meta.spec.js b/test/http-meta.spec.js index 2ed7ed9204..8121cc5a9f 100644 --- a/test/http-meta.spec.js +++ b/test/http-meta.spec.js @@ -1,8 +1,7 @@ 'use strict'; var http = require('http'); - -var PORT = 8889; +var getPort = require('get-port'); var server = http.createServer(function (req, res) { var accept = req.headers.accept || ''; @@ -22,28 +21,36 @@ var server = http.createServer(function (req, res) { } }); -function get (url, body, header) { - return function (done) { - http.get({ - path: url, - port: PORT, - headers: header || {} - }, function (res) { - var buf = ''; - res.should.have.property('statusCode', 200); - res.setEncoding('utf8'); - res.on('data', function (chunk) { buf += chunk; }); - res.on('end', function () { - buf.should.equal(body); - done(); +describe('http requests', function () { + var port; + + function get (url, body, header) { + return function (done) { + http.get({ + path: url, + port: port, + headers: header || {} + }, function (res) { + var buf = ''; + res.should.have.property('statusCode', 200); + res.setEncoding('utf8'); + res.on('data', function (chunk) { buf += chunk; }); + res.on('end', function () { + buf.should.equal(body); + done(); + }); }); - }); - }; -} + }; + } -describe('http requests', function () { before(function (done) { - server.listen(PORT, done); + getPort(function (err, portNo) { + if (err) { + return done(err); + } + port = portNo; + server.listen(port, done); + }); }); beforeEach(function () {