From 8d4cc6ce85edcaa62ef12e8f937e4487b137af4f Mon Sep 17 00:00:00 2001 From: Oscar Date: Mon, 28 Nov 2016 07:24:58 -0800 Subject: [PATCH] ci(saucelabs): Report Build ID (#56) Reports Build ID to CircleCI so it can easily distinguish between builds and branches. It separates sauce tests from other environments to those from the master branch. That way the reported results on the README reflect the last master build (instead of anything that hit SauceLabs) Adds IE10 testing (previously only IE9 and IE11 were tested) Closes #55 --- README.md | 4 +- circle.yml | 23 ++++---- package.json | 4 +- spec/web/desiredCapabilities.json | 5 ++ spec/web/sauce.js | 91 ++++++++++++++++++++----------- 5 files changed, 81 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 84fffb4e..026344c5 100644 --- a/README.md +++ b/README.md @@ -76,5 +76,7 @@ The Node build is tested against Node 0.12 and above. Browser builds are tested | Code Quality | [![Code Climate](https://codeclimate.com/github/obartra/ssim/badges/gpa.svg)](https://codeclimate.com/github/obartra/ssim) [![Issue Count](https://codeclimate.com/github/obartra/ssim/badges/issue_count.svg)](https://codeclimate.com/github/obartra/ssim) | | Versioning | [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) [![Commitizen friendly](https://img.shields.io/badge/commitizen-friendly-brightgreen.svg)](http://commitizen.github.io/cz-cli/) [![npm](https://img.shields.io/npm/v/ssim.js.svg)](https://www.npmjs.com/package/ssim.js) | | Dependencies | [![Known Vulnerabilities](https://snyk.io/test/github/obartra/ssim/badge.svg)](https://snyk.io/test/github/obartra/ssim) [![DavidDM](https://david-dm.org/obartra/ssim.svg)](https://david-dm.org/obartra/ssim) | -| Environments | ![](https://img.shields.io/badge/node-0.12-brightgreen.svg) ![](https://img.shields.io/badge/node-7.2-brightgreen.svg) [![Sauce Test Status](https://saucelabs.com/browser-matrix/saucessim.svg)](https://saucelabs.com/u/saucessim) | | Documentation | [![InchCI](https://inch-ci.org/github/obartra/ssim.svg?branch=master)](https://inch-ci.org/github/obartra/ssim) | +| Environments | ![](https://img.shields.io/badge/node-0.12-brightgreen.svg) ![](https://img.shields.io/badge/node-7.2-brightgreen.svg) [![Sauce Test Status](https://saucelabs.com/buildstatus/saucessim-master)](https://saucelabs.com/u/saucessim-master)| + +[![Sauce Browser Matrix](https://saucelabs.com/browser-matrix/saucessim-master.svg)](https://saucelabs.com/u/saucessim-master) diff --git a/circle.yml b/circle.yml index c00ba51b..334f3652 100644 --- a/circle.yml +++ b/circle.yml @@ -17,20 +17,23 @@ dependencies: post: - ci/getVersion.sh # Determine the version semantic-release will use, so we can include it in the build test: - override: + pre: - npm run build # Create Node, web and test builds + override: # `canvas` needs a different install depending on the node version in use. - - nvm use $NODE_012 && rm -rf node_modules/canvas && npm i && npm run e2e:ivc || exit 1 + - nvm use $NODE_012 && rm -rf node_modules/canvas && npm i && npm run e2e:ivc # Clean up and switch back to current node version - nvm use $NODE_CURRENT && rm -rf node_modules/canvas && npm i - - npm run e2e || exit 1 - - npm run lint # Ensure all code adheres to the styleguide - - npm run docs:check # Validate documentation using inchjs - - npm run cover # Report test coverage locally - - npm run cover:check # Fail if coverage drops below 100% - - npm run test:perf # Run performance tests - - npm run test:web # Run web-based tests - - npm run codeclimate # Run tests and send coverage to code climate + + - npm run e2e # Run node end to end tests + - npm run lint # Ensure all code adheres to the styleguide + - npm run docs:check # Validate documentation using inchjs + - npm run test:perf # Run performance tests + - npm run test:web # Run web-based tests + + - npm run cover # Report test coverage locally + - npm run cover:check # Fail if coverage drops below 100% + - npm run codeclimate # Run tests and send coverage to code climate - cp -R coverage/* $CIRCLE_TEST_REPORTS deployment: publish: diff --git a/package.json b/package.json index df9d5db5..717b89ee 100644 --- a/package.json +++ b/package.json @@ -11,8 +11,8 @@ "test:perf": "blue-tape spec/perf/{*,**/*}.spec.js | tap-dot", "test:web": "node spec/web/sauce.js", "e2e": "npm-run-all --parallel e2e:*", - "e2e:live": "blue-tape spec/e2e_dist/live.spec.js | tap-dot", - "e2e:ivc": "blue-tape spec/e2e_dist/ivc.spec.js | tap-dot", + "e2e:live": "blue-tape spec/e2e_dist/live.spec.js | tap-dot || exit 1", + "e2e:ivc": "blue-tape spec/e2e_dist/ivc.spec.js | tap-dot || exit 1", "test:watch": "nodemon node_modules/.bin/blue-tape spec/unit/{*,**/*}.spec.js", "docs": "jsdoc --pedantic -c .jsdoc . --readme README.md", "docs:check": "./ci/doc.sh", diff --git a/spec/web/desiredCapabilities.json b/spec/web/desiredCapabilities.json index f0098e6c..4b9c90dd 100644 --- a/spec/web/desiredCapabilities.json +++ b/spec/web/desiredCapabilities.json @@ -28,4 +28,9 @@ "platform": "Windows 7", "version": "9.0", "name": "Win7 - IE9" +}, { + "browserName": "internet explorer", + "platform": "Windows 7", + "version": "10.0", + "name": "Win7 - IE10" }] diff --git a/spec/web/sauce.js b/spec/web/sauce.js index 72c468c0..99e8380d 100644 --- a/spec/web/sauce.js +++ b/spec/web/sauce.js @@ -6,8 +6,14 @@ const SauceLabs = require('saucelabs'); let testsCompleted = 0; const httpPort = 8080; -const maxTests = 5; -const maxTestTime = 60; +const maxTestTime = Infinity; + +const maxRetries = 3; +const isCIMaster = process.env.CI && process.env.CIRCLE_BRANCH === 'master'; +const tags = [process.env.CIRCLE_BRANCH || 'dev']; +const build = process.env.CIRCLE_BUILD_NUM || 0; +const username = isCIMaster ? process.env.SAUCE_USERNAME_MASTER : process.env.SAUCE_USERNAME; +const password = isCIMaster ? process.env.SAUCE_ACCESS_KEY_MASTER : process.env.SAUCE_ACCESS_KEY; const options = { selenium: { install: {}, @@ -17,12 +23,12 @@ const options = { } }, saucelabs: { - username: process.env.SAUCE_USERNAME, - password: process.env.SAUCE_ACCESS_KEY + username, + password }, webdriver: { - user: process.env.SAUCE_USERNAME, - key: process.env.SAUCE_ACCESS_KEY, + user: username, + key: password, host: 'ondemand.saucelabs.com', port: 80 } @@ -49,7 +55,7 @@ function startSauceLabs(ops = {}) { return Promise.resolve(new SauceLabs(ops)); } -function runTests(ops, saucelabs, url, browser) { +function runTests(ops, saucelabs, url, browser, retries = 0) { const name = browser.name; ops.desiredCapabilities = browser; @@ -57,32 +63,32 @@ function runTests(ops, saucelabs, url, browser) { const client = webdriverio.remote(ops); - client.addCommand('sauceJobStatus', (status, done) => - saucelabs.updateJob(client.requestHandler.sessionID, status, done) - ); + client.addCommand('sauceJobStatus', (passed, done) => { + const id = client.requestHandler.sessionID; + + saucelabs.updateJob(id, { passed, tags, build, public: true }, done); + }); return client .init() .url(`${url}/spec/web/index.html`) - .waitForVisible('#test-results.complete', maxTests * maxTestTime * 1000) + .waitForVisible('#test-results.complete', maxTestTime) .getAttribute('#test-results.complete', 'class') .then(onClassFound) .then((passed) => { - client - .sauceJobStatus({ - passed, - public: true - }) - .end(); - return passed; - }) - .then(passed => timeout(passed, 1000)) - .then(passed => onComplete(passed, `Completed tests for ${name} (${passed ? '✔️' : '❌'})`)) - .catch(err => onComplete(false, err)); -} - -function timeout(pass, delay) { - return new Promise(resolve => setTimeout(() => resolve(pass), delay)); + if (passed || ++retries >= maxRetries) { + return client + .sauceJobStatus(passed) + .end() + .then(() => + onComplete(passed, `Completed tests for ${name} (${passed ? '✔️' : '❌'})`) + ); + } + console.log(`Starting retry for ${name} (${retries}/${maxRetries})`); + return timeout(60 * 1000) // wait for 1 full minute to allow ngrok to cool down + .then(() => console.log('Retrying now...')) + .then(() => runTests(ops, saucelabs, url, browser, retries)); + }); } function onClassFound(classNames) { @@ -92,6 +98,10 @@ function onClassFound(classNames) { return classNames.split(' ').indexOf('all-good') !== -1; } +function timeout(delay, param) { + return new Promise(resolve => setTimeout(() => resolve(param), delay)); +} + function onComplete(passed, msg) { if (!passed) { console.error('Oops, tests failed'); @@ -108,16 +118,31 @@ function onComplete(passed, msg) { } } +function getGroups(items, size = 4) { + const copy = items.slice(0); + const split = []; + + while (copy.length > 0) { + split.push(copy.splice(0, size)); + } + + return split; +} + Promise.all([ startServer(httpPort), startSauceLabs(options.saucelabs) ]) -.then(([url, saucelabs]) => - Promise.all( - browserList.map(browser => - runTests(options.webdriver, saucelabs, url, browser) - ) - ) -) +.then(([url, saucelabs]) => { + function runTargetTests(target) { + return runTests(options.webdriver, saucelabs, url, target); + } + + // We split the tests into groups to limit the number of requests, otherwise tests may exceed + // the ngrok connection limit + return getGroups(browserList).reduce((p, targets) => + p.then(() => Promise.all(targets.map(runTargetTests)) + ), Promise.resolve()); +}) .catch(err => onComplete(false, err));