ci(saucelabs): Report Build ID (#56)

Reports Build ID to CircleCI so it can easily distinguish between builds and branches.

It separates sauce tests from other environments to those from the master branch. That way the
reported results on the README reflect the last master build (instead of anything that hit
SauceLabs)

Adds IE10 testing (previously only IE9 and IE11 were tested)

Closes #55

README.md

@@ -76,5 +76,7 @@ The Node build is tested against Node 0.12 and above. Browser builds are tested
 | Code Quality  | [![Code Climate](https://codeclimate.com/github/obartra/ssim/badges/gpa.svg)](https://codeclimate.com/github/obartra/ssim) [![Issue Count](https://codeclimate.com/github/obartra/ssim/badges/issue_count.svg)](https://codeclimate.com/github/obartra/ssim) |
 | Versioning    | [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) [![Commitizen friendly](https://img.shields.io/badge/commitizen-friendly-brightgreen.svg)](http://commitizen.github.io/cz-cli/) [![npm](https://img.shields.io/npm/v/ssim.js.svg)](https://www.npmjs.com/package/ssim.js) |
 | Dependencies  | [![Known Vulnerabilities](https://snyk.io/test/github/obartra/ssim/badge.svg)](https://snyk.io/test/github/obartra/ssim) [![DavidDM](https://david-dm.org/obartra/ssim.svg)](https://david-dm.org/obartra/ssim) |
-| Environments  | ![](https://img.shields.io/badge/node-0.12-brightgreen.svg) ![](https://img.shields.io/badge/node-7.2-brightgreen.svg) [![Sauce Test Status](https://saucelabs.com/browser-matrix/saucessim.svg)](https://saucelabs.com/u/saucessim) |
 | Documentation | [![InchCI](https://inch-ci.org/github/obartra/ssim.svg?branch=master)](https://inch-ci.org/github/obartra/ssim) |
+| Environments  | ![](https://img.shields.io/badge/node-0.12-brightgreen.svg) ![](https://img.shields.io/badge/node-7.2-brightgreen.svg) [![Sauce Test Status](https://saucelabs.com/buildstatus/saucessim-master)](https://saucelabs.com/u/saucessim-master)|
+
+[![Sauce Browser Matrix](https://saucelabs.com/browser-matrix/saucessim-master.svg)](https://saucelabs.com/u/saucessim-master)

circle.yml

@@ -17,20 +17,23 @@ dependencies:
   post:
     - ci/getVersion.sh # Determine the version semantic-release will use, so we can include it in the build
 test:
-  override:
+  pre:
     - npm run build        # Create Node, web and test builds
+  override:
     # `canvas` needs a different install depending on the node version in use.
-    - nvm use $NODE_012 && rm -rf node_modules/canvas && npm i && npm run e2e:ivc || exit 1
+    - nvm use $NODE_012 && rm -rf node_modules/canvas && npm i && npm run e2e:ivc
     # Clean up and switch back to current node version
     - nvm use $NODE_CURRENT && rm -rf node_modules/canvas && npm i
-    - npm run e2e || exit 1
-    - npm run lint         # Ensure all code adheres to the styleguide
-    - npm run docs:check   # Validate documentation using inchjs
-    - npm run cover        # Report test coverage locally
-    - npm run cover:check  # Fail if coverage drops below 100%
-    - npm run test:perf    # Run performance tests
-    - npm run test:web     # Run web-based tests
-    - npm run codeclimate  # Run tests and send coverage to code climate
+
+    - npm run e2e         # Run node end to end tests
+    - npm run lint        # Ensure all code adheres to the styleguide
+    - npm run docs:check  # Validate documentation using inchjs
+    - npm run test:perf   # Run performance tests
+    - npm run test:web    # Run web-based tests
+
+    - npm run cover       # Report test coverage locally
+    - npm run cover:check # Fail if coverage drops below 100%
+    - npm run codeclimate # Run tests and send coverage to code climate
     - cp -R coverage/* $CIRCLE_TEST_REPORTS
 deployment:
   publish:

package.json

@@ -11,8 +11,8 @@
     "test:perf": "blue-tape spec/perf/{*,**/*}.spec.js | tap-dot",
     "test:web": "node spec/web/sauce.js",
     "e2e": "npm-run-all --parallel e2e:*",
-    "e2e:live": "blue-tape spec/e2e_dist/live.spec.js | tap-dot",
-    "e2e:ivc": "blue-tape spec/e2e_dist/ivc.spec.js | tap-dot",
+    "e2e:live": "blue-tape spec/e2e_dist/live.spec.js | tap-dot || exit 1",
+    "e2e:ivc": "blue-tape spec/e2e_dist/ivc.spec.js | tap-dot || exit 1",
     "test:watch": "nodemon node_modules/.bin/blue-tape spec/unit/{*,**/*}.spec.js",
     "docs": "jsdoc --pedantic -c .jsdoc . --readme README.md",
     "docs:check": "./ci/doc.sh",

spec/web/desiredCapabilities.json

@@ -28,4 +28,9 @@
 	"platform": "Windows 7",
 	"version": "9.0",
 	"name": "Win7 - IE9"
+}, {
+	"browserName": "internet explorer",
+	"platform": "Windows 7",
+	"version": "10.0",
+	"name": "Win7 - IE10"
 }]

spec/web/sauce.js

@@ -6,8 +6,14 @@ const SauceLabs = require('saucelabs');
 
 let testsCompleted = 0;
 const httpPort = 8080;
-const maxTests = 5;
-const maxTestTime = 60;
+const maxTestTime = Infinity;
+
+const maxRetries = 3;
+const isCIMaster = process.env.CI && process.env.CIRCLE_BRANCH === 'master';
+const tags = [process.env.CIRCLE_BRANCH || 'dev'];
+const build = process.env.CIRCLE_BUILD_NUM || 0;
+const username = isCIMaster ? process.env.SAUCE_USERNAME_MASTER : process.env.SAUCE_USERNAME;
+const password = isCIMaster ? process.env.SAUCE_ACCESS_KEY_MASTER : process.env.SAUCE_ACCESS_KEY;
 const options = {
 	selenium: {
 		install: {},
@@ -17,12 +23,12 @@ const options = {
 		}
 	},
 	saucelabs: {
-		username: process.env.SAUCE_USERNAME,
-		password: process.env.SAUCE_ACCESS_KEY
+		username,
+		password
 	},
 	webdriver: {
-		user: process.env.SAUCE_USERNAME,
-		key: process.env.SAUCE_ACCESS_KEY,
+		user: username,
+		key: password,
 		host: 'ondemand.saucelabs.com',
 		port: 80
 	}
@@ -49,40 +55,40 @@ function startSauceLabs(ops = {}) {
 	return Promise.resolve(new SauceLabs(ops));
 }
 
-function runTests(ops, saucelabs, url, browser) {
+function runTests(ops, saucelabs, url, browser, retries = 0) {
 	const name = browser.name;
 
 	ops.desiredCapabilities = browser;
 	console.log('Starting tests for ', name);
 
 	const client = webdriverio.remote(ops);
 
-	client.addCommand('sauceJobStatus', (status, done) =>
-		saucelabs.updateJob(client.requestHandler.sessionID, status, done)
-	);
+	client.addCommand('sauceJobStatus', (passed, done) => {
+		const id = client.requestHandler.sessionID;
+
+		saucelabs.updateJob(id, { passed, tags, build, public: true }, done);
+	});
 
 	return client
 		.init()
 		.url(`${url}/spec/web/index.html`)
-		.waitForVisible('#test-results.complete', maxTests * maxTestTime * 1000)
+		.waitForVisible('#test-results.complete', maxTestTime)
 		.getAttribute('#test-results.complete', 'class')
 		.then(onClassFound)
 		.then((passed) => {
-			client
-				.sauceJobStatus({
-					passed,
-					public: true
-				})
-				.end();
-			return passed;
-		})
-		.then(passed => timeout(passed, 1000))
-		.then(passed => onComplete(passed, `Completed tests for ${name} (${passed ? '✔️' : '❌'})`))
-		.catch(err => onComplete(false, err));
-}
-
-function timeout(pass, delay) {
-	return new Promise(resolve => setTimeout(() => resolve(pass), delay));
+			if (passed || ++retries >= maxRetries) {
+				return client
+					.sauceJobStatus(passed)
+					.end()
+					.then(() =>
+						onComplete(passed, `Completed tests for ${name} (${passed ? '✔️' : '❌'})`)
+					);
+			}
+			console.log(`Starting retry for ${name} (${retries}/${maxRetries})`);
+			return timeout(60 * 1000) // wait for 1 full minute to allow ngrok to cool down
+				.then(() => console.log('Retrying now...'))
+				.then(() => runTests(ops, saucelabs, url, browser, retries));
+		});
 }
 
 function onClassFound(classNames) {
@@ -92,6 +98,10 @@ function onClassFound(classNames) {
 	return classNames.split(' ').indexOf('all-good') !== -1;
 }
 
+function timeout(delay, param) {
+	return new Promise(resolve => setTimeout(() => resolve(param), delay));
+}
+
 function onComplete(passed, msg) {
 	if (!passed) {
 		console.error('Oops, tests failed');
@@ -108,16 +118,31 @@ function onComplete(passed, msg) {
 	}
 }
 
+function getGroups(items, size = 4) {
+	const copy = items.slice(0);
+	const split = [];
+
+	while (copy.length > 0) {
+		split.push(copy.splice(0, size));
+	}
+
+	return split;
+}
+
 Promise.all([
 	startServer(httpPort),
 	startSauceLabs(options.saucelabs)
 ])
-.then(([url, saucelabs]) =>
-	Promise.all(
-		browserList.map(browser =>
-			runTests(options.webdriver, saucelabs, url, browser)
-		)
-	)
-)
+.then(([url, saucelabs]) => {
+	function runTargetTests(target) {
+		return runTests(options.webdriver, saucelabs, url, target);
+	}
+
+	// We split the tests into groups to limit the number of requests, otherwise tests may exceed
+	// the ngrok connection limit
+	return getGroups(browserList).reduce((p, targets) =>
+		p.then(() => Promise.all(targets.map(runTargetTests))
+	), Promise.resolve());
+})
 .catch(err => onComplete(false, err));