From fd7fe4a8c2c6fab6678d0c1f4d5619f7a2376990 Mon Sep 17 00:00:00 2001 From: "simon.van.casteren@gmail.com" Date: Thu, 27 Mar 2014 19:10:15 +0100 Subject: [PATCH] feat(launcher): Add support for splitTestsBetweenCapabilities. --- lib/configParser.js | 1 + lib/launcher.js | 318 +++++++++++++++++++++++++++-------------- lib/runFromLauncher.js | 3 + lib/runner.js | 12 +- scripts/test.js | 1 + spec/multiSplitConf.js | 32 +++++ 6 files changed, 247 insertions(+), 120 deletions(-) create mode 100644 spec/multiSplitConf.js diff --git a/lib/configParser.js b/lib/configParser.js index 92243d07b..4ac0ee76a 100644 --- a/lib/configParser.js +++ b/lib/configParser.js @@ -25,6 +25,7 @@ var ConfigParser = function() { capabilities: { browserName: 'chrome' }, + splitTestsBetweenCapabilities: false, multiCapabilities: [], rootElement: 'body', allScriptsTimeout: 11000, diff --git a/lib/launcher.js b/lib/launcher.js index 379ebed48..5a1e8bdb9 100644 --- a/lib/launcher.js +++ b/lib/launcher.js @@ -2,7 +2,7 @@ * The launcher is responsible for parsing the capabilities from the * input configuration and launching test runners. */ - 'use strict'; +'use strict'; var child = require('child_process'), ConfigParser = require('./configParser'); @@ -16,27 +16,6 @@ var log_ = function(stuff) { var noLineLog_ = function(stuff) { process.stdout.write(launcherPrefix + stuff); }; - -var reportHeader_ = function(driverInstance) { - var capability = driverInstance.capability; - var eol = require('os').EOL; - - var outputHeader = eol + '------------------------------------' + eol; - outputHeader += 'PID: ' + driverInstance.process.pid + ' (capability: '; - outputHeader += (capability.browserName) ? - capability.browserName : ''; - outputHeader += (capability.version) ? - capability.version : ''; - outputHeader += (capability.platform) ? - capability.platform : ''; - outputHeader += (driverInstance.runNumber) ? - ' #' + driverInstance.runNumber : ''; - outputHeader += ')' + eol; - outputHeader += '------------------------------------' + eol; - - console.log(outputHeader); -}; - /** * Initialize and run the tests. * @@ -45,12 +24,14 @@ var reportHeader_ = function(driverInstance) { */ var init = function(configFile, additionalConfig) { - var capabilityRunCount, - driverInstances = [], - launcherExitCode = 0; + // capabilities = array that holds one array per selected capability (Chrome, FF, Canary, ...) + // This array holds all the instances of the capability + var capabilities = [], + launcherExitCode = 0, + allSpecs, + excludes; var configParser = new ConfigParser(); - if (configFile) { configParser.addFileConfig(configFile); } @@ -59,27 +40,43 @@ var init = function(configFile, additionalConfig) { } var config = configParser.getConfig(); - var listRemainingForks = function() { - var remaining = 0; - driverInstances.forEach(function(driverInstance) { - if (!driverInstance.done) { - remaining++; - } + var countDriverInstances = function() { + var count = 0; + capabilities.forEach(function(capabilityDriverInstances) { + count += capabilityDriverInstances.length; }); + return count; + }; + var countRunningDriverInstances = function() { + var count = 0; + capabilities.forEach(function(capabilityDriverInstances) { + capabilityDriverInstances.forEach(function(driverInstance) { + if (!driverInstance.done) { + count += 1; + } + }); + }); + return count; + }; + + var listRemainingForks = function() { + var remaining = countRunningDriverInstances(); if (remaining) { noLineLog_(remaining + ' instance(s) of WebDriver still running'); } }; var logSummary = function() { - driverInstances.forEach(function(driverInstance) { - var shortChildName = driverInstance.capability.browserName + + capabilities.forEach(function(capabilityDriverInstances) { + capabilityDriverInstances.forEach(function(driverInstance) { + var shortChildName = driverInstance.capability.browserName + (driverInstance.runNumber ? ' #' + driverInstance.runNumber : ''); - if (driverInstance.failedCount) { - log_(shortChildName + ' failed ' + driverInstance.failedCount + ' test(s)'); - } else { - log_(shortChildName + ' passed'); - } + if (driverInstance.failedCount) { + log_(shortChildName + ' failed ' + driverInstance.failedCount + ' test(s)'); + } else { + log_(shortChildName + ' passed'); + } + }); }); }; @@ -88,124 +85,225 @@ var init = function(configFile, additionalConfig) { throw new Error('Cannot run in debug mode with multiCapabilities'); } log_('Running using config.multiCapabilities - ' + - 'config.capabilities will be ignored'); + 'config.capabilities will be ignored'); } - // Use capabilities if multiCapabilities is empty. if (!config.multiCapabilities.length) { config.multiCapabilities = [config.capabilities]; } - // Loop through capabilities and set up forks of runner.js - for (var i = 0; i < config.multiCapabilities.length; i++) { - - // Determine how many times to run the capability - capabilityRunCount = (config.multiCapabilities[i].count) ? - config.multiCapabilities[i].count : 1; + var Fork = function(configFile, additionalConfig, capability, specs, runNumber, single) { + var silent = single ? false: true; + + this.configFile = configFile; + this.additionalConfig = additionalConfig; + this.capability = capability; + this.runNumber = runNumber; + this.single = single; + this.specs = specs; + + this.process = child.fork( + __dirname + '/runFromLauncher.js', + process.argv.slice(2),{ + cwd: process.cwd(), + silent: silent + } + ); + }; - // Fork the child runners. - for (var j = 0; j < capabilityRunCount; j++) { - driverInstances.push({ - configFile: configFile, - additionalConfig: additionalConfig, - capability: config.multiCapabilities[i], - runNumber: j + 1 - }); + Fork.prototype.reportHeader_ = function() { + var capability = this.capability; + var eol = require('os').EOL; + var outputHeader = eol + '------------------------------------' + eol; + outputHeader += 'PID: ' + this.process.pid + ' (capability: '; + outputHeader += (capability.browserName) ? + capability.browserName : ''; + outputHeader += (capability.version) ? + capability.version : ''; + outputHeader += (capability.platform) ? + capability.platform : ''; + outputHeader += (this.runNumber) ? + ' #' + this.runNumber : ''; + outputHeader += ')' + eol; + if (config.splitTestsBetweenCapabilities) { + outputHeader += 'Specs: '+ this.specs.toString() + eol; } - } + outputHeader += '------------------------------------' + eol; - // If there is a single runner, avoid starting a separate process - // and print output directly. - // Otherwise, if we're launching multiple runners, aggregate output until - // completion. - if (driverInstances.length === 1) { - var driverInstance = driverInstances[0]; - - var Runner = require('./runner'); - config.capabilities = driverInstance.capability; - - var runner = new Runner(config); - runner.run().then(function(exitCode) { - process.exit(exitCode); - }).catch(function(err) { - log_('Error: ' + err.message); - process.exit(1); - }); - runner.on('testsDone', function(failedCount) { - driverInstance.failedCount = failedCount; - }); - } else { - noLineLog_('Running ' + driverInstances.length + - ' instances of WebDriver'); + console.log(outputHeader); + }; - // Launch each fork and set up listeners - driverInstances.forEach(function(driverInstance) { + // If we're launching multiple runners, aggregate output until completion. + // Otherwise, there is a single runner, let's pipe the output straight + // through to maintain realtime reporting. + Fork.prototype.addEventHandlers = function(testsDoneCallback) { + var self = this; + if (this.single) { + this.process.on('error', function(err) { + log_('Runner Process(' + self.process.pid + ') Error: ' + err); + }); - driverInstance.process = child.fork( - __dirname + '/runFromLauncher.js', - process.argv.slice(2), - {silent: true, cwd: process.cwd()}); + this.process.on('message', function(m) { + switch (m.event) { + case 'testsDone': + this.failedCount = m.failedCount; + break; + } + }); - driverInstance.output = ''; + this.process.on('exit', function(code) { + if (code) { + log_('Runner Process Exited With Error Code: ' + code); + launcherExitCode = 1; + } + }); + } else { + // Multiple capabilities and/or instances + this.output = ''; // stdin pipe - driverInstance.process.stdout.on('data', function(chunk) { - driverInstance.output += chunk; + this.process.stdout.on('data', function(chunk) { + self.output += chunk; }); // stderr pipe - driverInstance.process.stderr.on('data', function(chunk) { - driverInstance.output += chunk; + this.process.stderr.on('data', function(chunk) { + self.output += chunk; }); - driverInstance.process.on('message', function(m) { + this.process.on('message', function(m) { switch (m.event) { case 'testPass': - process.stdout.write('.'); + process.stdout.write('.'); break; case 'testFail': process.stdout.write('F'); break; case 'testsDone': - driverInstance.failedCount = m.failedCount; + self.failedCount = m.failedCount; + if (typeof testsDoneCallback === 'function') { + testsDoneCallback(); + } break; } }); // err handler - driverInstance.process.on('error', function(err) { - log_('Runner Process(' + driverInstance.process.pid + ') Error: ' + err); + this.process.on('error', function(err) { + log_('Runner Process(' + self.process.pid + ') Error: ' + err); }); // exit handlers - driverInstance.process.on('exit', function(code) { + this.process.on('exit', function(code) { if (code) { log_('Runner Process Exited With Error Code: ' + code); launcherExitCode = 1; } - reportHeader_(driverInstance); - console.log(driverInstance.output); - driverInstance.done = true; + self.reportHeader_(); + console.log(self.output); + self.done = true; listRemainingForks(); }); + } + }; - driverInstance.process.send({ - command: 'run', - configFile: driverInstance.configFile, - additionalConfig: driverInstance.additionalConfig, - capability: driverInstance.capability - }); + Fork.prototype.run = function() { + this.process.send({ + command: 'run', + configFile: this.configFile, + additionalConfig: this.additionalConfig, + capability: this.capability, + specs: this.specs + }); + }; + + excludes = ConfigParser.resolveFilePatterns( config.exclude, true, config.configDir); + + allSpecs = ConfigParser.resolveFilePatterns( + ConfigParser.getSpecs(config), false, config.configDir).filter(function(path) { + return excludes.indexOf(path) < 0; + }); + + // If there is a single capability with a single instance, avoid starting a separate process + // and print output directly. + // Otherwise, if we're launching multiple runners, aggregate output until + // completion. + if (config.multiCapabilities.length === 1 + && (config.multiCapabilities[0].count === 1 || !config.multiCapabilities[0].count)) { + var Runner = require('./runner'); + capabilities[0] = [{ + capability: config.multiCapabilities[0], + runNumber: 0 + }]; + config.capabilities = capabilities[0][0].capability; + config.specs = allSpecs; + + var runner = new Runner(config); + runner.run().then(function(exitCode) { + process.exit(exitCode); + }).catch(function(err) { + log_('Error: ' + err.message); + process.exit(1); + }); + + runner.on('testsDone', function(failedCount) { + capabilities[0][0].failedCount = failedCount; }); + } else { + // Loop over different capabilities in config file + // Make array for each of them in capabilities array + // Push driverInstances into this array, each with one spec (file!) + // When a driverInstance finishes it's spec, it will create a new driver (Fork) + // with the next spec in line + config.multiCapabilities.forEach(function(capability, index) { + var forksCounter = 0; + + capability.count = capability.count || 1; + + capabilities[index] = []; // Matrix: Dim 1: Capabilities, Dim 2: Instances of capability + if (allSpecs.length < capability.count && config.splitTestsBetweenCapabilities) { + // When we split the specs over multiple instances, + // we can have maximum as many instances as we have specs + capability.count = allSpecs.length; + } - process.on('exit', function(code) { - if (code) { - launcherExitCode = code; + while(forksCounter < capability.count) { + if (config.splitTestsBetweenCapabilities) { + var createAndRunPartialSpecFork = function() { + var specs = allSpecs[forksCounter]; + if (!specs) { + return; + } + var fork = new Fork(configFile,additionalConfig, + capability,[specs],forksCounter+1,false); + capabilities[index].push(fork); + fork.run(); + fork.addEventHandlers(createAndRunPartialSpecFork); + forksCounter++; + }; + createAndRunPartialSpecFork(); + } else { + var fork = new Fork(configFile, additionalConfig, + capability, allSpecs, forksCounter+1, false); + capabilities[index].push(fork); + fork.run(); + fork.addEventHandlers(); + forksCounter++; + } } - logSummary(); - process.exit(launcherExitCode); }); } + noLineLog_('Running ' + countDriverInstances() + + ' instances of WebDriver'); + + process.on('exit', function(code) { + if (code) { + launcherExitCode = code; + } + logSummary(); + process.exit(launcherExitCode); + }); }; exports.init = init; diff --git a/lib/runFromLauncher.js b/lib/runFromLauncher.js index 84260ee23..9eff8dbbb 100644 --- a/lib/runFromLauncher.js +++ b/lib/runFromLauncher.js @@ -25,6 +25,9 @@ process.on('message', function(m) { // Grab capability to run from launcher. config.capabilities = m.capability; + //Get specs to be executed by this runner + config.specs = m.specs; + // Launch test run. var runner = new Runner(config); diff --git a/lib/runner.js b/lib/runner.js index ee1dfaace..08b6c3f52 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -361,17 +361,9 @@ Runner.prototype.run = function() { var self = this, driver, specs, - excludes, testResult; - // Determine included and excluded specs based on file pattern. - excludes = ConfigParser.resolveFilePatterns( - this.config_.exclude, true, this.config_.configDir); - - specs = ConfigParser.resolveFilePatterns( - ConfigParser.getSpecs(this.config_), false, this.config_.configDir).filter(function(path) { - return excludes.indexOf(path) < 0; - }); + specs = this.config_.specs; if (!specs.length) { throw new Error('Spec patterns did not match any files.'); @@ -384,7 +376,7 @@ Runner.prototype.run = function() { driver = self.driverprovider_.getDriver(); return q.fcall(function() {}); - // 2) Execute test cases + // 2) Execute test cases }).then(function() { var deferred = q.defer(); diff --git a/scripts/test.js b/scripts/test.js index 217c93ca9..5783a9f9c 100755 --- a/scripts/test.js +++ b/scripts/test.js @@ -6,6 +6,7 @@ var spawn = require('child_process').spawn; var scripts = [ 'node lib/cli.js spec/basicConf.js', 'node lib/cli.js spec/multiConf.js', + 'node lib/cli.js spec/multiSplitConf.js', 'node lib/cli.js spec/altRootConf.js', 'node lib/cli.js spec/onPrepareConf.js', 'node lib/cli.js spec/mochaConf.js', diff --git a/spec/multiSplitConf.js b/spec/multiSplitConf.js new file mode 100644 index 000000000..590149321 --- /dev/null +++ b/spec/multiSplitConf.js @@ -0,0 +1,32 @@ +// The main suite of Protractor tests. +exports.config = { + seleniumAddress: 'http://localhost:4444/wd/hub', + + // Spec patterns are relative to this directory. + specs: [ + 'basic/*_spec.js' + ], + + // Exclude patterns are relative to this directory. + exclude: [ + 'basic/exclude*.js' + ], + + chromeOnly: false, + + splitTestsBetweenCapabilities: true, + multiCapabilities: [{ + 'browserName': 'chrome', + count: 2 + }], + + baseUrl: 'http://localhost:8000', + + params: { + login: { + user: 'Jane', + password: '1234' + } + } +}; +