Permalink
Browse files

enable to abort wdio test run after specific amount of failures - clo…

…ses #1750
  • Loading branch information...
1 parent 5554a08 commit 055ab6257678b8f9ff3d33d610b75ce4676335e9 @christian-bromann christian-bromann committed Dec 7, 2016
@@ -115,6 +115,12 @@ Enables colors for log output
Type: `Boolean`<br>
Default: *true*
+### bail
+If you only want to run your tests until a specific amount of tests have failed use bail (default is 0 - don't bail, run all tests).
+
+Type: `Number`<br>
+Default: *0* (don't bail, run all tests)
+
### screenshotPath
Saves a screenshot to a given path if the Selenium driver crashes
@@ -123,6 +123,10 @@ exports.config = {
// Enables colors for log output.
coloredLogs: true,
//
+ // If you only want to run your tests until a specific amount of tests have failed use
+ // bail (default is 0 - don't bail, run all tests).
+ bail: 0,
+ //
// Saves a screenshot to a given path if a command fails.
screenshotPath: 'shots',
//
@@ -106,3 +106,7 @@ $ wdio wdio.conf.js --spec ./test/specs/e2e/login.js
```
Note that each test file is running in a single test runner process. Since we don't scan files in advance you _can't_ use for example `describe.only` at the top of your spec file to say Mocha to only run that suite. This feature will help you though to do that in the same way.
+
+## Stop testing after failure
+
+With the `bail` option you can specify when WebdriverIO should stop the test run after test failures. This can be helpful when you have a big test suite and want to avoid long test runs when you already know that your build will break. The option expects a number that specifies after how many spec failures it should stop the whole test run. The default is `0` meaning that it always runs all tests specs it can find.
@@ -103,9 +103,13 @@ exports.config = {
screenshotPath: 'shots',
//
// Set a base URL in order to shorten url command calls. If your url parameter starts
- // with "/", then the base url gets prepended.
+ // with "/", then the base url gets prepended.
baseUrl: 'http://localhost:8080',
//
+ // If you only want to run your tests until a specific amount of tests have failed use
+ // bail (default is 0 - don't bail, run all tests).
+ bail: 0,
+ //
// Default timeout for all waitFor* commands.
waitforTimeout: 1000,
//
View
@@ -71,6 +71,7 @@ optimist
.alias('logLevel', 'l')
.describe('coloredLogs', 'if true enables colors for log output (default: true)')
.alias('coloredLogs', 'c')
+ .describe('bail', 'stop test runner after specific amount of tests have failed (default: 0 - don\'t bail)')
.describe('screenshotPath', 'saves a screenshot to a given path if a command fails')
.alias('screenshotPath', 's')
.describe('baseUrl', 'shorten url command calls by setting a base url')
@@ -91,6 +91,10 @@ exports.config = {
// Enables colors for log output.
coloredLogs: true,
//
+ // If you only want to run your tests until a specific amount of tests have failed use
+ // bail (default is 0 - don't bail, run all tests).
+ bail: 0,
+ //
// Saves a screenshot to a given path if a command fails.
screenshotPath: '<%= answers.screenshotPath %>',
//
View
@@ -22,6 +22,7 @@ class Launcher {
this.schedule = []
this.rid = []
this.processesStarted = 0
+ this.runnerFailed = 0
}
/**
@@ -205,6 +206,22 @@ class Launcher {
while (this.getNumberOfRunningInstances() < config.maxInstances) {
let schedulableCaps = this.schedule
/**
+ * bail if number of errors exceeds allowed
+ */
+ .filter(() => {
+ const filter = typeof config.bail !== 'number' || config.bail < 1 ||
+ config.bail > this.runnerFailed
+
+ /**
+ * clear number of specs when filter is false
+ */
+ if (!filter) {
+ this.schedule.forEach((t) => { t.specs = [] })
+ }
+
+ return filter
+ })
+ /**
* make sure complete number of running instances is not higher than general maxInstances number
*/
.filter((a) => this.getNumberOfRunningInstances() < config.maxInstances)
@@ -346,6 +363,7 @@ class Launcher {
*/
endHandler (rid, childProcessExitCode) {
this.exitCode = this.exitCode || childProcessExitCode
+ this.runnerFailed += this.exitCode !== 0 ? 1 : 0
// Update schedule now this process has ended
if (!this.isMultiremote()) {
@@ -22,6 +22,7 @@ const DEFAULT_CONFIGS = {
logLevel: 'silent',
coloredLogs: true,
baseUrl: null,
+ bail: 0,
waitforInterval: 500,
waitforTimeout: 1000,
framework: 'mocha',
@@ -162,6 +162,35 @@ describe('launcher', () => {
launcher.startInstance.callCount.should.be.equal(0)
})
+ it('should stop launching runners after bail number is reached', async () => {
+ launcher = getLauncer({
+ maxInstances: 1,
+ bail: 2
+ }, 5)
+ setTimeout(() => launcher.resolve(0), 10)
+
+ await launcher.run()
+ launcher.startInstance.callCount.should.be.equal(1)
+ launcher.schedule.forEach((cap) => {
+ cap.runningInstances = 0
+ cap.availableInstances = 1
+ })
+
+ launcher.runSpecs().should.be.not.ok
+ launcher.runnerFailed++
+ launcher.schedule.forEach((cap) => {
+ cap.runningInstances = 0
+ cap.availableInstances = 1
+ })
+ launcher.runSpecs().should.be.not.ok
+ launcher.runnerFailed++
+ launcher.schedule.forEach((cap) => {
+ cap.runningInstances = 0
+ cap.availableInstances = 1
+ })
+ launcher.runSpecs().should.be.ok
+ })
+
afterEach(() => {
launcher.startInstance.reset()
})

0 comments on commit 055ab62

Please sign in to comment.