fix quarantine for live mode (closes DevExpress#4093)

src/live/test-run-controller.js

@@ -1,5 +1,5 @@
 import EventEmitter from 'events';
-import noop from 'lodash';
+import { noop, uniq } from 'lodash';
 import { TestRunCtorFactory } from './test-run';
 import TEST_RUN_STATE from './test-run-state';
 
@@ -66,7 +66,6 @@ class LiveModeTestRunController extends EventEmitter {
     }
 
     _onTestRunCreated (testRun) {
-
         this.allTestsCompletePromise = new Promise(resolve => {
             this.completeAllRunningTests = resolve;
         });
@@ -81,25 +80,28 @@ class LiveModeTestRunController extends EventEmitter {
     _onTestRunDone (testRun) {
         testRun.state = TEST_RUN_STATE.done;
 
-        const hasRunningTests = this._getTestRuns().some(t => t.state !== TEST_RUN_STATE.done);
+        const testWillBeRestarted            = !this.isTestFinished(testRun);
+        const hasRunningTestsInOtherBrowsers = this._getTestRuns().some(t => t.state !== TEST_RUN_STATE.done);
 
-        if (!hasRunningTests)
+        if (!hasRunningTestsInOtherBrowsers && !testWillBeRestarted)
             this.emit('all-tests-complete');
 
         const browserTestRuns = this.testRuns[testRun.browserConnection.id];
+        const tests           = uniq(browserTestRuns.map(t => t.test));
 
         testRun.readyToNextPromise = new Promise(resolve => {
             testRun.setReadyToNext = resolve;
         });
 
+        const isLastTestRun = tests.length >= this.expectedTestCount;
 
-        if (browserTestRuns.length < this.expectedTestCount || browserTestRuns.some(t => t.state !== TEST_RUN_STATE.done))
+        if (testWillBeRestarted || !isLastTestRun)
             return Promise.resolve();
 
         return new Promise(resolve => {
             testRun.finish = () => {
                 testRun.finish = null;
-                testRun.state  = TEST_RUN_STATE.done;
+
                 resolve();
             };
         });
@@ -108,6 +110,15 @@ class LiveModeTestRunController extends EventEmitter {
     _onTestRunReadyToNext (testRun) {
         testRun.setReadyToNext();
     }
+
+    isTestFinished (testRun) {
+        const { quarantine, errs } = testRun;
+
+        if (!quarantine)
+            return true;
+
+        return quarantine.isFirstAttemptSuccessfull(errs) || quarantine.isThresholdReached(errs);
+    }
 }
 
 export default LiveModeTestRunController;

src/runner/test-run-controller.js

@@ -32,6 +32,12 @@ class Quarantine {
         return failedThresholdReached || passedThresholdReached;
     }
 
+    isFirstAttemptSuccessfull (extraErrors) {
+        const { failedTimes, passedTimes } = this._getAttemptsResult(extraErrors);
+
+        return failedTimes === 0 && passedTimes > 0;
+    }
+
     _getAttemptsResult (extraErrors) {
         let failedTimes = this.getFailedAttempts().length;
         let passedTimes = this.getPassedAttempts().length;

test/functional/fixtures/live/test-helper.js

@@ -6,7 +6,9 @@ class TestCompleteWatcher extends EventEmitter {
 }
 
 module.exports = {
-    watcher:   new TestCompleteWatcher(),
-    counter:   0,
-    testCount: 10
+    watcher:             new TestCompleteWatcher(),
+    counter:             0,
+    attempts:            0,
+    testCount:           10,
+    quarantineThreshold: 3
 };

test/functional/fixtures/live/test.js

@@ -1,28 +1,46 @@
+const { noop }       = require('lodash');
 const createTestCafe = require('../../../../lib');
 const config         = require('../../config');
 const path           = require('path');
 const { expect }     = require('chai');
 const helper         = require('./test-helper');
 
+const browsers = ['chrome', 'firefox'];
+let cafe       = null;
+
+function createRunner (tc, src) {
+    const runner = tc.createLiveModeRunner();
+
+    runner.controller._listenKeyPress   = () => { };
+    runner.controller._initFileWatching = () => { };
+    runner.controller.dispose           = () => { };
+
+    src = path.join(__dirname, src);
+
+    return runner
+        .src(src)
+        .browsers(browsers)
+        .reporter(() => {
+            return {
+                reportTaskStart:    noop,
+                reportTaskDone:     noop,
+                reportTestDone:     noop,
+                reportFixtureStart: noop
+            };
+        });
+}
 
 if (config.useLocalBrowsers && !config.useHeadlessBrowsers) {
-    describe('Live smoke', () => {
+    describe('Live Mode tests', () => {
         it('Live smoke', () => {
-            let cafe       = null;
-            const browsers = ['chrome', 'firefox'];
             const runCount = 2;
 
             return createTestCafe('127.0.0.1', 1335, 1336)
                 .then(tc => {
                     cafe = tc;
                 })
                 .then(() => {
-                    const runner      = cafe.createLiveModeRunner();
-                    const fixturePath = path.join(__dirname, '/testcafe-fixtures/index-test.js');
-
-                    runner.controller._listenKeyPress   = () => { };
-                    runner.controller._initFileWatching = () => { };
-                    runner.controller.dispose           = () => { };
+                    const runner = createRunner(cafe, '/testcafe-fixtures/smoke.js');
 
                     helper.watcher.once('test-complete', () => {
                         setTimeout(() => {
@@ -33,16 +51,38 @@ if (config.useLocalBrowsers && !config.useHeadlessBrowsers) {
                         }, 1000);
                     });
 
-                    return runner
-                        .src(fixturePath)
-                        .browsers(browsers)
-                        .run();
+                    return runner.run();
                 })
                 .then(() => {
                     expect(helper.counter).eql(browsers.length * helper.testCount * runCount);
 
                     return cafe.close();
                 });
         });
+
+        it('Live quarantine', () => {
+            return createTestCafe('127.0.0.1', 1335, 1336)
+                .then(tc => {
+                    cafe = tc;
+                })
+                .then(() => {
+                    const runner = createRunner(cafe, '/testcafe-fixtures/quarantine.js');
+
+                    helper.watcher.once('test-complete', () => {
+                        setTimeout(() => {
+                            runner.exit();
+                        }, 1000);
+                    });
+
+                    return runner.run({
+                        quarantineMode: true
+                    });
+                })
+                .then(() => {
+                    expect(helper.attempts).eql(browsers.length * helper.quarantineThreshold);
+
+                    return cafe.close();
+                });
+        });
     });
 }

test/functional/fixtures/live/testcafe-fixtures/quarantine.js

@@ -0,0 +1,14 @@
+import helper from '../test-helper';
+
+fixture `Live`
+    .page `../pages/index.html`
+    .afterEach(() => {
+        helper.attempts++;
+    })
+    .after(() => {
+        helper.watcher.emit('test-complete');
+    });
+
+test('quarantine', async () => {
+    throw new Error('error');
+});