diff --git a/lib/actor.js b/lib/actor.js index 76be1965c..6c1193419 100644 --- a/lib/actor.js +++ b/lib/actor.js @@ -61,10 +61,17 @@ function recordStep(step, args) { val = step.run.apply(step, args); }); + // ensure step result is passed to next step + recorder.add('return step result', () => val); + + //return execution time for logging + recorder.add(`return execution time`, () => { + output.stepExecutionTime(step, global.suiteDetails); + return val; + }); + // run async after step hooks event.emit(event.step.after, step); - // ensure step result is passed to next step - recorder.add('return step result', () => val); return recorder.promise(); } diff --git a/lib/command/run-multiple.js b/lib/command/run-multiple.js index ecb933dcd..3f2f5c963 100644 --- a/lib/command/run-multiple.js +++ b/lib/command/run-multiple.js @@ -93,6 +93,7 @@ function runSuite(suite, suiteConf, browser) { // tweaking default output directories and for mochawesome overriddenConfig = replaceValue(overriddenConfig, 'output', path.join(config.output, outputDir)); overriddenConfig = replaceValue(overriddenConfig, 'reportDir', path.join(config.output, outputDir)); + overriddenConfig = replaceValue(overriddenConfig, 'mochaFile', path.join(config.output, outputDir, 'report.xml')); // override grep param and collect all params let params = ['run', diff --git a/lib/event.js b/lib/event.js index 1c1d5d60b..0ac565695 100644 --- a/lib/event.js +++ b/lib/event.js @@ -11,6 +11,14 @@ module.exports = { passed: 'test.passed', failed: 'test.failed', }, + hook: { + afterSuite: { + failed: 'afterSuiteHook.failed', + }, + after: { + failed: 'afterTestHook.failed' + } + }, suite: { before: 'suite.before', after: 'suite.after' diff --git a/lib/helper.js b/lib/helper.js index 7644e7a73..f5c066746 100644 --- a/lib/helper.js +++ b/lib/helper.js @@ -105,6 +105,15 @@ class Helper { } + /** + * Hook executed after all tests are executed + * + * @param suite + */ + _finishTest() { + + } + /** * Access another configured helper: `this.helpers['AnotherHelper']` */ diff --git a/lib/helper/Appium.js b/lib/helper/Appium.js index 8d4162165..a00e76c7f 100644 --- a/lib/helper/Appium.js +++ b/lib/helper/Appium.js @@ -133,16 +133,6 @@ class Appium extends WebdriverIO { return this.browser; } - _after() { - if (this.options.restart) return this.browser.end(); - if (this.options.desiredCapabilities.browserName) { - this.debugSection('Session', 'cleaning cookies and localStorage'); - return this.browser.execute('localStorage.clear();').then(() => { - return this.browser.deleteCookie(); - }); - } - } - /** * Check if an app is installed. * diff --git a/lib/helper/Nightmare.js b/lib/helper/Nightmare.js index ea87aadf2..73669bb17 100644 --- a/lib/helper/Nightmare.js +++ b/lib/helper/Nightmare.js @@ -55,6 +55,8 @@ class Nightmare extends Helper { js_errors: null }; + this.isRunning = false; + // override defaults with config Object.assign(this.options, config); @@ -172,7 +174,8 @@ class Nightmare extends Helper { } _beforeSuite() { - if (!this.options.restart) { + if (!this.options.restart && !this.isRunning) { + this.isRunning = true; return this._startBrowser(); } } @@ -189,10 +192,13 @@ class Nightmare extends Helper { return this._stopBrowser(); } if (this.options.keepCookies) return; - return this.browser.cookies.clearAll(); + return Promise.all([this.browser.cookies.clearAll(), this.executeScript('localStorage.clear();')]); } _afterSuite() { + } + + _finishTest() { if (!this.options.restart) { this._stopBrowser(); } @@ -822,9 +828,9 @@ class Nightmare extends Helper { }, lctype(locator), lcval(locator)); } - /** - * {{> ../webapi/saveScreenshot }} - */ + /** + * {{> ../webapi/saveScreenshot }} + */ saveScreenshot(fileName, fullPage = this.options.fullPageScreenshots) { let outputFile = path.join(global.output_dir, fileName); this.debug('Screenshot is saving to ' + outputFile); @@ -837,9 +843,9 @@ class Nightmare extends Helper { height: document.body.scrollHeight, width: document.body.scrollWidth })).then(({ - width, - height - }) => { + width, + height + }) => { this.browser.viewport(width, height); return this.browser.screenshot(outputFile); }); @@ -907,7 +913,7 @@ function *proceedSeeInField(assertType, field, value) { let fieldVal = yield this.browser.evaluate(function (el) { return codeceptjs.fetchElement(el).value; } - , el); + , el); if (tag == 'select') { // locate option by values and check them let text = yield this.browser.evaluate(function (el, val) { diff --git a/lib/helper/SeleniumWebdriver.js b/lib/helper/SeleniumWebdriver.js index cdacf5d07..f3c105c99 100644 --- a/lib/helper/SeleniumWebdriver.js +++ b/lib/helper/SeleniumWebdriver.js @@ -78,6 +78,9 @@ class SeleniumWebdriver extends Helper { manualStart: false, capabilities: {} }; + + this.isRunning = false; + if (this.options.waitforTimeout) { console.log(`waitforTimeout is deprecated in favor of waitForTimeout, please update config`); this.options.waitForTimeout = this.options.waitforTimeout; @@ -132,8 +135,9 @@ class SeleniumWebdriver extends Helper { } _beforeSuite() { - if (!this.options.restart && !this.options.manualStart) { + if (!this.options.restart && !this.options.manualStart && !this.isRunning) { this.debugSection('Session', 'Starting singleton browser session'); + this.isRunning = true; return this._startBrowser(); } } @@ -146,13 +150,16 @@ class SeleniumWebdriver extends Helper { _after() { if (this.options.restart) return this.browser.quit(); - if (this.options.keepCookies) return; + if (this.options.keepCookies) return Promise.all([this.browser.executeScript('localStorage.clear();'), this.closeOtherTabs()]); // if browser should not be restarted this.debugSection('Session', 'cleaning cookies and localStorage'); - return Promise.all([this.browser.manage().deleteAllCookies(), this.browser.executeScript('localStorage.clear();')]); + return Promise.all([this.browser.manage().deleteAllCookies(), this.browser.executeScript('localStorage.clear();'), this.closeOtherTabs()]); } _afterSuite() { + } + + _finishTest() { if (!this.options.restart) return this.browser.quit(); } @@ -667,6 +674,27 @@ class SeleniumWebdriver extends Helper { return this.browser.manage().window().setSize(width, height); } + /** + * Close all tabs expect for one. + * + * ```js + * I.closeOtherTabs(); + * ``` + */ + closeOtherTabs() { + let client = this.browser; + + return client.getAllWindowHandles().then(function (handles){ + for (var i = 1; i < handles.length; i++) { + client.switchTo().window(handles[i]).then(function (){ + client.close(); + }); + } + return client.switchTo().window(handles[0]); + }); + } + + /** * {{> ../webapi/wait }} */ diff --git a/lib/helper/WebDriverIO.js b/lib/helper/WebDriverIO.js index 26ef1f815..2e2ea34c9 100644 --- a/lib/helper/WebDriverIO.js +++ b/lib/helper/WebDriverIO.js @@ -210,6 +210,8 @@ class WebDriverIO extends Helper { } }; + this.isRunning = false; + // override defaults with config Object.assign(this.options, config); @@ -256,8 +258,9 @@ class WebDriverIO extends Helper { } _beforeSuite() { - if (!this.options.restart && !this.options.manualStart) { + if (!this.options.restart && !this.options.manualStart && !this.isRunning) { this.debugSection('Session', 'Starting singleton browser session'); + this.isRunning = true; return this._startBrowser(); } } @@ -295,12 +298,17 @@ class WebDriverIO extends Helper { _after() { if (this.options.restart) return this.browser.end(); - if (this.options.keepCookies) return; - this.debugSection('Session', 'cleaning cookies and localStorage'); - return Promise.all([this.browser.deleteCookie(), this.browser.execute('localStorage.clear();')]); + if (this.options.keepCookies) return Promise.all([this.browser.execute('localStorage.clear();'), this.closeOtherTabs()]); + if (this.options.desiredCapabilities.browserName) { + this.debugSection('Session', 'cleaning cookies and localStorage'); + return Promise.all([this.browser.deleteCookie(), this.browser.execute('localStorage.clear();'), this.closeOtherTabs()]); + } } _afterSuite() { + } + + _finishTest() { if (!this.options.restart) return this.browser.end(); } @@ -922,10 +930,10 @@ class WebDriverIO extends Helper { return this.browser.saveScreenshot(outputFile); } return this.browser.execute(function () { - return ({ + return { height: document.body.scrollHeight, width: document.body.scrollWidth - }) + }; }).then(({ width, height @@ -1088,6 +1096,24 @@ class WebDriverIO extends Helper { ); } + /** + * Close all tabs expect for one. + * Appium: support web test + * + * ```js + * I.closeOtherTabs(); + * ``` + */ + closeOtherTabs() { + let client = this.browser; + return client.getTabIds().then(function (handles) { + for (var i = 1; i < handles.length; i++) { + this.close(handles[i]); + } + return this.switchTab(); + }); + } + /** * {{> ../webapi/wait }} * Appium: support @@ -1122,13 +1148,13 @@ class WebDriverIO extends Helper { sec = sec || this.options.waitForTimeout; context = context || 'body'; return this.browser.waitUntil(function () { - return this.getText(withStrictLocator(context)).then(function (source) { - if (Array.isArray(source)) { - return source.filter(part => part.indexOf(text) >= 0).length > 0; - } - return source.indexOf(text) >= 0; - }); - }, sec * 1000) + return this.getText(withStrictLocator(context)).then(function (source) { + if (Array.isArray(source)) { + return source.filter(part => part.indexOf(text) >= 0).length > 0; + } + return source.indexOf(text) >= 0; + }); + }, sec * 1000) .catch((e) => { if (e.type === 'WaitUntilTimeoutError') { return proceedSee.call(this, 'assert', text, withStrictLocator(context)); @@ -1363,15 +1389,15 @@ function withStrictLocator(locator) { locator.toString = () => `{${key}: '${value}'}`; switch (key) { - case 'by': - case 'xpath': - return value; - case 'css': - return value; - case 'id': - return '#' + value; - case 'name': - return `[name="${value}"]`; + case 'by': + case 'xpath': + return value; + case 'css': + return value; + case 'id': + return '#' + value; + case 'name': + return `[name="${value}"]`; } } diff --git a/lib/interfaces/bdd.js b/lib/interfaces/bdd.js index 99cabd8a8..3a8e8fa35 100644 --- a/lib/interfaces/bdd.js +++ b/lib/interfaces/bdd.js @@ -24,6 +24,7 @@ var escapeRe = require('escape-string-regexp'); module.exports = function (suite) { var suites = [suite]; suite.timeout(0); + var afterAllHooks, afterEachHooks, afterAllHooksAreLoaded, afterEachHooksAreLoaded; suite.on('pre-require', function (context, file, mocha) { var common = require('mocha/lib/interfaces/common')(suites, context); @@ -45,6 +46,12 @@ module.exports = function (suite) { suites.shift(); } if (!opts) opts = {}; + + afterAllHooks = []; + afterEachHooks = []; + afterAllHooksAreLoaded = false; + afterEachHooksAreLoaded = false; + var suite = Suite.create(suites[0], title); suite.timeout(0); @@ -54,28 +61,28 @@ module.exports = function (suite) { suite.file = file; suites.unshift(suite); suite.beforeEach('codeceptjs.before', scenario.setup); - suite.afterEach('finialize codeceptjs', scenario.teardown); + afterEachHooks.push(['finialize codeceptjs', scenario.teardown]); suite.beforeAll('codeceptjs.beforeSuite', () => scenario.suiteSetup(suite)); - suite.afterAll('codeceptjs.afterSuite', () => scenario.suiteTeardown(suite)); + afterAllHooks.push(['codeceptjs.afterSuite', () => scenario.suiteTeardown(suite)]); return suite; }; context.BeforeSuite = function (fn) { - suites[0].beforeAll('BeforeSuite', scenario.injected(fn, suites[0])); + suites[0].beforeAll('BeforeSuite', scenario.injected(fn, suites[0], 'beforeSuite')); }; context.AfterSuite = function (fn) { - suites[0].afterAll('AfterSuite', scenario.injected(fn, suites[0])); + afterAllHooks.unshift(['AfterSuite', scenario.injected(fn, suites[0], 'afterSuite')]); }; context.Background = context.Before = function (fn) { - suites[0].beforeEach('Before', scenario.injected(fn, suites[0])); + suites[0].beforeEach('Before', scenario.injected(fn, suites[0], 'before')); }; context.After = function (fn) { - suites[0].afterEach('After', scenario.injected(fn, suites[0])); + afterEachHooks.unshift(['After', scenario.injected(fn, suites[0]), 'after']); }; /** @@ -88,7 +95,22 @@ module.exports = function (suite) { fn = opts; opts = {}; } - + /** + * load hooks from arrays to suite to prevent reordering + * + */ + if (!afterAllHooksAreLoaded) { + afterAllHooks.forEach(function (hook) { + suites[0].afterAll(hook[0], hook[1]); + }); + afterAllHooksAreLoaded = true; + } + if (!afterEachHooksAreLoaded) { + afterEachHooks.forEach(function (hook) { + suites[0].afterEach(hook[0], hook[1]); + }); + afterEachHooksAreLoaded = true; + } var suite = suites[0]; if (suite.pending) { fn = null; diff --git a/lib/listener/helpers.js b/lib/listener/helpers.js index 6f3619566..a8820bfdf 100644 --- a/lib/listener/helpers.js +++ b/lib/listener/helpers.js @@ -36,6 +36,14 @@ module.exports = function () { runAsyncHelpersHook('_afterSuite', suite, true); }); + event.dispatcher.on(event.hook.after.failed, function (suite) { + runAsyncHelpersHook('_after', suite, true); + }); + + event.dispatcher.on(event.hook.afterSuite.failed, function (suite) { + runAsyncHelpersHook('_afterSuite', suite, true); + }); + event.dispatcher.on(event.test.started, function (test) { runHelpersHook('_test', test); recorder.catch((e) => error(e)); @@ -48,7 +56,7 @@ module.exports = function () { event.dispatcher.on(event.test.failed, function (test) { runAsyncHelpersHook('_failed', test, true); // should not fail test execution, so errors should be catched - recorder.catch((e) => error(e)); + recorder.catchWithoutStop((e) => error(e)); }); event.dispatcher.on(event.test.after, function () { @@ -62,4 +70,8 @@ module.exports = function () { event.dispatcher.on(event.step.after, function (step) { runAsyncHelpersHook('_afterStep', step); }); + + event.dispatcher.on(event.all.result, function () { + runAsyncHelpersHook('_finishTest', {}, true); + }); }; diff --git a/lib/listener/steps.js b/lib/listener/steps.js index d73ff169f..8aa53d18c 100644 --- a/lib/listener/steps.js +++ b/lib/listener/steps.js @@ -1,5 +1,7 @@ 'use strict'; const event = require('../event'); +const recorder = require('../recorder'); +const output = require('../output'); let currentTest; let steps; diff --git a/lib/output.js b/lib/output.js index fc7aa2d5b..58f87c9f6 100644 --- a/lib/output.js +++ b/lib/output.js @@ -14,6 +14,7 @@ let styles = { let outputLevel = 0; let outputProcess = ''; +let newline = true; module.exports = { colors, @@ -73,9 +74,24 @@ module.exports = { if (outputLevel === 0) return; if (!step) return; let sym = process.platform === 'win32' ? '*' : '•'; + if (outputLevel >= 2) { + newline = false; + return process.stdout.write(`${' '.repeat(this.stepShift)} ${sym} ${step.toString()}`); + } print(' '.repeat(this.stepShift), `${sym} ${step.toString()}`); }, + /** + * Print a step execution time + */ + stepExecutionTime: function (step) { + if (outputLevel < 2) return; + if (!step) return; + step.endTime = new Date(); + newline = true; + process.stdout.write(` ${styles.debug(" (" + (step.endTime - step.startTime) / 1000 + " sec)\n")}`); + }, + suite: { started: (suite) => { if (!suite.title) return; @@ -140,6 +156,10 @@ function print(...msg) { if (outputProcess) { msg.unshift(outputProcess); } + if (!newline) { + console.log(); + newline = true; + } console.log.apply(this, msg); } diff --git a/lib/recorder.js b/lib/recorder.js index d6d976d66..2a69345d3 100644 --- a/lib/recorder.js +++ b/lib/recorder.js @@ -26,6 +26,16 @@ module.exports = { this.reset(); }, + isRunning() { + return running; + }, + + startUnlessRunning() { + if (!this.isRunning()) { + this.start(); + } + }, + /** * Add error handler to catch rejected promises * @@ -115,6 +125,20 @@ module.exports = { }); }, + catchWithoutStop(customErrFn) { + return promise = promise.catch((err) => { + log(currentQueue() + `Error | ${err}`); + if (!(err instanceof Error)) { // strange things may happen + err = new Error('[Wrapped Error] ' + JSON.stringify(err)); // we should be prepared for them + } + if (customErrFn) { + customErrFn(err); + } else if (errFn) { + errFn(err); + } + }); + }, + /** * Adds a promise which throws an error into a chain * @@ -167,4 +191,3 @@ function currentQueue() { if (sessionId) session = `<${sessionId}> `; return `[${queueId}] ${session}`; } - diff --git a/lib/scenario.js b/lib/scenario.js index 6bdbaca2b..856cebf32 100644 --- a/lib/scenario.js +++ b/lib/scenario.js @@ -71,15 +71,19 @@ module.exports.test = (test) => { /** * Injects arguments to function from controller */ -module.exports.injected = function (fn, suite) { +module.exports.injected = function (fn, suite, hookName) { return function () { try { + recorder.startUnlessRunning(); fn.apply(this, getInjectedArguments(fn)); } catch (err) { recorder.throw(err); } recorder.catch((err) => { event.emit(event.test.failed, suite, err); // emit + if (hookName === 'after' || hookName === 'afterSuite') { + event.emit(event.hook[hookName].failed, suite); + } throw err; }); return recorder.promise(); @@ -90,21 +94,22 @@ module.exports.injected = function (fn, suite) { * Starts promise chain, so helpers could enqueue their hooks */ module.exports.setup = function () { - recorder.start(); + recorder.startUnlessRunning(); event.emit(event.test.before); }; module.exports.teardown = function () { + recorder.startUnlessRunning(); event.emit(event.test.after); }; module.exports.suiteSetup = function (suite) { - recorder.start(); + recorder.startUnlessRunning(); event.emit(event.suite.before, suite); }; module.exports.suiteTeardown = function (suite) { - recorder.start(); + recorder.startUnlessRunning(); event.emit(event.suite.after, suite); }; diff --git a/lib/step.js b/lib/step.js index 055a3da93..4c1135dae 100644 --- a/lib/step.js +++ b/lib/step.js @@ -22,6 +22,7 @@ class Step { } run() { + this.startTime = new Date(); this.args = Array.prototype.slice.call(arguments); let result; try { diff --git a/test/runner/interface_test.js b/test/runner/interface_test.js index 553104328..70ba75b74 100644 --- a/test/runner/interface_test.js +++ b/test/runner/interface_test.js @@ -30,27 +30,20 @@ describe('CodeceptJS Interface', () => { var queue1 = stdout.match(/\[1\] .+/g); queue1.should.eql([ "[1] Starting recording promises", - "[1] Queued | hook FileSystem._beforeSuite()" - ]); - - var queue2 = stdout.match(/\[2\] .+/g); - queue2.should.eql([ - `[2] Starting recording promises`, - `[2] Queued | hook FileSystem._before()`, - `[2] Queued | amInPath: "."`, - `[2] Queued | return step result`, - `[2] Queued | say hello world`, - `[2] Queued | seeFile: "codecept.json"`, - `[2] Queued | return step result`, - `[2] Queued | fire test.passed`, - `[2] Queued | finish test`, - `[2] Queued | hook FileSystem._after()` - ]); - - var queue3 = stdout.match(/\[3\] .+/g); - queue3.should.eql([ - `[3] Starting recording promises`, - `[3] Queued | hook FileSystem._afterSuite()` + "[1] Queued | hook FileSystem._beforeSuite()", + `[1] Queued | hook FileSystem._before()`, + `[1] Queued | amInPath: "."`, + `[1] Queued | return step result`, + `[1] Queued | return execution time`, + `[1] Queued | say hello world`, + `[1] Queued | seeFile: "codecept.json"`, + `[1] Queued | return step result`, + `[1] Queued | return execution time`, + `[1] Queued | fire test.passed`, + `[1] Queued | finish test`, + `[1] Queued | hook FileSystem._after()`, + `[1] Queued | hook FileSystem._afterSuite()`, + `[1] Queued | hook FileSystem._finishTest()`, ]); let lines = stdout.match(/\S.+/g); @@ -58,7 +51,6 @@ describe('CodeceptJS Interface', () => { // before hooks let beforeStep = [ `Emitted | step.before (I am in path ".")`, - `[2] Queued | amInPath: "."`, `Emitted | step.after (I am in path ".")`, `Emitted | step.start (I am in path ".")`, `• I am in path "."` @@ -82,4 +74,4 @@ describe('CodeceptJS Interface', () => { }); }); -}); \ No newline at end of file +});