From 456af87334d7441ba217c93548167b12fe969f52 Mon Sep 17 00:00:00 2001 From: Harshit Agrawal <94462364+harshit-bs@users.noreply.github.com> Date: Tue, 16 May 2023 18:30:50 +0530 Subject: [PATCH] Skipped tests have categorized as either user-marked or runtime-failed (#3711) * splitted skipped tests marked by user and failed at runtime --------- Co-authored-by: Binayak Ghosh --- lib/reporter/base-reporter.js | 6 ++- lib/reporter/index.js | 4 +- lib/reporter/reporters/html.js | 17 +++++-- lib/reporter/reporters/junit.xml.ejs | 8 +-- lib/reporter/results.js | 21 ++++---- lib/reporter/summary.js | 15 ++++-- lib/testsuite/context.js | 27 ++++++++++ lib/testsuite/index.js | 6 ++- test/extra/reportObject.js | 64 ++++++++++++++++++++++++ test/src/runner/testRunnerJunitOutput.js | 5 ++ 10 files changed, 147 insertions(+), 26 deletions(-) diff --git a/lib/reporter/base-reporter.js b/lib/reporter/base-reporter.js index 03f49e38a4..bcc6322f42 100644 --- a/lib/reporter/base-reporter.js +++ b/lib/reporter/base-reporter.js @@ -11,7 +11,7 @@ module.exports = class BaseReporter { adaptAssertions(module) { module.completed = Object.keys(module.completed).reduce(function(prev, item) { - if (module.skipped.includes(item)) { + if ((module.skipped && module.skipped.includes(item))) { return prev; } @@ -39,7 +39,9 @@ module.exports = class BaseReporter { }, {}); module.completedSections = Object.keys(module.completedSections).reduce(function(prev, item) { - if (module.skipped && module.skipped.includes(item)) { + if ((module.skippedAtRuntime && module.skippedAtRuntime.includes(item)) || + (module.skippedByUser && module.skippedByUser.includes(item)) + ) { return prev; } diff --git a/lib/reporter/index.js b/lib/reporter/index.js index e2085f3bda..7e6b03e873 100644 --- a/lib/reporter/index.js +++ b/lib/reporter/index.js @@ -22,12 +22,12 @@ class Reporter extends SimplifiedReporter { * @param {Object} settings * @param {Object} addOpts */ - constructor({settings, tests, suiteRetries, addOpts = {}}) { + constructor({settings, tests, suiteRetries, addOpts = {}, skippedTests, allScreenedTests}) { super(settings); this.suiteRetries = suiteRetries; this.suiteName = addOpts.suiteName; - this.testResults = new TestResults(tests, addOpts, settings); + this.testResults = new TestResults(tests, addOpts, settings, skippedTests, allScreenedTests); this.currentContext = null; this.__printA11Report = false; this.reporter = addOpts.repoter; diff --git a/lib/reporter/reporters/html.js b/lib/reporter/reporters/html.js index 0e4387e19d..b993428847 100644 --- a/lib/reporter/reporters/html.js +++ b/lib/reporter/reporters/html.js @@ -198,12 +198,21 @@ class HtmlReporter extends BaseReporter { } } - // Add skipped tests in completed section - for (const skippedTestName of module.skipped) { + // Add skipped tests by user in completed section + module.skippedByUser && module.skippedByUser.forEach(skippedTestName => { module.completedSections[skippedTestName] = { - status: 'skip' + status: 'skip', + runtimeFailure: false }; - } + }); + + // Add skipped tests at runtime in completed section + module.skippedAtRuntime && module.skippedAtRuntime.forEach(skippedTestName => { + module.completedSections[skippedTestName] = { + status: 'skip', + runtimeFailure: true + }; + }); return result; } diff --git a/lib/reporter/reporters/junit.xml.ejs b/lib/reporter/reporters/junit.xml.ejs index 68af6bf2b1..d6330195b4 100644 --- a/lib/reporter/reporters/junit.xml.ejs +++ b/lib/reporter/reporters/junit.xml.ejs @@ -4,7 +4,7 @@ tests="<%= module.tests %>"> <% for (var item in module.completed) { var testcase = module.completed[item]; @@ -34,10 +34,10 @@ <%= module.lastError.stack %> ]]> <% } %> - <% if (module.skipped && (module.skipped.length > 0)) { %> - <% for (var j = 0; j < module.skipped.length; j++) { %> + <% if (module.skippedAtRuntime && (module.skippedAtRuntime.length > 0)) { %> + <% for (var j = 0; j < module.skippedAtRuntime.length; j++) { %> + name="<%= module.skippedAtRuntime[j] %>" classname="<%= className %>"> <% } %> diff --git a/lib/reporter/results.js b/lib/reporter/results.js index 055a3a28c1..c88ea48fc9 100644 --- a/lib/reporter/results.js +++ b/lib/reporter/results.js @@ -16,8 +16,9 @@ module.exports = class Results { return 'skip'; } - constructor(tests = [], opts, settings) { - this.skipped = tests.slice(0); + constructor(tests = [], opts, settings, skippedTests=[], allScreenedTests=[]) { + this.skippedByUser = skippedTests; + this.skippedAtRuntime = tests.slice(0); this.testcases = {}; this.testSections = {}; this.suiteName = opts.suiteName; @@ -53,7 +54,7 @@ module.exports = class Results { this.tags = opts.tags || []; this.__retryTest = false; - this.initCount(tests); + this.initCount(allScreenedTests); } get initialResult() { @@ -81,7 +82,7 @@ module.exports = class Results { const currentTest = this.testcases[testName]; if (returnFullResult) { - currentTest.steps = this.skipped; + currentTest.steps = this.skippedAtRuntime; currentTest.stackTrace = this.stackTrace; currentTest.testcases = Object.keys(this.testcases).reduce((prev, key) => { prev[key] = Object.keys(this.testcases[key]).reduce((prevVal, prop) => { @@ -215,7 +216,9 @@ module.exports = class Results { } suiteResults.results.lastError = lastError; - suiteResults.results.skipped = this.skipped; + suiteResults.results.skippedAtRuntime = this.skippedAtRuntime; + suiteResults.results.skippedByUser = this.skippedByUser; + suiteResults.results.skipped = [...this.skippedAtRuntime, ...this.skippedByUser]; suiteResults.results.time = this.time; suiteResults.results.timeMs = this.timeMs; suiteResults.results.completed = this.testcases; @@ -408,11 +411,11 @@ module.exports = class Results { return this; } - initCount(tests) { + initCount(allScreenedTests) { this.__passedCount = 0; this.__failedCount = 0; this.__errorsCount = 0; - this.__skippedCount = tests.length; + this.__skippedCount = allScreenedTests.length; this.__testsCount = 0; this.__timestamp = new Date().toUTCString(); this.errmessages = []; @@ -504,9 +507,9 @@ module.exports = class Results { this.currentTestName = testName; this.testcases[testName] = this.createTestCaseResults(testcase); - const index = this.skipped.indexOf(testName); + const index = this.skippedAtRuntime.indexOf(testName); if (index > -1) { - this.skipped.splice(index, 1); + this.skippedAtRuntime.splice(index, 1); this.__skippedCount -= 1; } diff --git a/lib/reporter/summary.js b/lib/reporter/summary.js index bb24e57b05..a0aef38ef0 100644 --- a/lib/reporter/summary.js +++ b/lib/reporter/summary.js @@ -98,10 +98,19 @@ module.exports = class Summary { * @param {object} testSuite */ static printSkipped(testSuite) { - if (testSuite.skipped.length > 0) { + if (testSuite.skippedAtRuntime.length > 0) { // eslint-disable-next-line no-console - console.log(colors.cyan(' SKIPPED:')); - testSuite.skipped.forEach(function(testcase) { + console.log(colors.cyan(' SKIPPED (at runtime):')); + testSuite.skippedAtRuntime.forEach(function(testcase) { + // eslint-disable-next-line no-console + console.log(` - ${testcase}`); + }); + } + + if (testSuite.skippedByUser.length > 0) { + // eslint-disable-next-line no-console + console.log(colors.cyan(' SKIPPED (by user):')); + testSuite.skippedByUser.forEach(function(testcase) { // eslint-disable-next-line no-console console.log(` - ${testcase}`); }); diff --git a/lib/testsuite/context.js b/lib/testsuite/context.js index 3187860ccf..a358440062 100644 --- a/lib/testsuite/context.js +++ b/lib/testsuite/context.js @@ -74,6 +74,8 @@ class Context extends EventEmitter { this.__hooks = []; this.__testcases = []; + this.__skippedTestCases = []; + this.__allScreenedTests = []; this.__contextBinding = {}; this.__transforms = client ? await client.transforms : []; @@ -138,6 +140,22 @@ class Context extends EventEmitter { this.__testcases = value; } + get skippedTests() { + return this.__skippedTestCases; + } + + set skippedTests(value) { + this.__skippedTestCases = value; + } + + get allScreenedTests() { + return this.__allScreenedTests; + } + + set allScreenedTests(value) { + this.__allScreenedTests = value; + } + get hooks() { return this.__hooks; } @@ -297,11 +315,20 @@ class Context extends EventEmitter { // TODO: warn if test name already exists if (!skipTest) { this.tests.push(testName); + } else { + this.skippedTests.push(testName); } + this.addTestSuiteMethod(testName, testFn, describeInstance); if (runOnly) { this.__currentTestName = testName; + this.skippedTests = [...this.allScreenedTests]; + this.runOnly = true; + } else if (this.runOnly) { + this.skippedTests.push(testName); } + + this.allScreenedTests.push(testName); } /** diff --git a/lib/testsuite/index.js b/lib/testsuite/index.js index d254546fa5..58ec207614 100644 --- a/lib/testsuite/index.js +++ b/lib/testsuite/index.js @@ -202,7 +202,7 @@ class TestSuite { } const {suiteRetries, suiteName} = this; - const {tests, moduleKey, modulePath, groupName} = this.context; + const {tests, moduleKey, modulePath, groupName, skippedTests, allScreenedTests} = this.context; this.reporter = new Reporter({ settings: this.client.settings, @@ -218,7 +218,9 @@ class TestSuite { groupName, isMobile: this.client.api.isMobile(), tags: this.context.getTags() - } + }, + skippedTests, + allScreenedTests }); } diff --git a/test/extra/reportObject.js b/test/extra/reportObject.js index de6f2e94a0..b06d7c7991 100644 --- a/test/extra/reportObject.js +++ b/test/extra/reportObject.js @@ -12,6 +12,8 @@ module.exports = { reportPrefix: 'FIREFOX_111.0.1__', assertionsCount: 1, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '1.082', @@ -507,6 +509,8 @@ module.exports = { abortOnFailure: true, stack: '+ actual - expected\n\n [\n+ \'abortOnFailure\'\n- \'documents\',\n- \'strings\'\n ]\n at Assertion.assert (/Users/vaibhavsingh/Dev/nightwatch/lib/api/_loaders/static.js:112:34)\n at StaticAssert.assertFn (/Users/vaibhavsingh/Dev/nightwatch/lib/api/_loaders/static.js:146:17)\n at Proxy. (/Users/vaibhavsingh/Dev/nightwatch/lib/api/index.js:157:30)\n at DescribeInstance. (/Users/vaibhavsingh/Dev/nightwatch/examples/tests/chromeCDP_example.js:10:20)' }, + skippedAtRuntime: [ + ], skipped: [ ], time: '4.669', @@ -919,6 +923,8 @@ module.exports = { reportPrefix: 'FIREFOX_111.0.1__', assertionsCount: 2, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '1.908', @@ -1477,6 +1483,8 @@ module.exports = { waitFor: true, stack: 'Error\n at DescribeInstance. (/Users/vaibhavsingh/Dev/nightwatch/examples/tests/duckDuckGo.js:8:8)\n at Context.call (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/context.js:478:35)\n at TestCase.run (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/testcase.js:58:31)\n at Runnable.__runFn (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:759:80)\n at Runnable.run (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/runnable.js:126:21)\n at TestSuite.executeRunnable (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:912:49)\n at TestSuite.handleRunnable (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:927:33)\n at /Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:759:21\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async DefaultRunner.runTestSuite (/Users/vaibhavsingh/Dev/nightwatch/lib/runner/test-runners/default.js:78:7)' }, + skippedAtRuntime: [ + ], skipped: [ ], time: '5.777', @@ -2069,6 +2077,9 @@ module.exports = { reportPrefix: '', assertionsCount: 0, lastError: null, + skippedAtRuntime: [ + 'should complete the consent form' + ], skipped: [ 'should complete the consent form' ], @@ -2118,6 +2129,8 @@ module.exports = { reportPrefix: 'FIREFOX_111.0.1__', assertionsCount: 2, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '10.68', @@ -2833,6 +2846,9 @@ module.exports = { reportPrefix: '', assertionsCount: 0, lastError: null, + skippedAtRuntime: [ + 'retrieve the shadowRoot' + ], skipped: [ 'retrieve the shadowRoot' ], @@ -2882,6 +2898,8 @@ module.exports = { reportPrefix: 'FIREFOX_111.0.1__', assertionsCount: 3, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '1.251', @@ -3467,6 +3485,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 5, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '3.056', @@ -4166,6 +4186,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 5, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '0.2180', @@ -4926,6 +4948,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 1, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '2.865', @@ -5417,6 +5441,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 1, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '4.659', @@ -5732,6 +5758,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 2, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '3.868', @@ -6292,6 +6320,8 @@ module.exports = { waitFor: true, stack: 'Error\n at DescribeInstance. (/Users/vaibhavsingh/Dev/nightwatch/examples/tests/duckDuckGo.js:8:8)\n at Context.call (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/context.js:478:35)\n at TestCase.run (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/testcase.js:58:31)\n at Runnable.__runFn (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:759:80)\n at Runnable.run (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/runnable.js:126:21)\n at TestSuite.executeRunnable (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:912:49)\n at TestSuite.handleRunnable (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:927:33)\n at /Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:759:21\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async DefaultRunner.runTestSuite (/Users/vaibhavsingh/Dev/nightwatch/lib/runner/test-runners/default.js:78:7)' }, + skippedAtRuntime: [ + ], skipped: [ ], time: '8.932', @@ -6883,6 +6913,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 0, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '5.086', @@ -7373,6 +7405,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 2, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '49.04', @@ -8050,6 +8084,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 3, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '1.116', @@ -8615,6 +8651,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 3, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '0.8370', @@ -9202,6 +9240,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 5, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '0.2180', @@ -9959,6 +9999,8 @@ module.exports = { reportPrefix: 'CHROME_111.0.5563.146__', assertionsCount: 5, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '3.056', @@ -10660,6 +10702,8 @@ module.exports = { reportPrefix: 'FIREFOX_111.0.1__', assertionsCount: 2, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '1.908', @@ -11217,6 +11261,8 @@ module.exports = { abortOnFailure: true, stack: '+ actual - expected\n\n [\n+ \'abortOnFailure\'\n- \'documents\',\n- \'strings\'\n ]\n at Assertion.assert (/Users/vaibhavsingh/Dev/nightwatch/lib/api/_loaders/static.js:112:34)\n at StaticAssert.assertFn (/Users/vaibhavsingh/Dev/nightwatch/lib/api/_loaders/static.js:146:17)\n at Proxy. (/Users/vaibhavsingh/Dev/nightwatch/lib/api/index.js:157:30)\n at DescribeInstance. (/Users/vaibhavsingh/Dev/nightwatch/examples/tests/chromeCDP_example.js:10:20)' }, + skippedAtRuntime: [ + ], skipped: [ ], time: '4.669', @@ -11636,6 +11682,8 @@ module.exports = { waitFor: true, stack: 'Error\n at DescribeInstance. (/Users/vaibhavsingh/Dev/nightwatch/examples/tests/duckDuckGo.js:8:8)\n at Context.call (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/context.js:478:35)\n at TestCase.run (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/testcase.js:58:31)\n at Runnable.__runFn (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:759:80)\n at Runnable.run (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/runnable.js:126:21)\n at TestSuite.executeRunnable (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:912:49)\n at TestSuite.handleRunnable (/Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:927:33)\n at /Users/vaibhavsingh/Dev/nightwatch/lib/testsuite/index.js:759:21\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async DefaultRunner.runTestSuite (/Users/vaibhavsingh/Dev/nightwatch/lib/runner/test-runners/default.js:78:7)' }, + skippedAtRuntime: [ + ], skipped: [ ], time: '5.777', @@ -12228,6 +12276,9 @@ module.exports = { reportPrefix: '', assertionsCount: 0, lastError: null, + skippedAtRuntime: [ + 'should complete the consent form' + ], skipped: [ 'should complete the consent form' ], @@ -12277,6 +12328,8 @@ module.exports = { reportPrefix: 'FIREFOX_111.0.1__', assertionsCount: 5, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '3.401', @@ -12974,6 +13027,8 @@ module.exports = { reportPrefix: 'FIREFOX_111.0.1__', assertionsCount: 1, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '1.082', @@ -13463,6 +13518,8 @@ module.exports = { reportPrefix: 'FIREFOX_111.0.1__', assertionsCount: 5, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '0.09500', @@ -14218,6 +14275,9 @@ module.exports = { reportPrefix: '', assertionsCount: 0, lastError: null, + skippedAtRuntime: [ + 'retrieve the shadowRoot' + ], skipped: [ 'retrieve the shadowRoot' ], @@ -14267,6 +14327,8 @@ module.exports = { reportPrefix: 'FIREFOX_111.0.1__', assertionsCount: 2, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '10.68', @@ -14982,6 +15044,8 @@ module.exports = { reportPrefix: 'FIREFOX_111.0.1__', assertionsCount: 3, lastError: null, + skippedAtRuntime: [ + ], skipped: [ ], time: '1.251', diff --git a/test/src/runner/testRunnerJunitOutput.js b/test/src/runner/testRunnerJunitOutput.js index 647705aa47..e081a1365b 100644 --- a/test/src/runner/testRunnerJunitOutput.js +++ b/test/src/runner/testRunnerJunitOutput.js @@ -142,6 +142,7 @@ describe('testRunnerJUnitOutput', function() { assert.strictEqual(results.modules.sampleWithAssertionFailedInBefore.assertionsCount, 1); assert.strictEqual(results.modules.sampleWithAssertionFailedInBefore.errmessages.length, 1); assert.strictEqual(results.modules.sampleWithAssertionFailedInBefore.failedCount, 1); + assert.strictEqual(results.modules.sampleWithAssertionFailedInBefore.skippedAtRuntime.length, 0); assert.strictEqual(results.modules.sampleWithAssertionFailedInBefore.skipped.length, 0); assert.strictEqual(results.modules.sampleWithAssertionFailedInBefore.completed['demo test async'].assertions.length, 0); } @@ -182,6 +183,7 @@ describe('testRunnerJUnitOutput', function() { assert.strictEqual(results.modules.sampleWithAssertionFailedInAfter.assertionsCount, 1); assert.strictEqual(results.modules.sampleWithAssertionFailedInAfter.errmessages.length, 1); assert.strictEqual(results.modules.sampleWithAssertionFailedInAfter.failedCount, 1); + assert.strictEqual(results.modules.sampleWithAssertionFailedInAfter.skippedAtRuntime.length, 0); assert.strictEqual(results.modules.sampleWithAssertionFailedInAfter.skipped.length, 0); assert.strictEqual(results.modules.sampleWithAssertionFailedInAfter.completed.demoTestAsyncOne.assertions.length, 0); } @@ -222,6 +224,7 @@ describe('testRunnerJUnitOutput', function() { assert.strictEqual(results.modules.sampleWithFailureInTestcaseAndAfter.assertionsCount, 2); assert.strictEqual(results.modules.sampleWithFailureInTestcaseAndAfter.errmessages.length, 1); assert.strictEqual(results.modules.sampleWithFailureInTestcaseAndAfter.failedCount, 2); + assert.strictEqual(results.modules.sampleWithFailureInTestcaseAndAfter.skippedAtRuntime.length, 0); assert.strictEqual(results.modules.sampleWithFailureInTestcaseAndAfter.skipped.length, 0); assert.strictEqual(results.modules.sampleWithFailureInTestcaseAndAfter.completed['demo test async'].assertions.length, 1); } @@ -263,6 +266,7 @@ describe('testRunnerJUnitOutput', function() { assert.strictEqual(results.modules.sampleWithErrorInTestcaseAndAfter.assertionsCount, 1); assert.strictEqual(results.modules.sampleWithErrorInTestcaseAndAfter.errmessages.length, 2); assert.strictEqual(results.modules.sampleWithErrorInTestcaseAndAfter.failedCount, 1); + assert.strictEqual(results.modules.sampleWithErrorInTestcaseAndAfter.skippedAtRuntime.length, 0); assert.strictEqual(results.modules.sampleWithErrorInTestcaseAndAfter.skipped.length, 0); assert.strictEqual(results.modules.sampleWithErrorInTestcaseAndAfter.completed['demo test async'].assertions.length, 0); assert.strictEqual(results.modules.sampleWithErrorInTestcaseAndAfter.completed['demo test async'].errors, 1); @@ -307,6 +311,7 @@ describe('testRunnerJUnitOutput', function() { assert.strictEqual(results.modules.sampleWithFailureInBeforeAndAfter.assertionsCount, 2); assert.strictEqual(results.modules.sampleWithFailureInBeforeAndAfter.errmessages.length, 2); assert.strictEqual(results.modules.sampleWithFailureInBeforeAndAfter.failedCount, 2); + assert.strictEqual(results.modules.sampleWithFailureInBeforeAndAfter.skippedAtRuntime.length, 0); assert.strictEqual(results.modules.sampleWithFailureInBeforeAndAfter.skipped.length, 0); } },