Skipped tests have categorized as either user-marked or runtime-failed (

#3711)

* splitted skipped tests marked by user and failed at runtime


---------

Co-authored-by: Binayak Ghosh <ghoshbinayak@gmail.com>

lib/reporter/base-reporter.js

@@ -11,7 +11,7 @@ module.exports = class BaseReporter {
 
   adaptAssertions(module) {
     module.completed = Object.keys(module.completed).reduce(function(prev, item) {
-      if (module.skipped.includes(item)) {
+      if ((module.skipped && module.skipped.includes(item))) {
         return prev;
       }
 
@@ -39,7 +39,9 @@ module.exports = class BaseReporter {
     }, {});
 
     module.completedSections = Object.keys(module.completedSections).reduce(function(prev, item) {
-      if (module.skipped && module.skipped.includes(item)) {
+      if ((module.skippedAtRuntime && module.skippedAtRuntime.includes(item)) || 
+          (module.skippedByUser && module.skippedByUser.includes(item))
+      ) {
         return prev;
       }
 

lib/reporter/index.js

@@ -22,12 +22,12 @@ class Reporter extends SimplifiedReporter {
    * @param {Object} settings
    * @param {Object} addOpts
    */
-  constructor({settings, tests, suiteRetries, addOpts = {}}) {
+  constructor({settings, tests, suiteRetries, addOpts = {}, skippedTests, allScreenedTests}) {
     super(settings);
 
     this.suiteRetries = suiteRetries;
     this.suiteName = addOpts.suiteName;
-    this.testResults = new TestResults(tests, addOpts, settings);
+    this.testResults = new TestResults(tests, addOpts, settings, skippedTests, allScreenedTests);
     this.currentContext = null;
     this.__printA11Report = false;
     this.reporter = addOpts.repoter;

lib/reporter/reporters/html.js

@@ -198,12 +198,21 @@ class HtmlReporter extends BaseReporter {
       }
     }
 
-    // Add skipped tests in completed section
-    for (const skippedTestName of module.skipped) {
+    // Add skipped tests by user in completed section
+    module.skippedByUser && module.skippedByUser.forEach(skippedTestName => {
       module.completedSections[skippedTestName] = {
-        status: 'skip'
+        status: 'skip',
+        runtimeFailure: false
       };
-    }
+    });
+
+    // Add skipped tests at runtime in completed section
+    module.skippedAtRuntime && module.skippedAtRuntime.forEach(skippedTestName => {
+      module.completedSections[skippedTestName] = {
+        status: 'skip',
+        runtimeFailure: true
+      };
+    });
 
     return result;
   }

lib/reporter/reporters/junit.xml.ejs

@@ -4,7 +4,7 @@
             tests="<%= module.tests %>">
 
   <testsuite name="<%= className %>"
-    errors="<%= module.errors %>" failures="<%= module.failures %>" hostname="" id="" package="<%= module.group || moduleName %>" skipped="<%= (Array.isArray(module.skipped)) ? module.skipped.length : 0 %>"
+    errors="<%= module.errors %>" failures="<%= module.failures %>" hostname="" id="" package="<%= module.group || moduleName %>" skipped="<%= (Array.isArray(module.skippedAtRuntime)) ? module.skippedAtRuntime.length : 0 %>"
     tests="<%= module.tests %>" time="<%= module.time %>" timestamp="<%= module.timestamp %>">
   <% for (var item in module.completed) {
     var testcase = module.completed[item];
@@ -34,10 +34,10 @@
       <%= module.lastError.stack %>
       ]]></error>  <% } %>
 
-  <% if (module.skipped && (module.skipped.length > 0)) { %>
-    <% for (var j = 0; j < module.skipped.length; j++) { %>
+  <% if (module.skippedAtRuntime && (module.skippedAtRuntime.length > 0)) { %>
+    <% for (var j = 0; j < module.skippedAtRuntime.length; j++) { %>
     <testcase
-      name="<%= module.skipped[j] %>" classname="<%= className %>">
+      name="<%= module.skippedAtRuntime[j] %>" classname="<%= className %>">
       <skipped />
     </testcase>
     <% } %>

lib/reporter/results.js

@@ -16,8 +16,9 @@ module.exports = class Results {
     return 'skip';
   }
 
-  constructor(tests = [], opts, settings) {
-    this.skipped = tests.slice(0);
+  constructor(tests = [], opts, settings, skippedTests=[], allScreenedTests=[]) {
+    this.skippedByUser = skippedTests;
+    this.skippedAtRuntime = tests.slice(0);
     this.testcases = {};
     this.testSections = {};
     this.suiteName = opts.suiteName;
@@ -53,7 +54,7 @@ module.exports = class Results {
     this.tags = opts.tags || [];
     this.__retryTest = false;
 
-    this.initCount(tests);
+    this.initCount(allScreenedTests);
   }
 
   get initialResult() {
@@ -81,7 +82,7 @@ module.exports = class Results {
     const currentTest = this.testcases[testName];
 
     if (returnFullResult) {
-      currentTest.steps = this.skipped;
+      currentTest.steps = this.skippedAtRuntime;
       currentTest.stackTrace = this.stackTrace;
       currentTest.testcases = Object.keys(this.testcases).reduce((prev, key) => {
         prev[key] = Object.keys(this.testcases[key]).reduce((prevVal, prop) => {
@@ -215,7 +216,9 @@ module.exports = class Results {
     }
 
     suiteResults.results.lastError = lastError;
-    suiteResults.results.skipped = this.skipped;
+    suiteResults.results.skippedAtRuntime = this.skippedAtRuntime;
+    suiteResults.results.skippedByUser = this.skippedByUser;
+    suiteResults.results.skipped = [...this.skippedAtRuntime, ...this.skippedByUser];
     suiteResults.results.time = this.time;
     suiteResults.results.timeMs = this.timeMs;
     suiteResults.results.completed = this.testcases;
@@ -408,11 +411,11 @@ module.exports = class Results {
     return this;
   }
 
-  initCount(tests) {
+  initCount(allScreenedTests) {
     this.__passedCount = 0;
     this.__failedCount = 0;
     this.__errorsCount = 0;
-    this.__skippedCount = tests.length;
+    this.__skippedCount = allScreenedTests.length;
     this.__testsCount = 0;
     this.__timestamp = new Date().toUTCString();
     this.errmessages = [];
@@ -504,9 +507,9 @@ module.exports = class Results {
     this.currentTestName = testName;
     this.testcases[testName] = this.createTestCaseResults(testcase);
 
-    const index = this.skipped.indexOf(testName);
+    const index = this.skippedAtRuntime.indexOf(testName);
     if (index > -1) {
-      this.skipped.splice(index, 1);
+      this.skippedAtRuntime.splice(index, 1);
       this.__skippedCount -= 1;
     }
 

lib/reporter/summary.js

@@ -98,10 +98,19 @@ module.exports = class Summary {
    * @param {object} testSuite
    */
   static printSkipped(testSuite) {
-    if (testSuite.skipped.length > 0) {
+    if (testSuite.skippedAtRuntime.length > 0) {
       // eslint-disable-next-line no-console
-      console.log(colors.cyan('    SKIPPED:'));
-      testSuite.skipped.forEach(function(testcase) {
+      console.log(colors.cyan('    SKIPPED (at runtime):'));
+      testSuite.skippedAtRuntime.forEach(function(testcase) {
+        // eslint-disable-next-line no-console
+        console.log(`    - ${testcase}`);
+      });
+    }
+
+    if (testSuite.skippedByUser.length > 0) {
+      // eslint-disable-next-line no-console
+      console.log(colors.cyan('    SKIPPED (by user):'));
+      testSuite.skippedByUser.forEach(function(testcase) {
         // eslint-disable-next-line no-console
         console.log(`    - ${testcase}`);
       });

lib/testsuite/context.js

@@ -74,6 +74,8 @@ class Context extends EventEmitter {
 
     this.__hooks = [];
     this.__testcases = [];
+    this.__skippedTestCases = [];
+    this.__allScreenedTests = [];
     this.__contextBinding = {};
 
     this.__transforms = client ? await client.transforms : [];
@@ -138,6 +140,22 @@ class Context extends EventEmitter {
     this.__testcases = value;
   }
 
+  get skippedTests() {
+    return this.__skippedTestCases;
+  }
+
+  set skippedTests(value) {
+    this.__skippedTestCases = value;
+  }
+
+  get allScreenedTests() {
+    return this.__allScreenedTests;
+  }
+
+  set allScreenedTests(value) {
+    this.__allScreenedTests = value;
+  }
+
   get hooks() {
     return this.__hooks;
   }
@@ -297,11 +315,20 @@ class Context extends EventEmitter {
     // TODO: warn if test name already exists
     if (!skipTest) {
       this.tests.push(testName);
+    } else {
+      this.skippedTests.push(testName);
     }
+
     this.addTestSuiteMethod(testName, testFn, describeInstance);
     if (runOnly) {
       this.__currentTestName = testName;
+      this.skippedTests = [...this.allScreenedTests];
+      this.runOnly = true;
+    } else if (this.runOnly) {
+      this.skippedTests.push(testName);
     }
+
+    this.allScreenedTests.push(testName);
   }
 
   /**

lib/testsuite/index.js

@@ -202,7 +202,7 @@ class TestSuite {
     }
 
     const {suiteRetries, suiteName} = this;
-    const {tests, moduleKey, modulePath, groupName} = this.context;
+    const {tests, moduleKey, modulePath, groupName, skippedTests, allScreenedTests} = this.context;
 
     this.reporter = new Reporter({
       settings: this.client.settings,
@@ -218,7 +218,9 @@ class TestSuite {
         groupName,
         isMobile: this.client.api.isMobile(),
         tags: this.context.getTags()
-      }
+      },
+      skippedTests,
+      allScreenedTests
     });
   }