Report skipped tests

glacejs · Dec 27, 2017 · ca22c76 · ca22c76
1 parent 6999d51
commit ca22c76
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 10 deletions.
diff --git a/lib/globals.js b/lib/globals.js
@@ -192,6 +192,7 @@ global.session = (name, ctx, fixtures, func) => {
  * @arg {string} name - Name of test case.
  * @arg {object} [opts] - Options.
  * @arg {boolean} [opts.skip=false] - Flag to skip test.
+ * @arg {string} [opts.skipReason=null] - Skip reason if test is marked as skipped.
  * @arg {number} [opts.retry=0] - Number of retries on failure. May be managed
  *  in config.
  * @arg {?number} [opts.chunkTimeout=null] - Time to execute chunk or hook, sec.
@@ -202,10 +203,9 @@ var baseTest = (names => {
     return (name, opts, fixtures, func) => {
 
         if (names.includes(name)) {
-            LOG.warn(`Test case '${name}' is added already.`,
-                     `Test case with the same name will be omitted.`);
-            return;
+            throw new Error(`Test case '${name}' is added already`);
         };
+        names.push(name);
 
         if (opts instanceof Function) {
             func = opts;
@@ -220,20 +220,21 @@ var baseTest = (names => {
         fixtures = fixtures || [];
 
         opts.skip = U.defVal(opts.skip, false);
+        opts.skipReason = U.defVal(opts.skipReason);
         opts.retry = U.defVal(opts.retry, CONF.testRetries, 0);
         opts.chunkTimeout = U.defVal(opts.chunkTimeout);
 
+        var testCase = new TestCase(name);
+        CONF.testCases.push(testCase);
+
         if (opts.skip) {
-            LOG.warn(`Test '${name}' is marked as skipped and will be omitted`);
+            testCase.status = TestCase.SKIPPED;
+            if (opts.skipReason) testCase.addDetails(opts.skipReason);
             return;
         };
-        names.push(name);
 
-        var testCase = new TestCase(name);
         var retries = opts.retry;
 
-        CONF.testCases.push(testCase);
-
         var testFunc = ctxs => {
             ctxs = ctxs || [{}];
 

diff --git a/lib/reporter/stdout.js b/lib/reporter/stdout.js
@@ -33,6 +33,7 @@ var epilogue = () => {
 
     var passedTests = [];
     var failedTests = [];
+    var skippedTests = [];
 
     for (var testCase of CONF.testCases) {
         if (testCase.status === TestCase.FAILED) {
@@ -41,6 +42,9 @@ var epilogue = () => {
         if (testCase.status === TestCase.PASSED) {
             passedTests.push(testCase);
         };
+        if (testCase.status === TestCase.SKIPPED) {
+            skippedTests.push(testCase);
+        };
     };
 
     if (passedTests.length !== 0) {
@@ -71,6 +75,21 @@ var epilogue = () => {
                `${execTime} sec`.white.bold);
     };
 
+    if (skippedTests.length !== 0) {
+        var msg = "skipped test" + (skippedTests.length === 1 ? "" : "s");
+        stdout();
+        stdout((indent + "# " +
+                String(skippedTests.length).bold + " " + msg).gray);
+
+        for (var skip of skippedTests) {
+            var msg = `* '${skip.name}'`;
+            if (skip.rawInfo[0]) {
+                msg += " - " + skip.rawInfo[0].bold;
+            };
+            stdout(indent + indent + msg.gray);
+        };
+    };
+
     if (failedTests.length) {
         stdout();
         stdout("TEST FAILURES:".bold)

diff --git a/lib/reporter/testrail.js b/lib/reporter/testrail.js
@@ -128,6 +128,9 @@ var testrailReporter = module.exports = {
                 testResult.comment += "\n" + info;
             };
         };
+        if (testCase.status === TestCase.SKIPPED) {
+            testResult.status_id = Results.BLOCKED;
+        };
         if (testCase.status === TestCase.FAILED) {
             testResult.status_id = Results.FAILED;
             testResult.comment += "\n\nErrors:";

diff --git a/lib/reporter/xunit.js b/lib/reporter/xunit.js
@@ -34,7 +34,7 @@ var xunitReporter = module.exports = {
             tests: CONF.testCases.length,
             failures: CONF.testCases.filter(t => t.status === TestCase.FAILED).length,
             errors: CONF.testCases.filter(t => t.status === TestCase.FAILED).length,
-            skipped: 0,  // FIXME support skipped tests in report.
+            skipped: CONF.testCases.filter(t => t.status === TestCase.SKIPPED).length,
             timestamp: new Date().toUTCString(),
             time: (CONF.testCases.map(t => t.duration).reduce((a, b) => a + b, 0) / 1000) || 0
           }, false));
@@ -73,6 +73,8 @@ var writeTest = test => {
     if (test.status === TestCase.FAILED) {
         var err = test.errors.join("\n");
         write(tag('testcase', attrs, false, tag('failure', {}, false, escape(err))));
+    } else if (test.status === TestCase.SKIPPED) {
+        write(tag('testcase', attrs, false, tag('skipped', {}, true)));
     } else {
         write(tag('testcase', attrs, true));
     };

diff --git a/lib/testing.js b/lib/testing.js
@@ -28,6 +28,7 @@ var TestCase = module.exports.TestCase = function (name) {
     this.name = name;
     this.status = TestCase.NOT_STARTED;
     this.chunks = [];
+    this.reset();
 };
 /**
  * Starts test case.
@@ -130,3 +131,4 @@ TestCase.NOT_STARTED = "not started";
 TestCase.IN_PROGRESS = "in progress";
 TestCase.FAILED = "failed";
 TestCase.PASSED = "passed";
+TestCase.SKIPPED = "skipped";
diff --git a/tests/e2e/testBasic.js b/tests/e2e/testBasic.js
@@ -66,7 +66,13 @@ test("It should involve fixture in iterator", ctx => {
     });
 });
 
-test("It should be skipped and absent in report", { skip: true }, () => {
+test("It should be skipped with reason",
+     { skip: true,
+       skipReason: "bug https://bug.tracker.io/BUG-1001" }, () => {
+    chunk(() => {});
+});
+
+test("It should be skipped without reason", { skip: true }, () => {
     chunk(() => {});
 });