Feat/rerun (#3703)

* Added json reporter for rerun

* Added rerun functionality for failed tests



---------

Co-authored-by: Harshit Agrawal <harshit.a@browserstack.com>
Co-authored-by: Harshit Agrawal <94462364+harshit-bs@users.noreply.github.com>
Co-authored-by: Binayak Ghosh <ghoshbinayak@gmail.com>

lib/reporter/reporters/minimalJson.js

@@ -0,0 +1,68 @@
+const lodashPick = require('lodash.pick');
+const path = require('path');
+const Utils = require('../../utils');
+const {Logger} = Utils;
+const BaseReporter = require('../base-reporter.js');
+
+class MinimalJsonReporter extends BaseReporter {
+
+  adaptModule(module) {
+    // Pick only the necessary fields
+    const result = lodashPick(module, [
+      'modulePath',
+      'status'
+    ]);
+
+    return result;
+  }
+
+  adaptResults() {
+    const {modules}  = this.results;
+    this.modules = {};
+
+    Object.keys(modules).forEach((moduleKey) => {
+      this.modules[moduleKey] = this.adaptModule(modules[moduleKey]);
+    });
+
+    return this.modules;
+  }
+
+  writeReport() {
+    const results = this.adaptResults();
+    const {output_folder} = this.options;
+    const filename = path.join(output_folder, 'minimal_report.json');
+    const shouldCreateFolder = !Utils.dirExistsSync(output_folder);
+
+    const report = {
+      modules: results
+    };
+
+    return this.writeReportFile(filename, JSON.stringify(report), shouldCreateFolder, output_folder)
+      .then(_ => {
+        Logger.info(Logger.colors.stack_trace(`Wrote Rerun Json report file to: ${path.resolve(filename)}`));
+
+        // Setting env varaible with minimalJsonReporter path.
+        // Next time user runs nightwatch with rerun functionality, json reporter can be read from this
+        process.env.NIGHTWATCH_RERUN_REPORT_FILE = filename;
+        process.env.NEW_FOO = filename;
+      });
+  }
+}
+
+function write(results, options, callback) {
+  const reporter = new MinimalJsonReporter(results, options);
+
+  reporter.writeReport()
+    .then(_ => {
+      callback();
+    })
+    .catch(err => {
+      Logger.error(err);
+      callback(err);
+    });
+}
+
+module.exports = {
+  MinimalJsonReporter,
+  write
+};

lib/runner/cli/argv-setup.js

@@ -495,6 +495,11 @@ module.exports = new (function () {
         description: 'Set the udid of real iOS device'
       });
 
+      this.option('rerun-failed', {
+        description: 'Rerun failed test suites'
+
+      });
+
       return this;
     }
   }

lib/runner/test-source.js

@@ -2,7 +2,9 @@ const path = require('path');
 const fs = require('fs');
 const minimatch = require('minimatch');
 const Utils = require('../utils');
+const Concurrency = require('./concurrency');
 const {Logger, validExtensions, tsFileExt, jsFileExt, singleSourceFile} = Utils;
+const {MinimalJsonReporter} = require('../reporter/reporters/minimalJson');
 
 class TestSource {
   static get GroupNameDelimiter() {
@@ -46,11 +48,70 @@ class TestSource {
     return testsource;
   }
 
+  getRerunFailedFile(minimal_report_file_path) {
+    const jsonFile = path.resolve(process.env.NIGHTWATCH_RERUN_REPORT_FILE || minimal_report_file_path || '');
+
+    if (!Utils.fileExistsSync(jsonFile)) {
+      const err = new Error('Unable to find the Json reporter file to rerun failed tests');
+
+      err.showTrace = false;
+      err.detailedErr = 'Configure the environment variable NIGHTWATCH_RERUN_REPORT_FILE with Json reporter file path';
+      err.help = [
+        `Try setting ${Logger.colors.cyan('minimal_report_file_path: "JSON-REPORTER-PATH"')} in nightwatch configuration`,
+        `Or, try running: ${Logger.colors.cyan('export NIGHTWATCH_RERUN_REPORT_FILE="JSON-REPORTER-PATH"')}`
+      ];
+
+      throw err;
+    }
+
+    return jsonFile;
+  }
+
+  getTestSourceForRerunFailed() {
+    const {reporter_options: {minimal_report_file_path}} = this.settings;
+    const minimalJsonFile = this.getRerunFailedFile(minimal_report_file_path);
+
+    try {
+      const {modules = {}} = require(minimalJsonFile);
+      const testsource = [];
+
+      Object.keys(modules).forEach(moduleKey => {
+        if (modules[moduleKey] && modules[moduleKey].status === 'fail') {
+          testsource.push(modules[moduleKey].modulePath);
+        }
+      });
+
+      if (testsource.length === 0) {
+        const err = new Error('Rerun Failed Tests: No failed tests found to rerun.');
+        err.noFailedTestFound = true;
+        err.showTrace = false;
+        err.detailedErr = 'Run nightwatch with --help to display usage info.';
+
+        throw err;
+      }
+
+      return testsource;
+    } catch (err) {
+      if (err.noFailedTestFound) {
+        err.message = 'Rerun Failed Tests: Invalid Json reporter.';
+
+        err.showTrace = false;
+        err.detailedErr = 'Please set env variable NIGHTWATCH_RERUN_REPORT_FILE with valid Json reporter path.';
+      }
+
+      throw err;
+    }
+  }
+
   /**
    * Returns the path where the tests are located
    * @returns {*}
    */
   getSource() {
+    if ((process.env.NIGHTWATCH_RERUN_FAILED === 'true' || this.argv['rerun-failed']) && Concurrency.isMasterProcess()) {
+      return this.getTestSourceForRerunFailed(this.argv);
+    }
+
     if (this.argv['test-worker'] || singleSourceFile(this.argv)) {
       return this.getTestSourceForSingle(this.argv.test || this.argv._source);
     }

lib/settings/defaults.js

@@ -146,7 +146,9 @@ module.exports = {
     folder_format: null,
 
     // The file name formatting to use while saving HTML report
-    filename_format: null
+    filename_format: null,
+
+    minimal_report_file_path: 'tests_output/minimal_report.json'
   },
 
   // A string or array of folders (excluding subfolders) where the tests are located.
@@ -322,7 +324,7 @@ module.exports = {
 
   unit_testing_mode: false,
 
-  default_reporter: ['junit', 'json', 'html'],
+  default_reporter: ['junit', 'json', 'minimalJson', 'html'],
 
   // In Nightwatch v1.x, when used with "await" operator, API commands will return the full result object as {value: `<VALUE>`}
   // whereas in v2, the value is return directly; if using a callback, the behaviour remains unchanged

test/extra/minimalJsonReporter.json

@@ -0,0 +1,12 @@
+{
+    "modules": {
+        "demoTagTest": {
+            "modulePath": "./test/sampletests/tags/sample.js",
+            "status": "pass"
+        },
+        "demoTest": {
+            "modulePath": "./test/sampletests/simple/test/sample.js",
+            "status": "fail"
+        }
+    }
+}

test/mochatests/integration/sampleWithAsyncTestcase.js

@@ -7,7 +7,7 @@ describe('Mocha tests with async testcase', function() {
   this.suiteRetries(2);
 
   assert.deepStrictEqual(this.settings.desiredCapabilities, {browserName: 'firefox'});
-  assert.deepStrictEqual(this.argv.reporter, ['junit', 'json', 'html']);
+  assert.deepStrictEqual(this.argv.reporter, ['junit', 'json', 'minimalJson', 'html']);
   assert.strictEqual(this.mochaOptions.timeout, 5000);
   assert.strictEqual(this.waitForTimeout(), 1100);
   assert.strictEqual(this.waitForRetryInterval(), 100);

test/src/index/testNightwatchIndex.js

@@ -256,7 +256,7 @@ describe('test NightwatchIndex', function () {
       assert.deepStrictEqual(argv, {
         config: path.resolve('./test/extra/nightwatch.json'),
         verbose: true,
-        reporter: ['junit', 'json', 'html'],
+        reporter: ['junit', 'json', 'minimalJson', 'html'],
         source: 'test.js',
         _source: ['test.js']
       });

test/src/runner/testRerunFunctionality.js

@@ -0,0 +1,106 @@
+const path = require('path');
+const assert = require('assert');
+const common = require('../../common.js');
+const CommandGlobals = require('../../lib/globals/commands.js');
+const MockServer = require('../../lib/mockserver.js');
+const {settings} = common;
+const {runTests} = common.require('index.js');
+
+describe('testRerun', function () {
+  beforeEach(function (done) {
+    process.removeAllListeners('exit');
+    process.removeAllListeners('uncaughtException');
+    process.removeAllListeners('unhandledRejection');
+
+    this.server = MockServer.init();
+
+    this.server.on('listening', () => {
+      done();
+    });
+  });
+
+  afterEach(function (done) {
+    CommandGlobals.afterEach.call(this, function () {
+      Object.keys(require.cache).forEach(function (module) {
+        delete require.cache[module];
+      });
+
+      done();
+    });
+  });
+
+  it('Rerun with --rerun-failed cli argument without setting env variable for json reporter', function () {
+
+    return runTests({
+      'rerun-failed': true
+    }, settings({
+      output: false
+    })).catch(err => {
+      return err;
+    }).then(err => {
+      assert.ok(err instanceof Error);
+      assert.strictEqual(err.message, 'Unable to find the Json reporter file to rerun failed tests');
+      assert.strictEqual(err.detailedErr, 'Configure the environment variable NIGHTWATCH_RERUN_REPORT_FILE with Json reporter file path');
+    });
+  });
+
+  it('Rerun with env varaible without setting env variable for json reporter', function () {
+    process.env.NIGHTWATCH_RERUN_FAILED = 'true';
+
+    return runTests({
+      'rerun-failed': true
+    }, settings({
+      output: false
+    })).catch(err => {
+      return err;
+    }).then(err => {
+      assert.ok(err instanceof Error);
+      assert.strictEqual(err.message, 'Unable to find the Json reporter file to rerun failed tests');
+      assert.strictEqual(err.detailedErr, 'Configure the environment variable NIGHTWATCH_RERUN_REPORT_FILE with Json reporter file path');
+
+      delete process.env.NIGHTWATCH_RERUN_FAILED;
+    });
+  });
+
+  it('Rerun with env varaible set and with setting env variable for json reporter', function () {
+    process.env.NIGHTWATCH_RERUN_FAILED = 'true';
+    process.env.NIGHTWATCH_RERUN_REPORT_FILE = path.join(__dirname, '../../extra/minimalJsonReporter.json');
+
+    return runTests({
+    }, settings({
+      output: false,
+      globals: {
+        reporter(results) {
+          assert.strictEqual(Object.keys(results.modules).length, 1);
+          assert.strictEqual(results.assertions, 2);
+        }
+      }
+    })).then(err => {
+      assert.strictEqual(err, false);
+
+      delete process.env.NIGHTWATCH_RERUN_FAILED;
+      delete process.env.NIGHTWATCH_RERUN_REPORT_FILE;
+    });
+  });
+
+  it('Rerun with cli flag and with setting env variable for json reporter', function () {
+    process.env.NIGHTWATCH_RERUN_REPORT_FILE = path.join(__dirname, '../../extra/minimalJsonReporter.json');
+
+    return runTests({
+      'rerun-failed': true
+    }, settings({
+      output: false,
+      globals: {
+        reporter(results) {
+          assert.strictEqual(Object.keys(results.modules).length, 1);
+          assert.strictEqual(results.assertions, 2);
+        }
+      }
+    })).then(err => {
+      assert.strictEqual(err, false);
+      delete process.env.NIGHTWATCH_RERUN_REPORT_FILE;
+    });
+  });
+
+
+});