From 020601e08b5875509018f3777ba840705b50c454 Mon Sep 17 00:00:00 2001 From: Prudhvi Date: Mon, 15 May 2023 18:00:43 +0530 Subject: [PATCH] Feat/rerun (#3703) * Added json reporter for rerun * Added rerun functionality for failed tests --------- Co-authored-by: Harshit Agrawal Co-authored-by: Harshit Agrawal <94462364+harshit-bs@users.noreply.github.com> Co-authored-by: Binayak Ghosh --- lib/reporter/reporters/minimalJson.js | 68 +++++++++++ lib/runner/cli/argv-setup.js | 5 + lib/runner/test-source.js | 61 ++++++++++ lib/settings/defaults.js | 6 +- test/extra/minimalJsonReporter.json | 12 ++ .../integration/sampleWithAsyncTestcase.js | 2 +- test/src/index/testNightwatchIndex.js | 2 +- test/src/runner/testRerunFunctionality.js | 106 ++++++++++++++++++ test/src/runner/testRunnerRerunJsonOutput.js | 87 ++++++++++++++ 9 files changed, 345 insertions(+), 4 deletions(-) create mode 100644 lib/reporter/reporters/minimalJson.js create mode 100644 test/extra/minimalJsonReporter.json create mode 100644 test/src/runner/testRerunFunctionality.js create mode 100644 test/src/runner/testRunnerRerunJsonOutput.js diff --git a/lib/reporter/reporters/minimalJson.js b/lib/reporter/reporters/minimalJson.js new file mode 100644 index 0000000000..a2da709167 --- /dev/null +++ b/lib/reporter/reporters/minimalJson.js @@ -0,0 +1,68 @@ +const lodashPick = require('lodash.pick'); +const path = require('path'); +const Utils = require('../../utils'); +const {Logger} = Utils; +const BaseReporter = require('../base-reporter.js'); + +class MinimalJsonReporter extends BaseReporter { + + adaptModule(module) { + // Pick only the necessary fields + const result = lodashPick(module, [ + 'modulePath', + 'status' + ]); + + return result; + } + + adaptResults() { + const {modules} = this.results; + this.modules = {}; + + Object.keys(modules).forEach((moduleKey) => { + this.modules[moduleKey] = this.adaptModule(modules[moduleKey]); + }); + + return this.modules; + } + + writeReport() { + const results = this.adaptResults(); + const {output_folder} = this.options; + const filename = path.join(output_folder, 'minimal_report.json'); + const shouldCreateFolder = !Utils.dirExistsSync(output_folder); + + const report = { + modules: results + }; + + return this.writeReportFile(filename, JSON.stringify(report), shouldCreateFolder, output_folder) + .then(_ => { + Logger.info(Logger.colors.stack_trace(`Wrote Rerun Json report file to: ${path.resolve(filename)}`)); + + // Setting env varaible with minimalJsonReporter path. + // Next time user runs nightwatch with rerun functionality, json reporter can be read from this + process.env.NIGHTWATCH_RERUN_REPORT_FILE = filename; + process.env.NEW_FOO = filename; + }); + } +} + +function write(results, options, callback) { + const reporter = new MinimalJsonReporter(results, options); + + reporter.writeReport() + .then(_ => { + callback(); + }) + .catch(err => { + Logger.error(err); + callback(err); + }); +} + +module.exports = { + MinimalJsonReporter, + write +}; diff --git a/lib/runner/cli/argv-setup.js b/lib/runner/cli/argv-setup.js index 5b0f44b165..dd1284e04f 100644 --- a/lib/runner/cli/argv-setup.js +++ b/lib/runner/cli/argv-setup.js @@ -495,6 +495,11 @@ module.exports = new (function () { description: 'Set the udid of real iOS device' }); + this.option('rerun-failed', { + description: 'Rerun failed test suites' + + }); + return this; } } diff --git a/lib/runner/test-source.js b/lib/runner/test-source.js index 1cb96d2521..b764cd4824 100644 --- a/lib/runner/test-source.js +++ b/lib/runner/test-source.js @@ -2,7 +2,9 @@ const path = require('path'); const fs = require('fs'); const minimatch = require('minimatch'); const Utils = require('../utils'); +const Concurrency = require('./concurrency'); const {Logger, validExtensions, tsFileExt, jsFileExt, singleSourceFile} = Utils; +const {MinimalJsonReporter} = require('../reporter/reporters/minimalJson'); class TestSource { static get GroupNameDelimiter() { @@ -46,11 +48,70 @@ class TestSource { return testsource; } + getRerunFailedFile(minimal_report_file_path) { + const jsonFile = path.resolve(process.env.NIGHTWATCH_RERUN_REPORT_FILE || minimal_report_file_path || ''); + + if (!Utils.fileExistsSync(jsonFile)) { + const err = new Error('Unable to find the Json reporter file to rerun failed tests'); + + err.showTrace = false; + err.detailedErr = 'Configure the environment variable NIGHTWATCH_RERUN_REPORT_FILE with Json reporter file path'; + err.help = [ + `Try setting ${Logger.colors.cyan('minimal_report_file_path: "JSON-REPORTER-PATH"')} in nightwatch configuration`, + `Or, try running: ${Logger.colors.cyan('export NIGHTWATCH_RERUN_REPORT_FILE="JSON-REPORTER-PATH"')}` + ]; + + throw err; + } + + return jsonFile; + } + + getTestSourceForRerunFailed() { + const {reporter_options: {minimal_report_file_path}} = this.settings; + const minimalJsonFile = this.getRerunFailedFile(minimal_report_file_path); + + try { + const {modules = {}} = require(minimalJsonFile); + const testsource = []; + + Object.keys(modules).forEach(moduleKey => { + if (modules[moduleKey] && modules[moduleKey].status === 'fail') { + testsource.push(modules[moduleKey].modulePath); + } + }); + + if (testsource.length === 0) { + const err = new Error('Rerun Failed Tests: No failed tests found to rerun.'); + err.noFailedTestFound = true; + err.showTrace = false; + err.detailedErr = 'Run nightwatch with --help to display usage info.'; + + throw err; + } + + return testsource; + } catch (err) { + if (err.noFailedTestFound) { + err.message = 'Rerun Failed Tests: Invalid Json reporter.'; + + err.showTrace = false; + err.detailedErr = 'Please set env variable NIGHTWATCH_RERUN_REPORT_FILE with valid Json reporter path.'; + } + + throw err; + } + } + /** * Returns the path where the tests are located * @returns {*} */ getSource() { + if ((process.env.NIGHTWATCH_RERUN_FAILED === 'true' || this.argv['rerun-failed']) && Concurrency.isMasterProcess()) { + return this.getTestSourceForRerunFailed(this.argv); + } + if (this.argv['test-worker'] || singleSourceFile(this.argv)) { return this.getTestSourceForSingle(this.argv.test || this.argv._source); } diff --git a/lib/settings/defaults.js b/lib/settings/defaults.js index 05fcb6c81d..36adfbc733 100644 --- a/lib/settings/defaults.js +++ b/lib/settings/defaults.js @@ -146,7 +146,9 @@ module.exports = { folder_format: null, // The file name formatting to use while saving HTML report - filename_format: null + filename_format: null, + + minimal_report_file_path: 'tests_output/minimal_report.json' }, // A string or array of folders (excluding subfolders) where the tests are located. @@ -326,7 +328,7 @@ module.exports = { unit_testing_mode: false, - default_reporter: ['junit', 'json', 'html'], + default_reporter: ['junit', 'json', 'minimalJson', 'html'], // In Nightwatch v1.x, when used with "await" operator, API commands will return the full result object as {value: ``} // whereas in v2, the value is return directly; if using a callback, the behaviour remains unchanged diff --git a/test/extra/minimalJsonReporter.json b/test/extra/minimalJsonReporter.json new file mode 100644 index 0000000000..1ec64e553a --- /dev/null +++ b/test/extra/minimalJsonReporter.json @@ -0,0 +1,12 @@ +{ + "modules": { + "demoTagTest": { + "modulePath": "./test/sampletests/tags/sample.js", + "status": "pass" + }, + "demoTest": { + "modulePath": "./test/sampletests/simple/test/sample.js", + "status": "fail" + } + } +} diff --git a/test/mochatests/integration/sampleWithAsyncTestcase.js b/test/mochatests/integration/sampleWithAsyncTestcase.js index 7e4b108406..661fd8dd76 100644 --- a/test/mochatests/integration/sampleWithAsyncTestcase.js +++ b/test/mochatests/integration/sampleWithAsyncTestcase.js @@ -7,7 +7,7 @@ describe('Mocha tests with async testcase', function() { this.suiteRetries(2); assert.deepStrictEqual(this.settings.desiredCapabilities, {browserName: 'firefox'}); - assert.deepStrictEqual(this.argv.reporter, ['junit', 'json', 'html']); + assert.deepStrictEqual(this.argv.reporter, ['junit', 'json', 'minimalJson', 'html']); assert.strictEqual(this.mochaOptions.timeout, 5000); assert.strictEqual(this.waitForTimeout(), 1100); assert.strictEqual(this.waitForRetryInterval(), 100); diff --git a/test/src/index/testNightwatchIndex.js b/test/src/index/testNightwatchIndex.js index d3d0190d75..bfb0129b46 100644 --- a/test/src/index/testNightwatchIndex.js +++ b/test/src/index/testNightwatchIndex.js @@ -256,7 +256,7 @@ describe('test NightwatchIndex', function () { assert.deepStrictEqual(argv, { config: path.resolve('./test/extra/nightwatch.json'), verbose: true, - reporter: ['junit', 'json', 'html'], + reporter: ['junit', 'json', 'minimalJson', 'html'], source: 'test.js', _source: ['test.js'] }); diff --git a/test/src/runner/testRerunFunctionality.js b/test/src/runner/testRerunFunctionality.js new file mode 100644 index 0000000000..6237db57a1 --- /dev/null +++ b/test/src/runner/testRerunFunctionality.js @@ -0,0 +1,106 @@ +const path = require('path'); +const assert = require('assert'); +const common = require('../../common.js'); +const CommandGlobals = require('../../lib/globals/commands.js'); +const MockServer = require('../../lib/mockserver.js'); +const {settings} = common; +const {runTests} = common.require('index.js'); + +describe('testRerun', function () { + beforeEach(function (done) { + process.removeAllListeners('exit'); + process.removeAllListeners('uncaughtException'); + process.removeAllListeners('unhandledRejection'); + + this.server = MockServer.init(); + + this.server.on('listening', () => { + done(); + }); + }); + + afterEach(function (done) { + CommandGlobals.afterEach.call(this, function () { + Object.keys(require.cache).forEach(function (module) { + delete require.cache[module]; + }); + + done(); + }); + }); + + it('Rerun with --rerun-failed cli argument without setting env variable for json reporter', function () { + + return runTests({ + 'rerun-failed': true + }, settings({ + output: false + })).catch(err => { + return err; + }).then(err => { + assert.ok(err instanceof Error); + assert.strictEqual(err.message, 'Unable to find the Json reporter file to rerun failed tests'); + assert.strictEqual(err.detailedErr, 'Configure the environment variable NIGHTWATCH_RERUN_REPORT_FILE with Json reporter file path'); + }); + }); + + it('Rerun with env varaible without setting env variable for json reporter', function () { + process.env.NIGHTWATCH_RERUN_FAILED = 'true'; + + return runTests({ + 'rerun-failed': true + }, settings({ + output: false + })).catch(err => { + return err; + }).then(err => { + assert.ok(err instanceof Error); + assert.strictEqual(err.message, 'Unable to find the Json reporter file to rerun failed tests'); + assert.strictEqual(err.detailedErr, 'Configure the environment variable NIGHTWATCH_RERUN_REPORT_FILE with Json reporter file path'); + + delete process.env.NIGHTWATCH_RERUN_FAILED; + }); + }); + + it('Rerun with env varaible set and with setting env variable for json reporter', function () { + process.env.NIGHTWATCH_RERUN_FAILED = 'true'; + process.env.NIGHTWATCH_RERUN_REPORT_FILE = path.join(__dirname, '../../extra/minimalJsonReporter.json'); + + return runTests({ + }, settings({ + output: false, + globals: { + reporter(results) { + assert.strictEqual(Object.keys(results.modules).length, 1); + assert.strictEqual(results.assertions, 2); + } + } + })).then(err => { + assert.strictEqual(err, false); + + delete process.env.NIGHTWATCH_RERUN_FAILED; + delete process.env.NIGHTWATCH_RERUN_REPORT_FILE; + }); + }); + + it('Rerun with cli flag and with setting env variable for json reporter', function () { + process.env.NIGHTWATCH_RERUN_REPORT_FILE = path.join(__dirname, '../../extra/minimalJsonReporter.json'); + + return runTests({ + 'rerun-failed': true + }, settings({ + output: false, + globals: { + reporter(results) { + assert.strictEqual(Object.keys(results.modules).length, 1); + assert.strictEqual(results.assertions, 2); + } + } + })).then(err => { + assert.strictEqual(err, false); + delete process.env.NIGHTWATCH_RERUN_REPORT_FILE; + }); + }); + + +}); diff --git a/test/src/runner/testRunnerRerunJsonOutput.js b/test/src/runner/testRunnerRerunJsonOutput.js new file mode 100644 index 0000000000..88361e991d --- /dev/null +++ b/test/src/runner/testRunnerRerunJsonOutput.js @@ -0,0 +1,87 @@ +const path = require('path'); +const fs = require('fs'); +const assert = require('assert'); +const common = require('../../common.js'); +const MockServer = require('../../lib/mockserver.js'); +const CommandGlobals = require('../../lib/globals/commands.js'); +const rimraf = require('rimraf'); +const {settings} = common; +const {runTests} = common.require('index.js'); +const {mkpath} = common.require('utils'); + +describe('testRunnerRerunJsonOutput', function() { + before(function(done) { + this.server = MockServer.init(); + this.server.on('listening', () => done()); + }); + + beforeEach(function(done) { + mkpath('output', function(err) { + if (err) { + return done(err); + } + done(); + }); + }); + + afterEach(function(done) { + rimraf('output', done); + }); + + after(function(done) { + CommandGlobals.afterEach.call(this, done); + }); + + it('testRunWithRerunJsonOutput', function() { + const testsPath = [ + path.join(__dirname, '../../sampletests/withsubfolders') + ]; + + return runTests({source: testsPath, reporter: 'minimalJson'}, settings({ + output_folder: 'output', + silent: true, + globals: {reporter: function() {}} + })) + .then(_ => { + return readDirPromise(testsPath[0]); + }) + .then(list => { + const rerunJsonReportFile = path.resolve('output/minimal_report.json'); + assert.ok(fileExistsSync(rerunJsonReportFile), 'The simple report file was not created.'); + + return require(rerunJsonReportFile); + }) + .then(data => { + assert.strictEqual(typeof data.modules, 'object'); + Object.keys(data.modules).forEach((moduleKey) => { + const module = data.modules[moduleKey]; + + assert.ok(Object.keys(module).includes('modulePath')); + assert.ok(Object.keys(module).includes('status')); + }); + }); + }); +}); + +function readDirPromise(dirName) { + return new Promise(function(resolve, reject) { + fs.readdir(dirName, function(err, result) { + if (err) { + return reject(err); + } + + resolve(result); + }); + }); +} + +// util to replace deprecated fs.existsSync +function fileExistsSync(path) { + try { + fs.statSync(path); + + return true; + } catch (e) { + return false; + } +}