From 22ccbee6fbdef92cade5279fc54150e41469f48d Mon Sep 17 00:00:00 2001 From: Mykhailo Bodnarchuk Date: Tue, 3 Mar 2020 15:46:28 +0100 Subject: [PATCH 1/4] updated docs and worker events --- docs/parallel.md | 181 ++++++++----------------------------- lib/command/run-workers.js | 12 +-- lib/event.js | 12 +++ lib/workers.js | 61 +++++++++---- 4 files changed, 99 insertions(+), 167 deletions(-) diff --git a/docs/parallel.md b/docs/parallel.md index 781c8be97..53792a617 100644 --- a/docs/parallel.md +++ b/docs/parallel.md @@ -32,166 +32,65 @@ By default the tests are assigned one by one to the avaible workers this may lea npx codeceptjs run-workers --suites 2 ``` +## Custom Parallel Execution -## Multiple Browsers Execution +To get a full control of parallelization create a custom execution script to match your needs. +This way you can configure which tests are matched, how the groups are formed, and with which configuration each worker is executed. -This is useful if you want to execute same tests but on different browsers and with different configurations or different tests on same browsers in parallel. +Start with creating file `bin/parallel.js`. -Create `multiple` section in configuration file, and fill it with run suites. Each suite should have `browser` array with browser names or driver helper's configuration: -```js -multiple: { - basic: { - // run all tests in chrome and firefox - browsers: ["chrome", "firefox"] - }, - - smoke: { - browsers: [ - firefox, - // replace any config values from WebDriver helper - { - browser: "chrome", - windowSize: "maximize", - desiredCapabilities: { - acceptSslCerts: true - } - }, - ] - }, -} -``` - -You can use `grep` and `outputName` params to filter tests and output directory for suite: -```js -"multiple": { - "smoke": { - // run only tests containing "@smoke" in name - "grep": "@smoke", - - // store results into `output/smoke` directory - "outputName": "smoke", - - "browsers": [ - "firefox", - {"browser": "chrome", "windowSize": "maximize"} - ] - } -} -``` - -Then tests can be executed using `run-multiple` command. - -Run all suites for all browsers: - -```sh -codeceptjs run-multiple --all -``` - -Run `basic` suite for all browsers - -```sh -codeceptjs run-multiple basic -``` - -Run `basic` suite for chrome only: - -```sh -codeceptjs run-multiple basic:chrome -``` - -Run `basic` suite for chrome and `smoke` for firefox - -```sh -codeceptjs run-multiple basic:chrome smoke:firefox -``` - -Run basic tests with grep and junit reporter +On MacOS/Linux run following commands: -```sh -codeceptjs run-multiple basic --grep signin --reporter mocha-junit-reporter ``` - -Run regression tests specifying different config path: - -```sh -codeceptjs run-multiple regression -c path/to/config +mkdir bin +touch bin/parallel.js +chmod +x bin/parallel.js ``` -Each executed process uses custom folder for reports and output. It is stored in subfolder inside an output directory. Subfolders will be named in `suite_browser` format. - -Output is printed for all running processes. Each line is tagged with a suite and browser name: - -```sh -[basic:firefox] GitHub -- -[basic:chrome] GitHub -- -[basic:chrome] it should not enter -[basic:chrome] ✓ signin in 2869ms +> Filename or directory can be customized. You are creating your own custom runner so take this paragraph as an example. -[basic:chrome] OK | 1 passed // 30s -[basic:firefox] it should not enter -[basic:firefox] ✖ signin in 2743ms +Create a placeholder in file: -[basic:firefox] -- FAILURES: +```js +#!/usr/bin/env node +const { Workers } = require('codeceptjs'); +// here will go magic ``` -### Hooks - -Hooks are available when using the `run-multiple` command to perform actions before the test suites start and after the test suites have finished. See [Hooks](/hooks/#bootstrap-teardown) for an example. - - -### Parallel Execution - -CodeceptJS can be configured to run tests in parallel. - -When enabled, it collects all test files and executes them in parallel by the specified amount of chunks. Given we have five test scenarios (`a_test.js`,`b_test.js`,`c_test.js`,`d_test.js` and `e_test.js`), by setting `"chunks": 2` we tell the runner to run two suites in parallel. The first suite will run `a_test.js`,`b_test.js` and `c_test.js`, the second suite will run `d_test.js` and `e_test.js`. +Now let's see how to update this file for different parallelization modes: +### Example: Running tests in 2 browsers in 4 threads ```js -multiple: { - parallel: { - // Splits tests into 2 chunks - chunks: 2 - } -} -``` +const workerConfig = { + testConfig: './test/data/sandbox/codecept.customworker.js', +}; -To execute them use `run-multiple` command passing configured suite, which is `parallel` in this example: +// don't initialize workers in constructor +const workers = new Workers(null, workerConfig); +// split tests by suites in 2 groups +const testGroups = workers.createGroupsOfSuites(2); -``` -codeceptjs run-multiple parallel -``` +const browsers = ['firefox', 'chrome']; -Grep and multiple browsers are supported. Passing more than one browser will multiply the amount of suites by the amount of browsers passed. The following example will lead to four parallel runs. +const configs = browsers.map(browser => { + return helpers: { + WebDriver: { browser } + } +}); -```js -multiple: { - // 2x chunks + 2x browsers = 4 - parallel: { - // Splits tests into chunks - chunks: 2, - // run all tests in chrome and firefox - browsers: ["chrome", "firefox"] - }, +for (const config of configs) { + for (group of groupOfTests) { + const worker = workers.spawn(); + worker.addTests(group); + worker.addConfig(config); + } } -``` -Passing a function will enable you to provide your own chunking algorithm. The first argument passed to you function is an array of all test files, if you enabled grep the test files passed are already filtered to match the grep pattern. +workers.run(); -```js -multiple: { - parallel: { - // Splits tests into chunks by passing an anonymous function, - // only execute first and last found test file - chunks: (files) => { - return [ - [ files[0] ], // chunk 1 - [ files[files.length-1] ], // chunk 2 - ] - }, - // run all tests in chrome and firefox - browsers: ["chrome", "firefox"] - } -} -``` +workers.on(event.all.result, (status, completed, workerStats) => { + // print output +}); -> Chunking will be most effective if you have many individual test files that contain only a small amount of scenarios. Otherwise the combined execution time of many scenarios or big scenarios in one single test file potentially lead to an uneven execution time. +``` \ No newline at end of file diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index b944fa316..cd83f1b4e 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -36,11 +36,6 @@ module.exports = function (workerCount, options) { }; const numberOfWorkers = parseInt(workerCount, 10); - const workers = new Workers(numberOfWorkers, config); - - workers.overrideConfig(overrideConfigs); - - workers.run(); stats.start = new Date(); @@ -48,6 +43,10 @@ module.exports = function (workerCount, options) { output.print(`Running tests in ${output.styles.bold(numberOfWorkers)} workers...`); output.print(); + const workers = new Workers(numberOfWorkers, config); + workers.overrideConfig(overrideConfigs); + workers.run(); + workers.on(event.test.failed, (failedTest) => { output.test.failed(failedTest); updateFinishedTests(failedTest); @@ -69,7 +68,7 @@ function printResults() { stats.duration = stats.end - stats.start; output.print(); if (stats.tests === 0 || (stats.passes && !errors.length)) { - output.result(stats.passes, stats.failures, 0, `${stats.duration / 1000}s`); + output.result(stats.passes, stats.failures, 0, `${stats.duration || 0 / 1000}s`); } if (stats.failures) { output.print(); @@ -81,7 +80,6 @@ function printResults() { } } - function appendStats(newStats) { stats.passes += newStats.passes; stats.failures += newStats.failures; diff --git a/lib/event.js b/lib/event.js index ce7dcf4b8..e73a2cd99 100644 --- a/lib/event.js +++ b/lib/event.js @@ -101,6 +101,18 @@ module.exports = { after: 'multiple.after', }, + /** + * @type {object} + * @constant + * @inner + * @property {'workers.before'} before + * @property {'workers.after'} after + */ + workers: { + before: 'workers.before', + after: 'workers.after', + }, + /** * @param {string} event * @param {*} param diff --git a/lib/workers.js b/lib/workers.js index e5e9bd258..56a0a2aac 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -12,13 +12,13 @@ const { isFunction, fileExists } = require('./utils'); const mainConfig = require('./config'); const output = require('./output'); const event = require('./event'); +const recorder = require('./recorder'); const runHook = require('./hooks'); const WorkerStorage = require('./workerStorage'); const pathToWorker = path.join(__dirname, 'command', 'workers', 'runTests.js'); const initializeCodecept = (configPath, options = {}) => { - console.log(options.debug); const codecept = new Codecept(mainConfig.load(configPath || '.'), options); codecept.init(getTestRoot(configPath)); codecept.loadTests(); @@ -175,13 +175,37 @@ class Workers extends EventEmitter { this.testGroups = []; createOutputDir(config.testConfig); - this._initWorkers(numberOfWorkers, config); + if (numberOfWorkers) this._initWorkers(numberOfWorkers, config); } _initWorkers(numberOfWorkers, config) { + this.splitTestsByGroups(numberOfWorkers, config); + this.workers = createWorkerObjects(this.testGroups, this.codecept.config, config.testConfig, config.options); + this.numberOfWorkers = this.workers.length; + } + + /** + * This splits tests by groups. + * Strategy for group split is taken from a constructor's config.by value: + * + * `config.by` can be: + * + * - `suite` + * - `test` + * - function(numberOfWorkers) + * + * This method can be overridden for a better split. + */ + splitTestsByGroups(numberOfWorkers, config) { + // prepare + const files = this.codecept.testFiles; + const mocha = Container.mocha(); + mocha.files = files; + mocha.loadFiles(); + if (isFunction(config.by)) { const createTests = config.by; - const testGroups = createTests(); + const testGroups = createTests(numberOfWorkers); if (!(testGroups instanceof Array)) { throw new Error('Test group should be an array'); } @@ -191,9 +215,6 @@ class Workers extends EventEmitter { } else if (typeof numberOfWorkers === 'number' && numberOfWorkers > 0) { this.testGroups = config.by === 'suite' ? this.createGroupsOfSuites(numberOfWorkers) : this.createGroupsOfTests(numberOfWorkers); } - - this.workers = createWorkerObjects(this.testGroups, this.codecept.config, config.testConfig, config.options); - this.numberOfWorkers = this.workers.length; } /** @@ -212,13 +233,11 @@ class Workers extends EventEmitter { * @param {Number} numberOfWorkers */ createGroupsOfTests(numberOfWorkers) { - const files = this.codecept.testFiles; - const mocha = Container.mocha(); - mocha.files = files; - mocha.loadFiles(); const groups = populateGroups(numberOfWorkers); let groupCounter = 0; + const mocha = Container.mocha(); + mocha.suite.eachTest((test) => { const i = groupCounter % groups.length; if (test) { @@ -234,12 +253,10 @@ class Workers extends EventEmitter { * @param {Number} numberOfWorkers */ createGroupsOfSuites(numberOfWorkers) { - const files = this.codecept.testFiles; - const mocha = Container.mocha(); - mocha.files = files; - mocha.loadFiles(); const groups = populateGroups(numberOfWorkers); + const mocha = Container.mocha(); + mocha.suite.suites.forEach((suite) => { const i = indexOfSmallestElement(groups); suite.tests.forEach((test) => { @@ -262,15 +279,20 @@ class Workers extends EventEmitter { } run() { - return new Promise((res, rej) => { + recorder.startUnlessRunning(); + recorder.add('bootstrapAll hook', () => new Promise((res, rej) => { runHook(this.codecept.config.bootstrapAll, () => { - for (const worker of this.workers) { - const workerThread = createWorker(worker); - this._listenWorkerEvents(workerThread); - } res(true); }, 'bootstrapAll'); + })); + event.dispatcher.emit(event.workers.before); + recorder.add('starting workers', () => { + for (const worker of this.workers) { + const workerThread = createWorker(worker); + this._listenWorkerEvents(workerThread); + } }); + return recorder.promise(); } /** @@ -311,6 +333,7 @@ class Workers extends EventEmitter { worker.on('exit', () => { this.closedWorkers += 1; if (this.closedWorkers === this.numberOfWorkers) { + event.dispatcher.emit(event.workers.after); runHook(this.codecept.config.teardownAll, () => { if (this.isFailed()) process.exitCode = 1; else process.exitCode = 0; From 6dd2b8a0b4183ea54dc242444e548a4a39b4b5d3 Mon Sep 17 00:00:00 2001 From: Paul Vincent Beigang Date: Fri, 27 Mar 2020 22:24:58 +0100 Subject: [PATCH 2/4] Store --profile in process.env.profile instead of process.profile to make it available in workers. --- docs/configuration.md | 4 ++-- examples/codecept.conf.example.js | 4 ++-- lib/command/interactive.js | 2 +- lib/command/run-multiple.js | 2 +- lib/command/run-rerun.js | 2 +- lib/command/run-workers.js | 2 +- lib/command/run.js | 2 +- test/data/sandbox/config.js | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index f8edcaa58..17940cafe 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -118,7 +118,7 @@ exports.config = { ## Profile -Using values from `process.profile` you can change the config dynamically. +Using `process.env.profile` you can change the config dynamically. It provides value of `--profile` option passed to runner. Use its value to change config value on the fly. @@ -134,7 +134,7 @@ exports.config = { WebDriver: { url: 'http://localhost:3000', // load value from `profile` - browser: process.profile || 'firefox' + browser: process.env.profile || 'firefox' } } diff --git a/examples/codecept.conf.example.js b/examples/codecept.conf.example.js index 8f5d63510..da0bc889d 100644 --- a/examples/codecept.conf.example.js +++ b/examples/codecept.conf.example.js @@ -1,6 +1,6 @@ console.log('Use JS config file'); -console.log(process.profile); +console.log(process.env.profile); exports.config = { tests: './*_test.js', @@ -9,7 +9,7 @@ exports.config = { helpers: { WebDriverIO: { url: 'http://localhost', - browser: process.profile || 'firefox', + browser: process.env.profile || 'firefox', restart: true, }, }, diff --git a/lib/command/interactive.js b/lib/command/interactive.js index f49542487..53fbdb3c6 100644 --- a/lib/command/interactive.js +++ b/lib/command/interactive.js @@ -6,7 +6,7 @@ const event = require('../event'); const output = require('../output'); module.exports = function (path, options) { - process.profile = options.profile; + process.env.profile = options.profile; const testsPath = getTestRoot(path); const config = getConfig(testsPath); diff --git a/lib/command/run-multiple.js b/lib/command/run-multiple.js index 8bf77b4e2..9265ae045 100644 --- a/lib/command/run-multiple.js +++ b/lib/command/run-multiple.js @@ -29,7 +29,7 @@ let processesDone; module.exports = function (selectedRuns, options) { // registering options globally to use in config - process.profile = options.profile; + process.env.profile = options.profile; const configFile = options.config; let codecept; diff --git a/lib/command/run-rerun.js b/lib/command/run-rerun.js index eee1f2b65..2faa0a7d2 100644 --- a/lib/command/run-rerun.js +++ b/lib/command/run-rerun.js @@ -7,7 +7,7 @@ const Codecept = require('../rerun'); module.exports = function (test, options) { // registering options globally to use in config - process.profile = options.profile; + process.env.profile = options.profile; const configFile = options.config; let codecept; diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index cd83f1b4e..ef4e15c72 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -23,7 +23,7 @@ module.exports = function (workerCount, options) { 'Required minimum Node version of 11.7.0 to work with "run-workers"', ); - process.profile = options.profile; + process.env.profile = options.profile; const { config: testConfig, override = '' } = options; const overrideConfigs = tryOrDefault(() => JSON.parse(override), {}); diff --git a/lib/command/run.js b/lib/command/run.js index f9a65ee2f..d03248566 100644 --- a/lib/command/run.js +++ b/lib/command/run.js @@ -10,7 +10,7 @@ const Codecept = require('../codecept'); module.exports = function (test, options) { // registering options globally to use in config - process.profile = options.profile; + process.env.profile = options.profile; const configFile = options.config; let codecept; diff --git a/test/data/sandbox/config.js b/test/data/sandbox/config.js index 460dea48c..e11369349 100644 --- a/test/data/sandbox/config.js +++ b/test/data/sandbox/config.js @@ -1,4 +1,4 @@ -const profile = process.profile; +const profile = process.env.profile; exports.config = { tests: './*_test.js', From 1b45c7db444bf8e96507c8e8ad1d078c3ff75ec9 Mon Sep 17 00:00:00 2001 From: Paul Vincent Beigang Date: Tue, 31 Mar 2020 12:29:05 +0200 Subject: [PATCH 3/4] Revert "Store --profile in process.env.profile instead of process.profile to make it available in workers." This reverts commit 6dd2b8a0b4183ea54dc242444e548a4a39b4b5d3. --- docs/configuration.md | 4 ++-- examples/codecept.conf.example.js | 4 ++-- lib/command/interactive.js | 2 +- lib/command/run-multiple.js | 2 +- lib/command/run-rerun.js | 2 +- lib/command/run-workers.js | 2 +- lib/command/run.js | 2 +- test/data/sandbox/config.js | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 17940cafe..f8edcaa58 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -118,7 +118,7 @@ exports.config = { ## Profile -Using `process.env.profile` you can change the config dynamically. +Using values from `process.profile` you can change the config dynamically. It provides value of `--profile` option passed to runner. Use its value to change config value on the fly. @@ -134,7 +134,7 @@ exports.config = { WebDriver: { url: 'http://localhost:3000', // load value from `profile` - browser: process.env.profile || 'firefox' + browser: process.profile || 'firefox' } } diff --git a/examples/codecept.conf.example.js b/examples/codecept.conf.example.js index da0bc889d..8f5d63510 100644 --- a/examples/codecept.conf.example.js +++ b/examples/codecept.conf.example.js @@ -1,6 +1,6 @@ console.log('Use JS config file'); -console.log(process.env.profile); +console.log(process.profile); exports.config = { tests: './*_test.js', @@ -9,7 +9,7 @@ exports.config = { helpers: { WebDriverIO: { url: 'http://localhost', - browser: process.env.profile || 'firefox', + browser: process.profile || 'firefox', restart: true, }, }, diff --git a/lib/command/interactive.js b/lib/command/interactive.js index 53fbdb3c6..f49542487 100644 --- a/lib/command/interactive.js +++ b/lib/command/interactive.js @@ -6,7 +6,7 @@ const event = require('../event'); const output = require('../output'); module.exports = function (path, options) { - process.env.profile = options.profile; + process.profile = options.profile; const testsPath = getTestRoot(path); const config = getConfig(testsPath); diff --git a/lib/command/run-multiple.js b/lib/command/run-multiple.js index 9265ae045..8bf77b4e2 100644 --- a/lib/command/run-multiple.js +++ b/lib/command/run-multiple.js @@ -29,7 +29,7 @@ let processesDone; module.exports = function (selectedRuns, options) { // registering options globally to use in config - process.env.profile = options.profile; + process.profile = options.profile; const configFile = options.config; let codecept; diff --git a/lib/command/run-rerun.js b/lib/command/run-rerun.js index 2faa0a7d2..eee1f2b65 100644 --- a/lib/command/run-rerun.js +++ b/lib/command/run-rerun.js @@ -7,7 +7,7 @@ const Codecept = require('../rerun'); module.exports = function (test, options) { // registering options globally to use in config - process.env.profile = options.profile; + process.profile = options.profile; const configFile = options.config; let codecept; diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index ef4e15c72..cd83f1b4e 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -23,7 +23,7 @@ module.exports = function (workerCount, options) { 'Required minimum Node version of 11.7.0 to work with "run-workers"', ); - process.env.profile = options.profile; + process.profile = options.profile; const { config: testConfig, override = '' } = options; const overrideConfigs = tryOrDefault(() => JSON.parse(override), {}); diff --git a/lib/command/run.js b/lib/command/run.js index d03248566..f9a65ee2f 100644 --- a/lib/command/run.js +++ b/lib/command/run.js @@ -10,7 +10,7 @@ const Codecept = require('../codecept'); module.exports = function (test, options) { // registering options globally to use in config - process.env.profile = options.profile; + process.profile = options.profile; const configFile = options.config; let codecept; diff --git a/test/data/sandbox/config.js b/test/data/sandbox/config.js index e11369349..460dea48c 100644 --- a/test/data/sandbox/config.js +++ b/test/data/sandbox/config.js @@ -1,4 +1,4 @@ -const profile = process.env.profile; +const profile = process.profile; exports.config = { tests: './*_test.js', From 4036a4aaab827b481e3a96b14515169059e8cea2 Mon Sep 17 00:00:00 2001 From: Davert Date: Tue, 5 May 2020 21:04:55 +0300 Subject: [PATCH 4/4] fixed merge for workers --- lib/workers.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/workers.js b/lib/workers.js index 737dcefb1..fe85aa95d 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -227,11 +227,14 @@ class Workers extends EventEmitter { * @param {Number} numberOfWorkers */ createGroupsOfTests(numberOfWorkers) { + const files = this.codecept.testFiles; + const mocha = Container.mocha(); + mocha.files = files; + mocha.loadFiles(); + const groups = populateGroups(numberOfWorkers); let groupCounter = 0; - const mocha = Container.mocha(); - mocha.suite.eachTest((test) => { const i = groupCounter % groups.length; if (test) {