diff --git a/.github/ISSUE_TEMPLATE/bug-report---android.md b/.github/ISSUE_TEMPLATE/bug-report---android.md index f84c7bb262..602b7c9c17 100644 --- a/.github/ISSUE_TEMPLATE/bug-report---android.md +++ b/.github/ISSUE_TEMPLATE/bug-report---android.md @@ -46,6 +46,6 @@ Device logs can be retrieved from the device using `adb logcat`, or if recorded, - Node: - Device: - OS: - - Test-runner (select one): Mocha | Jest+jasmine | Jest+jest-circus + - Test-runner (select one): `jest-circus` | `jest-jasmine2` (deprecated) | `mocha` diff --git a/detox/local-cli/init.js b/detox/local-cli/init.js index 7e4b8f6e7a..05cf299e9b 100644 --- a/detox/local-cli/init.js +++ b/detox/local-cli/init.js @@ -92,8 +92,8 @@ function createMochaFolderE2E() { function createJestFolderE2E() { createFolder('e2e', { 'config.json': jestTemplates.runnerConfig, - 'init.js': jestTemplates.initjs, - 'firstTest.spec.js': jestTemplates.firstTest + 'environment.js': jestTemplates.environment, + 'firstTest.e2e.js': jestTemplates.firstTest, }); createFile('.detoxrc.json', JSON.stringify({ diff --git a/detox/local-cli/templates/jest.js b/detox/local-cli/templates/jest.js index f38e679321..958308e38f 100644 --- a/detox/local-cli/templates/jest.js +++ b/detox/local-cli/templates/jest.js @@ -1,50 +1,40 @@ const firstTestContent = require('./firstTestContent'); + const runnerConfig = `{ - "setupFilesAfterEnv": ["./init.js"], - "testEnvironment": "node", + "testEnvironment": "./environment", + "testRunner": "jest-circus/runner", + "testTimeout": 120000, + "testRegex": "\\\\.e2e\\\\.js$", "reporters": ["detox/runners/jest/streamlineReporter"], "verbose": true } `; -const initjsContent = `const detox = require('detox'); -const adapter = require('detox/runners/jest/adapter'); -const specReporter = require('detox/runners/jest/specReporter'); - -// Set the default timeout -jest.setTimeout(120000); - -jasmine.getEnv().addReporter(adapter); - -// This takes care of generating status logs on a per-spec basis. By default, jest only reports at file-level. -// This is strictly optional. -jasmine.getEnv().addReporter(specReporter); - -beforeAll(async () => { - await detox.init(); -}, 300000); - -beforeEach(async () => { - try { - await adapter.beforeEach(); - } catch (err) { - // Workaround for the 'jest-jasmine' runner (default one): if 'beforeAll' hook above fails with a timeout, - // unfortunately, 'jest' might continue running other hooks and test suites. To prevent that behavior, - // adapter.beforeEach() will throw if detox.init() is still running; that allows us to run detox.cleanup() - // in that emergency case and disable calling 'device', 'element', 'expect', 'by' and other Detox globals. - // If you switch to 'jest-circus' runner, you can omit this try-catch workaround at all. - - await detox.cleanup(); - throw err; +const environmentJsContent = `const { + DetoxCircusEnvironment, + SpecReporter, + WorkerAssignReporter, +} = require('detox/runners/jest-circus'); + +class CustomDetoxEnvironment extends DetoxCircusEnvironment { + constructor(config) { + super(config); + + // Can be safely removed, if you are content with the default value (=300000ms) + this.initTimeout = 300000; + + // This takes care of generating status logs on a per-spec basis. By default, Jest only reports at file-level. + // This is strictly optional. + this.registerListeners({ + SpecReporter, + WorkerAssignReporter, + }); } -}); +} -afterAll(async () => { - await adapter.afterAll(); - await detox.cleanup(); -}); +module.exports = CustomDetoxEnvironment; `; -exports.initjs = initjsContent; +exports.environment = environmentJsContent; exports.firstTest = firstTestContent; exports.runnerConfig = runnerConfig; diff --git a/detox/package.json b/detox/package.json index b042512f2d..d439d8c04d 100644 --- a/detox/package.json +++ b/detox/package.json @@ -31,7 +31,7 @@ "devDependencies": { "eslint": "^4.11.0", "eslint-plugin-node": "^6.0.1", - "jest": "25.3.x", + "jest": "26.x.x", "mockdate": "^2.0.1", "prettier": "1.7.0", "react-native": "0.62.x", @@ -53,6 +53,7 @@ "proper-lockfile": "^3.0.2", "sanitize-filename": "^1.6.1", "shell-utils": "^1.0.9", + "signal-exit": "^3.0.3", "tail": "^2.0.0", "telnet-client": "1.2.8", "tempfile": "^2.0.0", diff --git a/detox/runners/integration.js b/detox/runners/integration.js new file mode 100644 index 0000000000..c4d646d36a --- /dev/null +++ b/detox/runners/integration.js @@ -0,0 +1,16 @@ +module.exports = { + lifecycle: { + onRunStart: Symbol('run_start'), + onRunDescribeStart: Symbol('run_describe_start'), + onTestStart: Symbol('test_start'), + onHookStart: Symbol('hook_start'), + onHookFailure: Symbol('hook_failure'), + onHookSuccess: Symbol('hook_success'), + onTestFnStart: Symbol('test_fn_start'), + onTestFnFailure: Symbol('test_fn_failure'), + onTestFnSuccess: Symbol('test_fn_success'), + onTestDone: Symbol('test_done'), + onRunDescribeFinish: Symbol('run_describe_finish'), + onRunFinish: Symbol('run_finish'), + }, +}; diff --git a/detox/runners/jest-circus/environment.js b/detox/runners/jest-circus/environment.js new file mode 100644 index 0000000000..d8e08de6b4 --- /dev/null +++ b/detox/runners/jest-circus/environment.js @@ -0,0 +1,111 @@ +const _ = require('lodash'); +const NodeEnvironment = require('jest-environment-node'); // eslint-disable-line node/no-extraneous-require +const DetoxCoreListener = require('./listeners/DetoxCoreListener'); +const DetoxInitErrorListener = require('./listeners/DetoxInitErrorListener'); +const assertJestCircus26 = require('./utils/assertJestCircus26'); +const wrapErrorWithNoopLifecycle = require('./utils/wrapErrorWithNoopLifecycle'); +const timely = require('../../src/utils/timely'); + +/** + * @see https://www.npmjs.com/package/jest-circus#overview + */ +class DetoxCircusEnvironment extends NodeEnvironment { + constructor(config) { + super(assertJestCircus26(config)); + + /** @private */ + this._listenerFactories = { + DetoxInitErrorListener, + DetoxCoreListener, + }; + /** @private */ + this._hookTimeout = undefined; + /** @protected */ + this.testEventListeners = []; + /** @protected */ + this.initTimeout = 300000; + } + + get detox() { + return require('../../src')._setGlobal(this.global); + } + + async handleTestEvent(event, state) { + const { name } = event; + + if (name === 'setup') { + await this._onSetup(state); + } + + await this._timely(async () => { + for (const listener of this.testEventListeners) { + if (typeof listener[name] === 'function') { + await listener[name](event, state); + } + } + }); + + if (name === 'teardown') { + await this._onTeardown(); + } + } + + _timely(fn) { + const ms = this._hookTimeout === undefined ? this.initTimeout : this._hookTimeout; + return timely(fn, ms, () => { + return new Error(`Exceeded timeout of ${ms}ms.`); + })(); + } + + async _onSetup(state) { + let detox = null; + + try { + try { + detox = await this._timely(() => this.initDetox()); + } finally { + this._hookTimeout = state.testTimeout; + } + } catch (initError) { + state.unhandledErrors.push(initError); + detox = wrapErrorWithNoopLifecycle(initError); + await this._onTeardown(); + } + + this._instantiateListeners(detox); + } + + _instantiateListeners(detoxInstance) { + for (const Listener of Object.values(this._listenerFactories)) { + this.testEventListeners.push(new Listener({ + detox: detoxInstance, + env: this, + })); + } + } + + async _onTeardown() { + try { + await this._timely(() => this.cleanupDetox()); + } catch (cleanupError) { + state.unhandledErrors.push(cleanupError); + } + } + + /** @protected */ + async initDetox() { + return this.detox.init(); + } + + /** @protected */ + async cleanupDetox() { + return this.detox.cleanup(); + } + + /** @protected */ + registerListeners(map) { + Object.assign(this._listenerFactories, map); + } +} + +module.exports = DetoxCircusEnvironment; diff --git a/detox/runners/jest-circus/index.js b/detox/runners/jest-circus/index.js new file mode 100644 index 0000000000..c393add1c2 --- /dev/null +++ b/detox/runners/jest-circus/index.js @@ -0,0 +1,9 @@ +const DetoxCircusEnvironment = require('./environment'); +const WorkerAssignReporterCircus = require('../jest/WorkerAssignReporterCircus'); +const SpecReporterCircus = require('../jest/SpecReporterCircus'); + +module.exports = { + DetoxCircusEnvironment, + SpecReporter: SpecReporterCircus, + WorkerAssignReporter: WorkerAssignReporterCircus, +}; diff --git a/detox/runners/jest-circus/listeners/DetoxCoreListener.js b/detox/runners/jest-circus/listeners/DetoxCoreListener.js new file mode 100644 index 0000000000..e8384380ef --- /dev/null +++ b/detox/runners/jest-circus/listeners/DetoxCoreListener.js @@ -0,0 +1,82 @@ +const _ = require('lodash'); +const {getFullTestName, hasTimedOut} = require('../../jest/utils'); +const { + onRunDescribeStart, + onTestStart, + onHookFailure, + onTestFnFailure, + onTestDone, + onRunDescribeFinish, +} = require('../../integration').lifecycle; + +class DetoxCoreListener { + constructor({ detox }) { + this._startedTests = new WeakSet(); + this._testsFailedBeforeStart = new WeakSet(); + this.detox = detox; + } + + async run_describe_start({describeBlock: {name, children}}) { + if (children.length) { + await this.detox[onRunDescribeStart]({ name }); + } + } + + async run_describe_finish({describeBlock: {name, children}}) { + if (children.length) { + await this.detox[onRunDescribeFinish]({ name }); + } + } + + async test_start({ test }) { + if (!_.isEmpty(test.errors)) { + this._testsFailedBeforeStart.add(test); + } + } + + async hook_start(_event, state) { + await this._onBeforeActualTestStart(state.currentlyRunningTest); + } + + async hook_failure({ error, hook }) { + await this.detox[onHookFailure]({ + error, + hook: hook.type, + }); + } + + async test_fn_start({ test }) { + await this._onBeforeActualTestStart(test); + } + + async test_fn_failure({ error }) { + await this.detox[onTestFnFailure]({ error }); + } + + async _onBeforeActualTestStart(test) { + if (!test || test.status === 'skip' || this._startedTests.has(test) || this._testsFailedBeforeStart.has(test)) { + return; + } + + this._startedTests.add(test); + + await this.detox[onTestStart]({ + title: test.name, + fullName: getFullTestName(test), + status: 'running', + }); + } + + async test_done({ test }) { + if (this._startedTests.has(test)) { + await this.detox[onTestDone]({ + title: test.name, + fullName: getFullTestName(test), + status: test.errors.length ? 'failed' : 'passed', + timedOut: hasTimedOut(test) + }); + } + } +} + +module.exports = DetoxCoreListener; diff --git a/detox/runners/jest-circus/listeners/DetoxInitErrorListener.js b/detox/runners/jest-circus/listeners/DetoxInitErrorListener.js new file mode 100644 index 0000000000..fb1e929d0e --- /dev/null +++ b/detox/runners/jest-circus/listeners/DetoxInitErrorListener.js @@ -0,0 +1,21 @@ +const _ = require('lodash'); + +class DetoxInitErrorListener { + setup(event, { unhandledErrors, rootDescribeBlock }) { + if (unhandledErrors.length > 0) { + rootDescribeBlock.mode = 'skip'; + } + } + + add_test(event, { currentDescribeBlock, rootDescribeBlock }) { + if (currentDescribeBlock === rootDescribeBlock && rootDescribeBlock.mode === 'skip') { + const currentTest = _.last(currentDescribeBlock.children); + + if (currentTest) { + currentTest.mode = 'skip'; + } + } + } +} + +module.exports = DetoxInitErrorListener; diff --git a/detox/runners/jest-circus/utils/assertJestCircus26.js b/detox/runners/jest-circus/utils/assertJestCircus26.js new file mode 100644 index 0000000000..f1b56b068b --- /dev/null +++ b/detox/runners/jest-circus/utils/assertJestCircus26.js @@ -0,0 +1,26 @@ +const fs = require('fs'); +const path = require('path'); + +function assertJestCircus26(config) { + if (!/jest-circus/.test(config.testRunner)) { + throw new Error('Cannot run tests without "jest-circus" npm package, exiting.'); + } + + const circusPackageJson = path.join(path.dirname(config.testRunner), 'package.json'); + if (!fs.existsSync(circusPackageJson)) { + throw new Error('Check that you have an installed copy of "jest-circus" npm package, exiting.'); + } + + const circusVersion = require(circusPackageJson).version || ''; + const [major] = circusVersion.split('.'); + if (major < 26) { + throw new Error( + `Cannot use older versions of "jest-circus", exiting.\n` + + `You have jest-circus@${circusVersion}. Update to ^26.0.0 or newer.` + ); + } + + return config; +} + +module.exports = assertJestCircus26; diff --git a/detox/runners/jest-circus/utils/wrapErrorWithNoopLifecycle.js b/detox/runners/jest-circus/utils/wrapErrorWithNoopLifecycle.js new file mode 100644 index 0000000000..9e0b35b53d --- /dev/null +++ b/detox/runners/jest-circus/utils/wrapErrorWithNoopLifecycle.js @@ -0,0 +1,13 @@ +const _ = require('lodash'); +const lifecycleSymbols = require('../../integration').lifecycle; + +function wrapErrorWithNoopLifecycle(rawError) { + const error = _.isError(rawError) ? rawError : new Error(rawError); + for (const symbol of Object.values(lifecycleSymbols)) { + error[symbol] = _.noop; + } + + return error; +} + +module.exports = wrapErrorWithNoopLifecycle; diff --git a/detox/runners/jest/CircusTestEventListenerBase.js b/detox/runners/jest/CircusTestEventListenerBase.js deleted file mode 100644 index 439493302a..0000000000 --- a/detox/runners/jest/CircusTestEventListenerBase.js +++ /dev/null @@ -1,59 +0,0 @@ -const _ = require('lodash'); - -class CircusTestEventListenerBase { - constructor() { - this._onBeforeEach = this._onBeforeEach.bind(this); - this._onBeforeAll = this._onBeforeAll.bind(this); - this._onAfterEach = this._onAfterEach.bind(this); - this._onAfterAll = this._onAfterAll.bind(this); - this._onSuiteStart = this._onSuiteStart.bind(this); - this._onSuiteEnd = this._onSuiteEnd.bind(this); - this._onTestStart = this._onTestStart.bind(this); - this._onTestComplete = this._onTestComplete.bind(this); - this._onTestSkip = this._onTestSkip.bind(this); - this._handleHookEvents = this._handleHookEvents.bind(this); - this._onError = this._onError.bind(this); - - this._dispatchMap = { - 'run_describe_start': this._onSuiteStart, - 'run_describe_finish': this._onSuiteEnd, - 'test_start': this._onTestStart, - 'test_done': this._onTestComplete, - 'test_skip': this._onTestSkip, - 'hook_start': this._handleHookEvents, - 'hook_failure': _.noop, // For clarity - 'hook_success': _.noop, // For clarity - 'error': this._onError, - }; - } - - async handleTestEvent(event, state) { - const fn = this._dispatchMap[event.name] || _.noop; - await fn(event, state); - } - - async _handleHookEvents(event, state) { - const { type } = event.hook; - const fnName = '_on' + type.charAt(0).toUpperCase() + type.slice(1); - const fn = this[fnName]; - await fn(event, state); - } - - _onSuiteStart(event, state) {} - _onSuiteEnd(event, state) {} - _onTestStart(event, state) {} - _onTestComplete(event, state) {} - _onTestSkip(event, state) {} - _onBeforeEach(event, state) {} - _onAfterEach(event, state) {} - _onBeforeAll(event, state) {} - _onAfterAll(event, state) {} - _onError(event, state) {} -} - -const stubEventsListener = { - handleTestEvent: _.noop, -}; - -module.exports = CircusTestEventListenerBase; -module.exports.stubEventsListener = stubEventsListener; diff --git a/detox/runners/jest/CircusTestEventListeners.js b/detox/runners/jest/CircusTestEventListeners.js deleted file mode 100644 index 2b327b04a3..0000000000 --- a/detox/runners/jest/CircusTestEventListeners.js +++ /dev/null @@ -1,17 +0,0 @@ -class CircusTestEventListeners { - constructor() { - this._listeners = []; - } - - addListener(listener) { - this._listeners.push(listener); - } - - async notifyAll(event, state) { - for (const listener of this._listeners) { - await listener.handleTestEvent(event, state); - } - } -} - -module.exports = CircusTestEventListeners; diff --git a/detox/runners/jest/DetoxAdapterCircus.js b/detox/runners/jest/DetoxAdapterCircus.js index 8989b724a9..8ec039f271 100644 --- a/detox/runners/jest/DetoxAdapterCircus.js +++ b/detox/runners/jest/DetoxAdapterCircus.js @@ -1,10 +1,9 @@ const _ = require('lodash'); -const CircusTestEventListenerBase = require('./CircusTestEventListenerBase'); const DetoxAdapter = require('./DetoxAdapterImpl'); +const { getFullTestName, hasTimedOut } = require('./utils'); -class DetoxAdapterCircus extends CircusTestEventListenerBase { +class DetoxAdapterCircus { constructor(detox) { - super(); this._adapter = new DetoxAdapter(detox, DetoxAdapterCircus._describeInitError); } @@ -25,15 +24,15 @@ class DetoxAdapterCircus extends CircusTestEventListenerBase { await this._adapter.afterAll(); } - async _onSuiteStart({describeBlock: {name, tests}}, state) { - if (tests.length) await this._adapter.suiteStart({name}); + async run_describe_start({describeBlock: {name, children}}, state) { + if (children.length) await this._adapter.suiteStart({name}); } - async _onSuiteEnd({describeBlock: {name, tests}}, state) { - if (tests.length) await this._adapter.suiteEnd({name}); + async run_describe_finish({describeBlock: {name, children}}, state) { + if (children.length) await this._adapter.suiteEnd({name}); } - _onTestStart(event) { + test_start(event) { const { test } = event; if (test.mode === 'skip' || test.mode === 'todo' || test.errors.length > 0) { return; @@ -41,44 +40,22 @@ class DetoxAdapterCircus extends CircusTestEventListenerBase { this._adapter.testStart({ title: test.name, - fullName: this._getFullTestName(test), + fullName: getFullTestName(test), status: 'running', }); } - _onTestComplete(event) { + test_done(event) { const { test } = event; this._adapter.testComplete({ status: test.errors.length ? 'failed' : 'passed', - timedOut: this._hasTimedOut(test) + timedOut: hasTimedOut(test) }); } - _onTestSkip(event) { + test_skip(event) { // Ignored (for clarity) } - - _getFullTestName(test, separator = ' ') { - let testName = ''; - for (let parent = test.parent; - parent.parent; // Since there's always an unwanted root made up by jest - parent = parent.parent) { - testName = parent.name + separator + testName; - } - testName += test.name; - return testName; - } - - _hasTimedOut(test) { - const { errors } = test; - const errorsArray = (_.isArray(errors) ? errors : [errors]); - const timedOut = _.chain(errorsArray) - .flattenDeep() - .filter(_.isObject) - .some(e => _.includes(e.message, 'Exceeded timeout')) - .value(); - return timedOut; - } } module.exports = DetoxAdapterCircus; diff --git a/detox/runners/jest/JestCircusEnvironment.js b/detox/runners/jest/JestCircusEnvironment.js index a558f5fc03..6b98d3c8b0 100644 --- a/detox/runners/jest/JestCircusEnvironment.js +++ b/detox/runners/jest/JestCircusEnvironment.js @@ -1,5 +1,4 @@ const NodeEnvironment = require('jest-environment-node'); // eslint-disable-line node/no-extraneous-require -const CircusTestEventListeners = require('./CircusTestEventListeners'); /** * @see https://www.npmjs.com/package/jest-circus#overview @@ -7,7 +6,7 @@ const CircusTestEventListeners = require('./CircusTestEventListeners'); class JestCircusEnvironment extends NodeEnvironment { constructor(config) { super(config); - this.testEventListeners = new CircusTestEventListeners(); + this.testEventListeners = []; // Enable access to this instance (single in each worker's scope) by exposing a get-function. // Note: whatever's set into this.global will be exported in that way. The reason behind it is that @@ -20,11 +19,17 @@ class JestCircusEnvironment extends NodeEnvironment { } addEventsListener(listener) { - this.testEventListeners.addListener(listener); + this.testEventListeners.push(listener); } async handleTestEvent(event, state) { - await this.testEventListeners.notifyAll(event, state); + const name = event.name; + + for (const listener of this.testEventListeners) { + if (typeof listener[name] === 'function') { + await listener[name](event, state); + } + } } } diff --git a/detox/runners/jest/ReporterBase.js b/detox/runners/jest/ReporterBase.js deleted file mode 100644 index a5ba6c539e..0000000000 --- a/detox/runners/jest/ReporterBase.js +++ /dev/null @@ -1,12 +0,0 @@ -class ReporterBase { - _traceln(message) { - this._trace(message); - process.stdout.write('\n'); - } - - _trace(message) { - process.stdout.write(message); - } -} - -module.exports = ReporterBase; diff --git a/detox/runners/jest/SpecReporterCircus.js b/detox/runners/jest/SpecReporterCircus.js index cade356a40..fa61de6532 100644 --- a/detox/runners/jest/SpecReporterCircus.js +++ b/detox/runners/jest/SpecReporterCircus.js @@ -1,13 +1,12 @@ -const CircusTestEventListenerBase = require('./CircusTestEventListenerBase'); +const argparse = require('../../src/utils/argparse'); const SpecReporter = require('./SpecReporterImpl'); -class SpecReporterCircus extends CircusTestEventListenerBase { +class SpecReporterCircus { constructor() { - super(); this._specReporter = new SpecReporter(); } - _onSuiteStart(event) { + run_describe_start(event) { if (event.describeBlock.parent !== undefined) { this._specReporter.onSuiteStart({ description: event.describeBlock.name, @@ -15,13 +14,13 @@ class SpecReporterCircus extends CircusTestEventListenerBase { } } - _onSuiteEnd(event) { + run_describe_finish(event) { if (event.describeBlock.parent !== undefined) { this._specReporter.onSuiteEnd(); } } - _onTestStart(event) { + test_start(event) { const { test } = event; this._specReporter.onTestStart({ description: test.name, @@ -29,7 +28,7 @@ class SpecReporterCircus extends CircusTestEventListenerBase { }); } - _onTestComplete(event) { + test_done(event) { const { test } = event; const testInfo = { description: test.name, @@ -38,7 +37,7 @@ class SpecReporterCircus extends CircusTestEventListenerBase { this._specReporter.onTestEnd(testInfo, test.errors.length ? 'failed' : 'success'); } - _onTestSkip(event) { + test_skip(event) { const testInfo = { description: event.test.name, }; @@ -46,4 +45,6 @@ class SpecReporterCircus extends CircusTestEventListenerBase { } } -module.exports = SpecReporterCircus; +module.exports = argparse.getArgValue('reportSpecs') === 'true' + ? SpecReporterCircus + : class {}; diff --git a/detox/runners/jest/SpecReporterImpl.js b/detox/runners/jest/SpecReporterImpl.js index af9ba1fb75..5bebd2cab0 100644 --- a/detox/runners/jest/SpecReporterImpl.js +++ b/detox/runners/jest/SpecReporterImpl.js @@ -1,5 +1,5 @@ const chalk = require('chalk').default; -const ReporterBase = require('./ReporterBase'); +const { traceln } = require('./utils/stdout'); const log = require('../../src/utils/logger').child(); const RESULT_SKIPPED = chalk.yellow('SKIPPED'); @@ -8,9 +8,8 @@ const RESULT_PENDING = chalk.yellow('PENDING'); const RESULT_SUCCESS = chalk.green('OK'); const RESULT_OTHER = 'UNKNOWN'; -class SpecReporter extends ReporterBase { +class SpecReporter { constructor() { - super(); this._suites = []; this._suitesDesc = ''; } @@ -25,7 +24,7 @@ class SpecReporter extends ReporterBase { this._regenerateSuitesDesc(); if (!this._suites.length) { - this._traceln(''); + traceln(''); } } diff --git a/detox/runners/jest/WorkerAssignReporterCircus.js b/detox/runners/jest/WorkerAssignReporterCircus.js index d9404260b5..6385ea51a6 100644 --- a/detox/runners/jest/WorkerAssignReporterCircus.js +++ b/detox/runners/jest/WorkerAssignReporterCircus.js @@ -1,14 +1,13 @@ -const CircusTestEventListenerBase = require('./CircusTestEventListenerBase'); const WorkerAssignReporter = require('./WorkerAssignReporterImpl'); -class WorkerAssignReporterCircus extends CircusTestEventListenerBase { - constructor(detox) { - super(); +class WorkerAssignReporterCircus { + constructor({ detox }) { this._reporter = new WorkerAssignReporter(detox); } - _onSuiteStart(event) { + run_describe_start(event) { const { describeBlock } = event; + if (describeBlock.parent && describeBlock.parent.parent === undefined) { this._reporter.report(describeBlock.name); } diff --git a/detox/runners/jest/WorkerAssignReporterImpl.js b/detox/runners/jest/WorkerAssignReporterImpl.js index 23a76bfdc8..0dc80f6d83 100644 --- a/detox/runners/jest/WorkerAssignReporterImpl.js +++ b/detox/runners/jest/WorkerAssignReporterImpl.js @@ -1,12 +1,10 @@ const _ = require('lodash'); const chalk = require('chalk').default; -const ReporterBase = require('./ReporterBase'); const log = require('../../src/utils/logger').child(); -class WorkerAssignReporterImpl extends ReporterBase { +class WorkerAssignReporterImpl { constructor(detox) { - super(); - this.device = detox.device; + this.device = detox && detox.device; } report(workerName) { diff --git a/detox/runners/jest/WorkerAssignReporterJasmine.js b/detox/runners/jest/WorkerAssignReporterJasmine.js index 69f75237c4..3b9756b1b4 100644 --- a/detox/runners/jest/WorkerAssignReporterJasmine.js +++ b/detox/runners/jest/WorkerAssignReporterJasmine.js @@ -2,7 +2,7 @@ const path = require('path'); const WorkerAssignReporter = require('./WorkerAssignReporterImpl'); class WorkerAssignReporterJasmine { - constructor(detox) { + constructor({ detox }) { this._reporter = new WorkerAssignReporter(detox); } diff --git a/detox/runners/jest/assignReporter.js b/detox/runners/jest/assignReporter.js index f57e8207b8..0d6d1166b3 100644 --- a/detox/runners/jest/assignReporter.js +++ b/detox/runners/jest/assignReporter.js @@ -2,4 +2,4 @@ const detox = require('../../src/index'); const runnerInfo = require('./runnerInfo'); const Reporter = runnerInfo.isJestCircus ? require('./WorkerAssignReporterCircus') : require('./WorkerAssignReporterJasmine'); -module.exports = new Reporter(detox); +module.exports = new Reporter({ detox }); diff --git a/detox/runners/jest/specReporter.js b/detox/runners/jest/specReporter.js index e3c235840e..75c0fd1162 100644 --- a/detox/runners/jest/specReporter.js +++ b/detox/runners/jest/specReporter.js @@ -5,5 +5,5 @@ if (argparse.getArgValue('reportSpecs') === 'true') { const Reporter = runnerInfo.isJestCircus ? require('./SpecReporterCircus') : require('./SpecReporterJasmine'); module.exports = new Reporter(); } else { - module.exports = runnerInfo.isJestCircus ? require('./CircusTestEventListenerBase').stubEventsListener : {}; + module.exports = {}; } diff --git a/detox/runners/jest/utils/getFullTestName.js b/detox/runners/jest/utils/getFullTestName.js new file mode 100644 index 0000000000..eb63066486 --- /dev/null +++ b/detox/runners/jest/utils/getFullTestName.js @@ -0,0 +1,12 @@ +function getFullTestName(test, separator = ' ') { + let testName = ''; + for (let parent = test.parent; + parent.parent; // Since there's always an unwanted root made up by jest + parent = parent.parent) { + testName = parent.name + separator + testName; + } + testName += test.name; + return testName; +} + +module.exports = getFullTestName; diff --git a/detox/runners/jest/utils/hasTimedOut.js b/detox/runners/jest/utils/hasTimedOut.js new file mode 100644 index 0000000000..69c471eade --- /dev/null +++ b/detox/runners/jest/utils/hasTimedOut.js @@ -0,0 +1,15 @@ +const _ = require('lodash'); + +function hasTimedOut(test) { + const { errors } = test; + const errorsArray = (_.isArray(errors) ? errors : [errors]); + const timedOut = _.chain(errorsArray) + .flattenDeep() + .filter(_.isObject) + .some(e => _.includes(e.message, 'Exceeded timeout')) + .value(); + + return timedOut; +} + +module.exports = hasTimedOut; diff --git a/detox/runners/jest/utils/index.js b/detox/runners/jest/utils/index.js new file mode 100644 index 0000000000..e0cb81f616 --- /dev/null +++ b/detox/runners/jest/utils/index.js @@ -0,0 +1,4 @@ +module.exports = { + getFullTestName: require('./getFullTestName'), + hasTimedOut: require('./hasTimedOut'), +}; diff --git a/detox/runners/jest/utils/stdout.js b/detox/runners/jest/utils/stdout.js new file mode 100644 index 0000000000..19fce90080 --- /dev/null +++ b/detox/runners/jest/utils/stdout.js @@ -0,0 +1,18 @@ +const { EOL } = require('os'); + +function trace(message) { + process.stdout.write(message); +} + +function traceln(message) { + if (message) { + trace(message); + } + + trace(EOL); +} + +module.exports = { + trace, + traceln, +}; diff --git a/detox/src/Detox.js b/detox/src/Detox.js index 22c28b7c03..099961d5ce 100644 --- a/detox/src/Detox.js +++ b/detox/src/Detox.js @@ -17,6 +17,8 @@ const driverRegistry = require('./devices/DriverRegistry').default; const _initHandle = Symbol('_initHandle'); const _assertNoPendingInit = Symbol('_assertNoPendingInit'); +const lifecycleSymbols = require('../runners/integration').lifecycle; + class Detox { constructor(config) { log.trace( @@ -27,6 +29,13 @@ class Detox { this[_initHandle] = null; + for (const [key, symbol] of Object.entries(lifecycleSymbols)) { + this[symbol] = (...args) => this._artifactsManager[key](...args); + } + + this[lifecycleSymbols.onTestStart] = this.beforeEach; + this[lifecycleSymbols.onTestDone] = this.afterEach; + const {artifactsConfig, behaviorConfig, deviceConfig, sessionConfig} = config; this._artifactsConfig = artifactsConfig; @@ -121,14 +130,6 @@ class Detox { }); } - async suiteStart(suite) { - await this._artifactsManager.onSuiteStart(suite); - } - - async suiteEnd(suite) { - await this._artifactsManager.onSuiteEnd(suite); - } - async _doInit() { const behaviorConfig = this._behaviorConfig.init; const sessionConfig = this._sessionConfig; @@ -158,7 +159,7 @@ class Detox { }); if (behaviorConfig.exposeGlobals) { - Object.assign(global, { + Object.assign(Detox.global, { ...deviceDriver.matchers, device: this.device, }); @@ -267,5 +268,6 @@ class Detox { } Detox.none = new MissingDetox(); +Detox.global = global; module.exports = Detox; diff --git a/detox/src/Detox.test.js b/detox/src/Detox.test.js index cb41ababfd..d8764c2748 100644 --- a/detox/src/Detox.test.js +++ b/detox/src/Detox.test.js @@ -24,6 +24,7 @@ describe('Detox', () => { let ArtifactsManager; let Detox; let detox; + let lifecycleSymbols; function client() { return Client.mock.instances[0]; @@ -57,6 +58,7 @@ describe('Detox', () => { Client = require('./client/Client'); DetoxServer = require('./server/DetoxServer'); Detox = require('./Detox'); + lifecycleSymbols = require('../runners/integration').lifecycle; }); describe('when detox.init() is called', () => { @@ -532,27 +534,27 @@ describe('Detox', () => { }); }); - describe('when detox.suiteStart() is called', () => { + describe.each([ + ['onRunStart', null], + ['onRunDescribeStart', { name: 'testSuiteName' }], + ['onTestStart', testSummaries.running()], + ['onHookStart', null], + ['onHookFailure', { error: new Error() }], + ['onHookSuccess', null], + ['onTestFnStart', null], + ['onTestFnFailure', { error: new Error() }], + ['onTestFnSuccess', null], + ['onTestDone', testSummaries.passed()], + ['onRunDescribeFinish', { name: 'testSuiteName' }], + ['onRunFinish', null], + ])('when detox[symbols.%s](%j) is called', (method, arg) => { beforeEach(async () => { detox = await new Detox(detoxConfig).init(); }); - it(`should pass suite info to artifactsManager.onSuiteStart`, async () => { - const suite = { name: 'testSuiteName' }; - await detox.suiteStart(suite); - expect(artifactsManager().onSuiteStart).toHaveBeenCalledWith(suite); - }); - }); - - describe('when detox.suiteEnd() is called', () => { - beforeEach(async () => { - detox = await new Detox(detoxConfig).init(); - }); - - it(`should pass suite info to artifactsManager.onSuiteStart`, async () => { - const suite = { name: 'testSuiteName' }; - await detox.suiteEnd(suite); - expect(artifactsManager().onSuiteEnd).toHaveBeenCalledWith(suite); + it(`should pass it through to artifactsManager.${method}()`, async () => { + await detox[lifecycleSymbols[method]](arg); + expect(artifactsManager()[method]).toHaveBeenCalledWith(arg); }); }); }); diff --git a/detox/src/DetoxExportWrapper.js b/detox/src/DetoxExportWrapper.js index f6b2b87224..9c84340b84 100644 --- a/detox/src/DetoxExportWrapper.js +++ b/detox/src/DetoxExportWrapper.js @@ -32,6 +32,8 @@ class DetoxExportWrapper { async init(configOverride, userParams) { let configError, exposeGlobals, resolvedConfig; + log.ensureLogFiles(); + try { resolvedConfig = await configuration.composeDetoxConfig({ override: configOverride, @@ -46,7 +48,7 @@ class DetoxExportWrapper { try { if (exposeGlobals) { - Detox.none.initContext(global); + Detox.none.initContext(Detox.global); } if (configError) { @@ -59,21 +61,19 @@ class DetoxExportWrapper { return this[_detox]; } catch (err) { - Detox.none.setError(err); - log.error({ event: 'DETOX_INIT_ERROR' }, '\n', err); + + Detox.none.setError(err); throw err; } } async cleanup() { - try { - if (this[_detox] !== Detox.none) { - await this[_detox].cleanup(); - } - } finally { + Detox.none.cleanupContext(Detox.global); + + if (this[_detox] !== Detox.none) { + await this[_detox].cleanup(); this[_detox] = Detox.none; - Detox.none.cleanupContext(global); } } @@ -86,6 +86,12 @@ class DetoxExportWrapper { _defineProxy(name) { this[name] = funpermaproxy(() => this[_detox][name]); } + + /** Use for test runners with sandboxed global */ + _setGlobal(global) { + Detox.global = global; + return this; + } } module.exports = DetoxExportWrapper; diff --git a/detox/src/artifacts/ArtifactsManager.js b/detox/src/artifacts/ArtifactsManager.js index a63ae4b6c0..293f8a278a 100644 --- a/detox/src/artifacts/ArtifactsManager.js +++ b/detox/src/artifacts/ArtifactsManager.js @@ -84,6 +84,18 @@ class ArtifactsManager { await this._callPlugins('plain', 'onBootDevice', deviceInfo); } + async onBeforeLaunchApp(appLaunchInfo) { + await this._callPlugins('plain', 'onBeforeLaunchApp', appLaunchInfo); + } + + async onLaunchApp(appLaunchInfo) { + await this._callPlugins('plain', 'onLaunchApp', appLaunchInfo); + } + + async onAppReady(appInfo) { + await this._callPlugins('plain', 'onAppReady', appInfo); + } + async onBeforeTerminateApp(appInfo) { await this._callPlugins('plain', 'onBeforeTerminateApp', appInfo); } @@ -104,18 +116,6 @@ class ArtifactsManager { await this._callPlugins('plain', 'onShutdownDevice', deviceInfo); } - async onBeforeLaunchApp(appLaunchInfo) { - await this._callPlugins('plain', 'onBeforeLaunchApp', appLaunchInfo); - } - - async onLaunchApp(appLaunchInfo) { - await this._callPlugins('plain', 'onLaunchApp', appLaunchInfo); - } - - async onAppReady(appInfo) { - await this._callPlugins('plain', 'onAppReady', appInfo); - } - async onCreateExternalArtifact({ pluginId, artifactName, artifactPath }) { await this._callSinglePlugin(pluginId, 'onCreateExternalArtifact', { artifact: new FileArtifact({ temporaryPath: artifactPath }), @@ -123,21 +123,41 @@ class ArtifactsManager { }); } + async onRunStart() {} + + async onRunDescribeStart(suite) { + await this._callPlugins('ascending', 'onRunDescribeStart', suite); + } + async onTestStart(testSummary) { await this._callPlugins('ascending', 'onTestStart', testSummary); } + async onHookStart() {} + + async onHookFailure(testSummary) { + await this._callPlugins('plain', 'onHookFailure', testSummary); + } + + async onHookSuccess() {} + + async onTestFnStart() {} + + async onTestFnFailure(testSummary) { + await this._callPlugins('plain', 'onTestFnFailure', testSummary); + } + + async onTestFnSuccess() {} + async onTestDone(testSummary) { await this._callPlugins('descending', 'onTestDone', testSummary); } - async onSuiteStart(suite) { - await this._callPlugins('descending', 'onSuiteStart', suite); + async onRunDescribeFinish(suite) { + await this._callPlugins('descending', 'onRunDescribeFinish', suite); } - async onSuiteEnd(suite) { - await this._callPlugins('descending', 'onSuiteEnd', suite); - } + async onRunFinish() {} async onBeforeCleanup() { await this._callPlugins('descending', 'onBeforeCleanup'); diff --git a/detox/src/artifacts/ArtifactsManager.test.js b/detox/src/artifacts/ArtifactsManager.test.js index 2e679ecb74..c1ca455cf2 100644 --- a/detox/src/artifacts/ArtifactsManager.test.js +++ b/detox/src/artifacts/ArtifactsManager.test.js @@ -2,6 +2,8 @@ const path = require('path'); const sleep = require('../utils/sleep'); const testSummaries = require('./__mocks__/testSummaries.mock'); const testSuite = require('./templates/plugin/__mocks__/testSuite.mock'); +const testHookError = () => ({ hook: 'beforeEach', error: new Error() }); +const testError = () => ({ error: new Error() }); describe('ArtifactsManager', () => { let proxy, FakePathBuilder; @@ -89,9 +91,11 @@ describe('ArtifactsManager', () => { onAppReady: jest.fn(), onCreateExternalArtifact: jest.fn(), onTestStart: jest.fn(), + onHookFailure: jest.fn(), + onTestFnFailure: jest.fn(), onTestDone: jest.fn(), - onSuiteStart: jest.fn(), - onSuiteEnd: jest.fn(), + onRunDescribeStart: jest.fn(), + onRunDescribeFinish: jest.fn(), onBeforeCleanup: jest.fn(), }); }; @@ -221,11 +225,15 @@ describe('ArtifactsManager', () => { itShouldCatchErrorsOnPhase('onTestStart', () => testSummaries.running()); + itShouldCatchErrorsOnPhase('onHookFailure', () => testHookError()); + + itShouldCatchErrorsOnPhase('onTestFnFailure', () => testError()); + itShouldCatchErrorsOnPhase('onTestDone', () => testSummaries.passed()); - itShouldCatchErrorsOnPhase('onSuiteStart', () => (testSuite.mock())); + itShouldCatchErrorsOnPhase('onRunDescribeStart', () => (testSuite.mock())); - itShouldCatchErrorsOnPhase('onSuiteEnd', () => (testSuite.mock())); + itShouldCatchErrorsOnPhase('onRunDescribeFinish', () => (testSuite.mock())); itShouldCatchErrorsOnPhase('onBeforeCleanup', () => undefined); @@ -291,6 +299,26 @@ describe('ArtifactsManager', () => { }); }); + describe('onHookFailure', () => { + it('should call onHookFailure in plugins with the passed argument', async () => { + const error = testError(); + + expect(testPlugin.onHookFailure).not.toHaveBeenCalled(); + await artifactsManager.onHookFailure(error); + expect(testPlugin.onHookFailure).toHaveBeenCalledWith(error); + }); + }); + + describe('onTestFnFailure', () => { + it('should call onTestFnFailure in plugins with the passed argument', async () => { + const error = testError(); + + expect(testPlugin.onTestFnFailure).not.toHaveBeenCalled(); + await artifactsManager.onTestFnFailure(error); + expect(testPlugin.onTestFnFailure).toHaveBeenCalledWith(error); + }); + }); + describe('onTestDone', () => { it('should call onTestDone in plugins with the passed argument', async () => { const testSummary = testSummaries.passed(); @@ -301,23 +329,23 @@ describe('ArtifactsManager', () => { }); }); - describe('onSuiteStart', () => { - it('should call onSuiteStart in plugins with the passed argument', async () => { + describe('onRunDescribeStart', () => { + it('should call onRunDescribeStart in plugins with the passed argument', async () => { const suite = testSuite.mock(); - expect(testPlugin.onSuiteStart).not.toHaveBeenCalled(); - await artifactsManager.onSuiteStart(suite); - expect(testPlugin.onSuiteStart).toHaveBeenCalledWith(suite); + expect(testPlugin.onRunDescribeStart).not.toHaveBeenCalled(); + await artifactsManager.onRunDescribeStart(suite); + expect(testPlugin.onRunDescribeStart).toHaveBeenCalledWith(suite); }); }); - describe('onSuiteEnd', () => { - it('should call onSuiteEnd in plugins with the passed argument', async () => { + describe('onRunDescribeFinish', () => { + it('should call onRunDescribeFinish in plugins with the passed argument', async () => { const suite = testSuite.mock(); - expect(testPlugin.onSuiteEnd).not.toHaveBeenCalled(); - await artifactsManager.onSuiteEnd(suite); - expect(testPlugin.onSuiteEnd).toHaveBeenCalledWith(suite); + expect(testPlugin.onRunDescribeFinish).not.toHaveBeenCalled(); + await artifactsManager.onRunDescribeFinish(suite); + expect(testPlugin.onRunDescribeFinish).toHaveBeenCalledWith(suite); }); }); @@ -499,4 +527,22 @@ describe('ArtifactsManager', () => { expect(emitter.on).toHaveBeenCalledWith(eventName, expect.any(Function)); }); }); + + describe('stubs', () => { + it.each([ + ['onRunStart'], + ['onHookStart'], + ['onHookSuccess'], + ['onTestFnStart'], + ['onTestFnSuccess'], + ['onRunFinish'], + ])('should have async .%s() stub', (method) => { + const artifactsManager = new proxy.ArtifactsManager({ + pathBuilder: new proxy.ArtifactPathBuilder({ rootDir: '/tmp' }), + plugins: {}, + }); + + expect(artifactsManager[method]()).toEqual(Promise.resolve()); + }); + }); }); diff --git a/detox/src/artifacts/templates/plugin/ArtifactPlugin.js b/detox/src/artifacts/templates/plugin/ArtifactPlugin.js index 1bcdfb8441..5a7eaae17a 100644 --- a/detox/src/artifacts/templates/plugin/ArtifactPlugin.js +++ b/detox/src/artifacts/templates/plugin/ArtifactPlugin.js @@ -206,6 +206,32 @@ class ArtifactPlugin { this.context.testSummary = testSummary; } + /** + * Hook that is called if a hook of a test fails + * e.g.: beforeAll, beforeEach, afterEach, afterAll + * + * @protected + * @async + * @param {string} failureDetails.hook + * @param {*} failureDetails.error + * @return {Promise} - when done + */ + async onHookFailure(failureDetails) { + this._hasFailingTests = true; + } + + /** + * Hook that is called if a test function fails + * + * @protected + * @async + * @param {*} failureDetails.error + * @return {Promise} - when done + */ + async onTestFnFailure(failureDetails) { + this._hasFailingTests = true; + } + /*** * @protected * @async @@ -228,7 +254,7 @@ class ArtifactPlugin { * @param {Suite} suite - has name of currently running test suite * @return {Promise} - when done */ - async onSuiteStart(suite) { + async onRunDescribeStart(suite) { this.context.suite = suite; } @@ -240,7 +266,7 @@ class ArtifactPlugin { * @param {Suite} suite - has name of currently running test suite * @return {Promise} - when done */ - async onSuiteEnd(suite) { + async onRunDescribeFinish(suite) { this.context.suite = null; } @@ -275,11 +301,12 @@ class ArtifactPlugin { this.onTerminateApp = _.noop; this.onBeforeLaunchApp = _.noop; this.onLaunchApp = _.noop; - this.onUserAction = _.noop; this.onTestStart = _.noop; + this.onHookFailure = _.noop; + this.onTestFnFailure = _.noop; this.onTestDone = _.noop; - this.onSuiteStart = _.noop; - this.onSuiteEnd = _.noop; + this.onRunDescribeStart = _.noop; + this.onRunDescribeFinish = _.noop; this.onBeforeCleanup = _.noop; } diff --git a/detox/src/artifacts/templates/plugin/ArtifactPlugin.test.js b/detox/src/artifacts/templates/plugin/ArtifactPlugin.test.js index f321c6ddf7..9fba1b49f4 100644 --- a/detox/src/artifacts/templates/plugin/ArtifactPlugin.test.js +++ b/detox/src/artifacts/templates/plugin/ArtifactPlugin.test.js @@ -223,21 +223,39 @@ describe('ArtifactPlugin', () => { expect(plugin.context.testSummary).toBe(testSummary); }); + it('should have .onHookFailure(), which remembers that there were failing tests ', async () => { + plugin.enabled = true; + plugin.keepOnlyFailedTestsArtifacts = true; + + expect(plugin.shouldKeepArtifactOfSession()).toBe(undefined); + await plugin.onHookFailure({ error: new Error, hook: 'beforeEach' }); + expect(plugin.shouldKeepArtifactOfSession()).toBe(true); + }); + + it('should have .onTestFnFailure(), which remembers that there were failing tests ', async () => { + plugin.enabled = true; + plugin.keepOnlyFailedTestsArtifacts = true; + + expect(plugin.shouldKeepArtifactOfSession()).toBe(undefined); + await plugin.onTestFnFailure({ error: new Error }); + expect(plugin.shouldKeepArtifactOfSession()).toBe(true); + }); + it('should have .onTestDone, which updates context.testSummary if called', async () => { const testSummary = testSummaries.failed(); await plugin.onTestDone(testSummary); expect(plugin.context.testSummary).toBe(testSummary); }); - it('should have .onSuiteStart, which updates context.suite if called', async () => { + it('should have .onRunDescribeStart, which updates context.suite if called', async () => { const suite = testSuite.mock(); - await plugin.onSuiteStart(suite); + await plugin.onRunDescribeStart(suite); expect(plugin.context.suite).toBe(suite); }); - it('should have .onSuiteEnd, which updates context.suite if called', async () => { + it('should have .onRunDescribeFinish, which updates context.suite if called', async () => { plugin.context.suite = testSuite.mock(); - await plugin.onSuiteEnd(); + await plugin.onRunDescribeFinish(); expect(plugin.context.suite).toBe(null); }); @@ -266,10 +284,9 @@ describe('ArtifactPlugin', () => { expect(plugin.onTerminateApp).toBe(plugin.onTerminate); expect(plugin.onTestStart).toBe(plugin.onTerminate); expect(plugin.onTestDone).toBe(plugin.onTerminate); - expect(plugin.onSuiteStart).toBe(plugin.onTerminate); - expect(plugin.onSuiteEnd).toBe(plugin.onTerminate); + expect(plugin.onRunDescribeStart).toBe(plugin.onTerminate); + expect(plugin.onRunDescribeFinish).toBe(plugin.onTerminate); expect(plugin.onBeforeCleanup).toBe(plugin.onTerminate); - expect(plugin.onUserAction).toBe(plugin.onTerminate); }); it('should not work after the first call', async () => { diff --git a/detox/src/artifacts/templates/plugin/TwoSnapshotsPerTestPlugin.js b/detox/src/artifacts/templates/plugin/TwoSnapshotsPerTestPlugin.js index 9d1add16ae..b25ab7f80d 100644 --- a/detox/src/artifacts/templates/plugin/TwoSnapshotsPerTestPlugin.js +++ b/detox/src/artifacts/templates/plugin/TwoSnapshotsPerTestPlugin.js @@ -9,10 +9,19 @@ class TwoSnapshotsPerTestPlugin extends ArtifactPlugin { this.shouldTakeAutomaticSnapshots = this.api.userConfig.shouldTakeAutomaticSnapshots; this.keepOnlyFailedTestsArtifacts = this.api.userConfig.keepOnlyFailedTestsArtifacts; - this.takeAutomaticSnapshots = this.api.userConfig.takeWhen || { - testStart: true, - testDone: true, - }; + + this.takeAutomaticSnapshots = this.api.userConfig.takeWhen + ? { + testStart: false, + testFailure: true, + testDone: false, + ...this.api.userConfig.takeWhen + } + : { + testStart: true, + testFailure: true, + testDone: true + }; this.snapshots = { fromTest: {}, @@ -28,6 +37,20 @@ class TwoSnapshotsPerTestPlugin extends ArtifactPlugin { await this._takeAutomaticSnapshot('testStart'); } + async onHookFailure(event) { + await super.onHookFailure(event); + + const shouldTake = this.takeAutomaticSnapshots.testFailure; + await this._takeAutomaticSnapshot(`${event.hook}Failure`, shouldTake); + } + + async onTestFnFailure(event) { + await super.onTestFnFailure(event); + + const shouldTake = this.takeAutomaticSnapshots.testFailure; + await this._takeAutomaticSnapshot('testFnFailure', shouldTake); + } + async onTestDone(testSummary) { await super.onTestDone(testSummary); @@ -71,9 +94,9 @@ class TwoSnapshotsPerTestPlugin extends ArtifactPlugin { */ createTestArtifact() {} - async _takeAutomaticSnapshot(name) { + async _takeAutomaticSnapshot(name, force) { if (this.enabled && this.shouldTakeAutomaticSnapshots) { - if (this.takeAutomaticSnapshots[name]) { + if (this.takeAutomaticSnapshots[name] || force) { await this._takeSnapshot(name); } } diff --git a/detox/src/artifacts/templates/plugin/TwoSnapshotsPerTestPlugin.test.js b/detox/src/artifacts/templates/plugin/TwoSnapshotsPerTestPlugin.test.js index 47a9da9d38..c203279a33 100644 --- a/detox/src/artifacts/templates/plugin/TwoSnapshotsPerTestPlugin.test.js +++ b/detox/src/artifacts/templates/plugin/TwoSnapshotsPerTestPlugin.test.js @@ -161,6 +161,96 @@ describe('TwoSnapshotsPerTestPlugin', () => { }); }); + describe('when takeWhen.testFailure is false', () => { + beforeEach(() => plugin.configureAutomaticSnapshots({ testFailure: false })); + beforeEach(() => plugin.onTestStart(testSummaries.running())); + + describe('when onHookFailure called', function() { + beforeEach(async () => { + await plugin.onHookFailure({ hook: 'beforeEach', error: new Error() }); + }); + + it('should not create any tests artifacts', () => { + expect(plugin.createTestArtifact).not.toHaveBeenCalled(); + expect(plugin.snapshots.fromTest['beforeEachFailure']).toBe(undefined); + }); + }); + + describe('when onTestFnFailure called', function() { + beforeEach(async () => { + await plugin.onTestFnFailure({ error: new Error() }); + }); + + it('should not create any tests artifacts', () => { + expect(plugin.createTestArtifact).not.toHaveBeenCalled(); + expect(plugin.snapshots.fromTest['testFailure']).toBe(undefined); + }); + }); + }); + + describe('when takeWhen.testFailure is true', () => { + beforeEach(() => plugin.configureAutomaticSnapshots({ testFailure: true })); + + describe('when onHookFailure (beforeAll) called', function() { + beforeEach(async () => { + await plugin.onHookFailure({ hook: 'beforeAll', error: new Error() }); + }); + + it('should create test artifact', () => { + expect(plugin.createTestArtifact).toHaveBeenCalledTimes(1); + }); + + it('should start and stop recording in the artifact', () => { + expect(plugin.snapshots.fromSession['beforeAllFailure'].start).toHaveBeenCalledTimes(1); + expect(plugin.snapshots.fromSession['beforeAllFailure'].stop).toHaveBeenCalledTimes(1); + }); + + it('should put the artifact under tracking', () => { + expect(api.trackArtifact).toHaveBeenCalledWith(plugin.snapshots.fromSession['beforeAllFailure']); + }); + }); + + describe('when onHookFailure (beforeEach) called', function() { + beforeEach(async () => { + await plugin.onTestStart(testSummaries.running()); + await plugin.onHookFailure({ hook: 'beforeEach', error: new Error() }); + }); + + it('should create test artifact', () => { + expect(plugin.createTestArtifact).toHaveBeenCalledTimes(1); + }); + + it('should start and stop recording in the artifact', () => { + expect(plugin.snapshots.fromTest['beforeEachFailure'].start).toHaveBeenCalledTimes(1); + expect(plugin.snapshots.fromTest['beforeEachFailure'].stop).toHaveBeenCalledTimes(1); + }); + + it('should put the artifact under tracking', () => { + expect(api.trackArtifact).toHaveBeenCalledWith(plugin.snapshots.fromTest['beforeEachFailure']); + }); + }); + + describe('when onTestFnFailure called', function() { + beforeEach(async () => { + await plugin.onTestStart(testSummaries.running()); + await plugin.onTestFnFailure({ error: new Error() }); + }); + + it('should create test artifact', () => { + expect(plugin.createTestArtifact).toHaveBeenCalledTimes(1); + }); + + it('should start and stop recording in the artifact', () => { + expect(plugin.snapshots.fromTest['testFnFailure'].start).toHaveBeenCalledTimes(1); + expect(plugin.snapshots.fromTest['testFnFailure'].stop).toHaveBeenCalledTimes(1); + }); + + it('should put the artifact under tracking', () => { + expect(api.trackArtifact).toHaveBeenCalledWith(plugin.snapshots.fromTest['testFnFailure']); + }); + }); + }); + describe('onCreateExternalArtifact', () => { it('should throw error if { artifact } is not defined', async () => { await expect(plugin.onCreateExternalArtifact({ name: 'Hello'})).rejects.toThrowError(); diff --git a/detox/src/artifacts/timeline/TimelineArtifactPlugin.js b/detox/src/artifacts/timeline/TimelineArtifactPlugin.js index 08c06f05b2..9ee1ce8759 100644 --- a/detox/src/artifacts/timeline/TimelineArtifactPlugin.js +++ b/detox/src/artifacts/timeline/TimelineArtifactPlugin.js @@ -37,26 +37,24 @@ class TimelineArtifactPlugin extends ArtifactPlugin { this._trace.startThread({id: deviceId, name: type}); } - async onSuiteStart(suite) { - super.onSuiteStart(suite); + async onRunDescribeStart(suite) { + await super.onRunDescribeStart(suite); this._trace.beginEvent(suite.name, {deviceId: this._deviceId}); } - async onSuiteEnd(suite) { - super.onSuiteEnd(suite); + async onRunDescribeFinish(suite) { this._trace.finishEvent(suite.name); + await super.onRunDescribeFinish(suite); } async onTestStart(testSummary) { - super.onTestStart(testSummary); - + await super.onTestStart(testSummary); this._trace.beginEvent(testSummary.title); } async onTestDone(testSummary) { - super.onTestDone(testSummary); - this._trace.finishEvent(testSummary.title, {status: testSummary.status}); + await super.onTestDone(testSummary); } async onBeforeCleanup() { diff --git a/detox/src/artifacts/timeline/TimelineArtifactPlugin.test.js b/detox/src/artifacts/timeline/TimelineArtifactPlugin.test.js index fa1f822e9c..0454c023d6 100644 --- a/detox/src/artifacts/timeline/TimelineArtifactPlugin.test.js +++ b/detox/src/artifacts/timeline/TimelineArtifactPlugin.test.js @@ -101,7 +101,7 @@ describe('TimelineArtifactPlugin', () => { }) }); - describe('onSuiteStart', () => { + describe('onRunDescribeStart', () => { const deviceId = 'testDeviceId'; const name = 'testSuiteName'; @@ -110,7 +110,7 @@ describe('TimelineArtifactPlugin', () => { const timelineArtifactPlugin = new TimelineArtifactPlugin(config); await timelineArtifactPlugin.onBootDevice({deviceId}); - await timelineArtifactPlugin.onSuiteStart({name}); + await timelineArtifactPlugin.onRunDescribeStart({name}); expect(traceMock.beginEvent).toHaveBeenCalledWith(name, {deviceId}); }); @@ -120,19 +120,19 @@ describe('TimelineArtifactPlugin', () => { const timelineArtifactPlugin = new TimelineArtifactPlugin(config); await timelineArtifactPlugin.onBootDevice({deviceId}); - await timelineArtifactPlugin.onSuiteStart({name}); + await timelineArtifactPlugin.onRunDescribeStart({name}); expect(traceMock.beginEvent).not.toHaveBeenCalled(); }); }); - describe('onSuiteEnd', () => { + describe('onRunDescribeFinish', () => { it('should finish trace event', async () => { const config = configMock(); const name = 'testSuiteName'; const timelineArtifactPlugin = new TimelineArtifactPlugin(config); - await timelineArtifactPlugin.onSuiteEnd({name}); + await timelineArtifactPlugin.onRunDescribeFinish({name}); expect(traceMock.finishEvent).toHaveBeenCalledWith(name); }); diff --git a/detox/src/client/DetoxError.js b/detox/src/client/DetoxError.js deleted file mode 100644 index 6cbb668c9f..0000000000 --- a/detox/src/client/DetoxError.js +++ /dev/null @@ -1,11 +0,0 @@ -class DetoxError extends Error { - constructor(message) { - super(message); - Error.stackTraceLimit = 0; - - Error.captureStackTrace(this, this.constructor); - this.name = this.constructor.name; - } -} - -module.exports = DetoxError; diff --git a/detox/src/index.js b/detox/src/index.js index e17b3c4855..a4846760a1 100644 --- a/detox/src/index.js +++ b/detox/src/index.js @@ -1,3 +1,6 @@ -const DetoxExportWrapper = require('./DetoxExportWrapper'); - -module.exports = new DetoxExportWrapper(); +if (global.detox) { + module.exports = global.detox; +} else { + const DetoxExportWrapper = require('./DetoxExportWrapper'); + module.exports = new DetoxExportWrapper(); +} diff --git a/detox/src/index.test.js b/detox/src/index.test.js index c486fa1011..aff2a30780 100644 --- a/detox/src/index.test.js +++ b/detox/src/index.test.js @@ -9,11 +9,11 @@ const testUtils = { randomObject: () => ({ [Math.random()]: Math.random() }), }; -describe('index', () => { +describe('index (regular)', () => { let logger; let configuration; let Detox; - let index; + let detox; let detoxConfig; let detoxInstance; @@ -35,8 +35,9 @@ describe('index', () => { const MissingDetox = require('./utils/MissingDetox'); Detox.none = new MissingDetox(); - - index = require('./index'); + detox = require('./index'); + detox._setGlobal(global); + detoxInstance = null; }); describe('public interface', () => { @@ -54,14 +55,14 @@ describe('index', () => { ['a', 'function', 'expect'], ['a', 'function', 'waitFor'], ])('should export %s %s called .%s', (_1, type, name) => { - expect(typeof index[name]).toBe(type); + expect(typeof detox[name]).toBe(type); }); }); describe('detox.init(config[, userParams])', () => { - it(`should pass args via calling configuration.composeDetoxConfig(config, userParams)`, async () => { + it(`should pass args via calling configuration.composeDetoxConfig({ override, userParams })`, async () => { const [config, userParams] = [1, 2].map(testUtils.randomObject); - await index.init(config, userParams).catch(() => {}); + await detox.init(config, userParams).catch(() => {}); expect(configuration.composeDetoxConfig).toHaveBeenCalledWith({ override: config, @@ -71,14 +72,14 @@ describe('index', () => { describe('when configuration is valid', () => { beforeEach(async () => { - detox = await index.init(); + detoxInstance = await detox.init(); }); it(`should create a Detox instance with the composed config object`, () => expect(Detox).toHaveBeenCalledWith(detoxConfig)); it(`should return a Detox instance`, () => - expect(detox).toBeInstanceOf(Detox)); + expect(detoxInstance).toBeInstanceOf(Detox)); it(`should set the last error to be null in Detox.none's storage`, () => expect(Detox.none.setError).toHaveBeenCalledWith(null)); @@ -94,7 +95,7 @@ describe('index', () => { throw configError; }); - initPromise = index.init(); + initPromise = detox.init(); await initPromise.catch(() => {}); }); @@ -126,7 +127,7 @@ describe('index', () => { describe('when behaviorConfig.init.exposeGlobals = true', () => { beforeEach(async () => { detoxConfig.behaviorConfig.init.exposeGlobals = true; - detox = await index.init(); + detoxInstance = await detox.init(); }); it(`should touch globals with Detox.none.initContext`, () => { @@ -137,7 +138,7 @@ describe('index', () => { describe('when behaviorConfig.init.exposeGlobals = false', () => { beforeEach(async () => { detoxConfig.behaviorConfig.init.exposeGlobals = false; - detox = await index.init(); + detoxInstance = await detox.init(); }); it(`should not touch globals with Detox.none.initContext`, () => { @@ -148,7 +149,7 @@ describe('index', () => { describe('detox.cleanup()', () => { describe('when called before detox.init()', () => { - beforeEach(() => index.cleanup()); + beforeEach(() => detox.cleanup()); it('should nevertheless cleanup globals with Detox.none.cleanupContext', () => expect(Detox.none.cleanupContext).toHaveBeenCalledWith(global)); @@ -156,21 +157,21 @@ describe('index', () => { describe('when called after detox.init()', () => { beforeEach(async () => { - detox = await index.init(); - await index.cleanup(); + detoxInstance = await detox.init(); + await detox.cleanup(); }); it('should call cleanup in the current Detox instance', () => - expect(detox.cleanup).toHaveBeenCalled()); + expect(detoxInstance.cleanup).toHaveBeenCalled()); it('should call cleanup globals with Detox.none.cleanupContext', () => expect(Detox.none.cleanupContext).toHaveBeenCalledWith(global)); describe('twice', () => { - beforeEach(() => index.cleanup()); + beforeEach(() => detox.cleanup()); it('should not call cleanup twice in the former Detox instance', () => - expect(detox.cleanup).toHaveBeenCalledTimes(1)); + expect(detoxInstance.cleanup).toHaveBeenCalledTimes(1)); }); }); }); @@ -196,7 +197,7 @@ describe('index', () => { }); it(`should forward calls to the Detox.none instance`, async () => { - await index[method](...randomArgs); + await detox[method](...randomArgs); expect(Detox.none[method]).toHaveBeenCalledWith(...randomArgs); }); }); @@ -204,12 +205,12 @@ describe('index', () => { describe('after detox.init() has been called', () => { beforeEach(async () => { detoxConfig = { behaviorConfig: { init: {} } }; - detoxInstance = await index.init(); + detoxInstance = await detox.init(); detoxInstance[method] = jest.fn(); }); it(`should forward calls to the current Detox instance`, async () => { - await index[method](...randomArgs); + await detoxInstance[method](...randomArgs); expect(detoxInstance[method]).toHaveBeenCalledWith(...randomArgs); }); }); @@ -225,22 +226,38 @@ describe('index', () => { }); it(`should return value of Detox.none["${property}"]`, () => { - expect(index[property]).toEqual(Detox.none[property]); + expect(detox[property]).toEqual(Detox.none[property]); }); }); describe('after detox.init() has been called', () => { beforeEach(async () => { detoxConfig = { behaviorConfig: { init: {} } }; - detoxInstance = await index.init(); + detoxInstance = await detox.init(); detoxInstance[property] = testUtils.randomObject(); }); it(`should forward calls to the current Detox instance`, () => { - expect(index[property]).toEqual(detoxInstance[property]); + expect(detox[property]).toEqual(detoxInstance[property]); }); }); }); +}); + +describe('index (global detox variable injected with Jest Circus)', () => { + beforeEach(() => { + if (global.detox) { + throw new Error('detox property should not be in globals during unit tests'); + } + + global.detox = jest.fn(); + }); + afterEach(() => { + delete global.detox; + }); + it('should reexport global.detox', () => { + expect(require('./index')).toBe(global.detox); + }); }); diff --git a/detox/src/utils/__mocks__/logger.js b/detox/src/utils/__mocks__/logger.js index 1ef756ac50..5278d2ddc2 100644 --- a/detox/src/utils/__mocks__/logger.js +++ b/detox/src/utils/__mocks__/logger.js @@ -6,6 +6,7 @@ class FakeLogger { constructor(opts = {}) { this.opts = opts; this.log = jest.fn(); + this.ensureLogFiles = jest.fn(); for (const method of METHODS) { this[method] = jest.fn().mockImplementation((...args) => { diff --git a/detox/src/utils/customConsoleLogger.js b/detox/src/utils/customConsoleLogger.js index 94f8629397..4318716a30 100644 --- a/detox/src/utils/customConsoleLogger.js +++ b/detox/src/utils/customConsoleLogger.js @@ -3,7 +3,6 @@ const path = require('path'); const callsites = require('./callsites'); const USER_STACK_FRAME_INDEX = 2; -const CONSOLE_ASSERT_USER_ARGS_INDEX = 1; function getStackDump() { return callsites.stackdump(USER_STACK_FRAME_INDEX); @@ -32,9 +31,9 @@ function overrideTrace(consoleLevel, bunyanFn) { function overrideAssertion(consoleLevel, bunyanFn) { console[consoleLevel] = (...args) => { - const [condition] = args; + const [condition, ...message] = args; if (!condition) { - bunyanFn({ event: 'USER_LOG' }, getOrigin(), '\n AssertionError:', util.format(...args.slice(CONSOLE_ASSERT_USER_ARGS_INDEX))); + bunyanFn({ event: 'USER_LOG' }, getOrigin(), '\n AssertionError:', util.format(...message)); } }; } diff --git a/detox/src/utils/isPromise.js b/detox/src/utils/isPromise.js new file mode 100644 index 0000000000..711b95e69a --- /dev/null +++ b/detox/src/utils/isPromise.js @@ -0,0 +1,5 @@ +function isPromise(value) { + return Promise.resolve(value) === value; +} + +module.exports = isPromise; \ No newline at end of file diff --git a/detox/src/utils/isPromise.test.js b/detox/src/utils/isPromise.test.js new file mode 100644 index 0000000000..9ee8074674 --- /dev/null +++ b/detox/src/utils/isPromise.test.js @@ -0,0 +1,13 @@ +const isPromise = require('./isPromise'); + +describe('isPromise', () => { + it.each([ + [true, 'a new promise', new Promise(() => {})], + [true, 'a resolved promise', Promise.resolve()], + [false, 'a function', () => {}], + [false, 'a promise-like object', { then: () => {}, catch: () => {}, finally: () => {} }], + [false, 'undefined', undefined], + ])('should return %j for %s', (expected, _comment, arg) => { + expect(isPromise(arg)).toBe(expected); + }); +}); diff --git a/detox/src/utils/logger.js b/detox/src/utils/logger.js index 95cbad4e19..bf12e79fa3 100644 --- a/detox/src/utils/logger.js +++ b/detox/src/utils/logger.js @@ -1,4 +1,5 @@ const fs = require('fs-extra'); +const onExit = require('signal-exit'); const path = require('path'); const bunyan = require('bunyan'); const bunyanDebugStream = require('bunyan-debug-stream'); @@ -88,6 +89,11 @@ function init() { logPath: plainFileStreamPath, })); } + + onExit(() => { + try { fs.unlinkSync(jsonFileStreamPath); } catch (e) {} + try { fs.unlinkSync(plainFileStreamPath); } catch (e) {} + }); } const logger = bunyan.createLogger({ @@ -107,6 +113,16 @@ function init() { overrideConsoleLogger(logger); } + Object.getPrototypeOf(logger).ensureLogFiles = () => { + if (jsonFileStreamPath) { + fs.ensureFileSync(jsonFileStreamPath); + } + + if (plainFileStreamPath) { + fs.ensureFileSync(plainFileStreamPath); + } + }; + return logger; } diff --git a/detox/src/utils/timely.js b/detox/src/utils/timely.js new file mode 100644 index 0000000000..ecc60a6980 --- /dev/null +++ b/detox/src/utils/timely.js @@ -0,0 +1,20 @@ +const isPromise = require('./isPromise'); + +function timely(fn, ms, createRejectReason) { + return function () { + return new Promise((resolve, reject) => { + const maybePromise = fn(...arguments); + if (!isPromise(maybePromise)) { + return resolve(maybePromise); + } + + const promise = maybePromise; + const rejectReason = createRejectReason(); + const handle = setTimeout(reject, ms, rejectReason); + promise.finally(() => clearTimeout(handle)); + promise.then(resolve, reject); + }); + }; +} + +module.exports = timely; diff --git a/detox/src/utils/timely.test.js b/detox/src/utils/timely.test.js new file mode 100644 index 0000000000..c8baef5a09 --- /dev/null +++ b/detox/src/utils/timely.test.js @@ -0,0 +1,23 @@ +const timely = require('./timely'); + +describe('timely', () => { + it('should wrap async functions with a timeout error (happy path)', async () => { + const fortyTwo = async () => 42; + const fortyTwoTimed = timely(fortyTwo, 1000, () => new Error()); + + await expect(fortyTwoTimed()).resolves.toBe(42); + }); + + it('should wrap async functions with a timeout error (unhappy path)', async () => { + const infinitely = () => new Promise(() => {}); + const infinitelyTimed = timely(infinitely, 0, () => new Error('No One Lives Forever')); + + await expect(infinitelyTimed()).rejects.toThrow('No One Lives Forever'); + }); + + it('should wrap sync functions as a fallback too', async () => { + const syncTimely = timely(() => 42, 1000, () => new Error()); + + await expect(syncTimely()).resolves.toBe(42); + }); +}); diff --git a/detox/test/e2e/23.flows.test.js b/detox/test/e2e/23.flows.test.js index 42f5986f73..0228eac498 100644 --- a/detox/test/e2e/23.flows.test.js +++ b/detox/test/e2e/23.flows.test.js @@ -1,11 +1,30 @@ describe('Flows', () => { + describe('- app termination -', () => { + it('should exit without timeouts if app was terminated inside test', async () => { + await device.launchApp({newInstance: true}); + await device.terminateApp(); + }); - it('should exit without timeouts if app was terminated inside test', async () => { - await device.launchApp({newInstance: true}); - await device.terminateApp(); + it('should be able to start the next test with the terminated app', async () => { + await device.launchApp({newInstance: true}); + }); }); - it('should be able to start the next test with the terminated app', async () => { - await device.launchApp({newInstance: true}); + describe('- beforeAll hooks -', () => { + it.skip('trigger false test_start glitch', () => {}); + + describe('inner suite', () => { + beforeAll(async () => { + await device.launchApp({ + newInstance: true, + delete: true, + permissions: {notifications: 'YES', camera: 'YES', photos: 'YES'} + }); + }); + + it('should tap on "Sanity"', async () => { + await element(by.text('Sanity')).tap(); + }); + }); }); }); diff --git a/detox/test/e2e/26.timeout.test.js b/detox/test/e2e/26.timeout.test.js index 8f67e99c0d..dc28cf80fc 100644 --- a/detox/test/e2e/26.timeout.test.js +++ b/detox/test/e2e/26.timeout.test.js @@ -11,7 +11,9 @@ function causeAppNotReady() { } if (process.env.TIMEOUT_E2E_TEST === '1') { - causeAppNotReady(); + if (typeof jasmine !== 'undefined') { + causeAppNotReady(); + } it('timeout test', () => {}); } else { diff --git a/detox/test/e2e/config-circus.js b/detox/test/e2e/config-circus.js deleted file mode 100644 index 3322863165..0000000000 --- a/detox/test/e2e/config-circus.js +++ /dev/null @@ -1,9 +0,0 @@ -const config = require('./config'); - -module.exports = { - ...config, - - "setupFilesAfterEnv": ["./test/e2e/init-circus.js"], - "testEnvironment": "/runners/jest/JestCircusEnvironment", - "testRunner": "./test/node_modules/jest-circus/runner", -}; diff --git a/detox/test/e2e/config-jasmine.js b/detox/test/e2e/config-jasmine.js new file mode 100644 index 0000000000..29ddfd8395 --- /dev/null +++ b/detox/test/e2e/config-jasmine.js @@ -0,0 +1,7 @@ +module.exports = { + ...require('./config'), + + setupFilesAfterEnv: ["./test/e2e/init-jasmine.js"], + testRunner: 'jasmine2', + testEnvironment: 'node', +}; diff --git a/detox/test/e2e/config.js b/detox/test/e2e/config.js index 1eb23ba986..b256c1fc51 100644 --- a/detox/test/e2e/config.js +++ b/detox/test/e2e/config.js @@ -1,7 +1,9 @@ module.exports = { "rootDir": "../..", - "setupFilesAfterEnv": ["./test/e2e/init.js"], - "testEnvironment": "node", + "testEnvironment": "./test/e2e/environment.js", + "testRunner": "./test/node_modules/jest-circus/runner", + "setupFilesAfterEnv": ['./test/e2e/init-coverage.js'], + "testTimeout": 120000, "reporters": process.env.DISABLE_JUNIT_REPORTER === '1' ? ["/runners/jest/streamlineReporter"] : ["/runners/jest/streamlineReporter", "/test/node_modules/jest-junit"], @@ -14,6 +16,7 @@ module.exports = { "!**/*.mock.js", "!**/*.test.js" ], + "coverageProvider": "v8", "coverageDirectory": "test/coverage", "coverageReporters": [["lcov", {"projectRoot": "../.." }], "html"] }; diff --git a/detox/test/e2e/environment.js b/detox/test/e2e/environment.js new file mode 100644 index 0000000000..3163b64946 --- /dev/null +++ b/detox/test/e2e/environment.js @@ -0,0 +1,47 @@ +const WorkerAssignReporterCircus = require('detox/runners/jest/WorkerAssignReporterCircus'); +const SpecReporterCircus = require('detox/runners/jest/SpecReporterCircus'); +const DetoxCircusEnvironment = require('detox/runners/jest-circus/environment'); + +class CustomDetoxEnvironment extends DetoxCircusEnvironment { + constructor(config) { + super(config); + + if (process.env.TIMEOUT_E2E_TEST) { + this.initTimeout = 30000; + } + + this.registerListeners({ + SpecReporterCircus, + WorkerAssignReporterCircus, + }); + } + + async initDetox() { + if (process.env.TIMEOUT_E2E_TEST) { + return this._initDetoxWithHangingServer(); + } else { + return super.initDetox(); + } + } + + async _initDetoxWithHangingServer() { + console.log('Making problems with server'); + const config = require('../package.json').detox; + const instance = await this.detox.init(config, { launchApp: false }); + const sendActionOriginal = instance._server.sendAction; + instance._server.sendAction = function(ws, action) { + if (action.type !== 'ready') { + sendActionOriginal.call(this, ws, action); + } + }; + + await instance.device.launchApp(); + return instance; + } +} + +process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); +}); + +module.exports = CustomDetoxEnvironment; diff --git a/detox/test/e2e/init-circus.js b/detox/test/e2e/init-circus.js deleted file mode 100644 index dd04c83428..0000000000 --- a/detox/test/e2e/init-circus.js +++ /dev/null @@ -1,29 +0,0 @@ -const detox = require('detox'); -const adapter = require('detox/runners/jest/adapter'); -const specReporter = require('detox/runners/jest/specReporter'); -const assignReporter = require('detox/runners/jest/assignReporter'); -const timeoutUtils = require('./utils/timeoutUtils'); - -detoxCircus.getEnv().addEventsListener(adapter); -detoxCircus.getEnv().addEventsListener(assignReporter); -detoxCircus.getEnv().addEventsListener(specReporter); - -// Set the default timeout -jest.setTimeout(timeoutUtils.testTimeout); - -beforeAll(async () => { - await detox.init(); -}, timeoutUtils.initTimeout); - -beforeEach(async () => { - await adapter.beforeEach(); -}); - -afterAll(async () => { - await adapter.afterAll(); - await detox.cleanup(); -}); - -process.on('unhandledRejection', (reason, p) => { - console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); -}); diff --git a/detox/test/e2e/init-coverage.js b/detox/test/e2e/init-coverage.js new file mode 100644 index 0000000000..98c671ac50 --- /dev/null +++ b/detox/test/e2e/init-coverage.js @@ -0,0 +1 @@ +require('detox'); // forces Jest to instrument Detox codebase diff --git a/detox/test/e2e/init.js b/detox/test/e2e/init-jasmine.js similarity index 93% rename from detox/test/e2e/init.js rename to detox/test/e2e/init-jasmine.js index 2703422054..f6c7442b02 100644 --- a/detox/test/e2e/init.js +++ b/detox/test/e2e/init-jasmine.js @@ -1,5 +1,4 @@ const detox = require('detox'); -const config = require('../package.json').detox; const adapter = require('detox/runners/jest/adapter'); const specReporter = require('detox/runners/jest/specReporter'); const assignReporter = require('detox/runners/jest/assignReporter'); @@ -19,7 +18,7 @@ jasmine.getEnv().addReporter(assignReporter); jest.setTimeout(timeoutUtils.testTimeout); beforeAll(async () => { - await detox.init(config); + await detox.init(); }, timeoutUtils.initTimeout); beforeEach(async () => { diff --git a/detox/test/package.json b/detox/test/package.json index f9ffb0f79e..bdc1a2d9f8 100644 --- a/detox/test/package.json +++ b/detox/test/package.json @@ -11,20 +11,18 @@ "start": "react-native start", "packager": "react-native start", "detox-server": "detox run-server", - "e2e:ios": "detox test --configuration ios.sim.release --debug-synchronization", - "e2e:jest-circus:ios": "detox test --configuration ios.sim.release --debug-synchronization -o e2e/config-circus.js", - "e2e:jest-circus-timeout:ios": "node scripts/assert_timeout.js npm run e2e:jest-circus:ios -- --take-screenshots failing e2e/26.timeout.test.js", - "e2e:ios-ci": "detox test --configuration ios.sim.release --workers 3 --debug-synchronization --take-screenshots all --record-logs all --record-timeline all -- --coverage", + "e2e:ios": "detox test -c ios.sim.release --debug-synchronization", + "e2e:android": "detox test -c android.emu.release", + "e2e:android-debug": "detox test -c android.emu.debug", + "e2e:ios-ci": "npm run e2e:ios -- --workers 3 -- --coverage", "e2e:ios-timeout-ci": "node scripts/assert_timeout.js npm run e2e:ios-ci -- e2e/26.timeout.test.js", - "e2e:android": "detox test --configuration android.emu.release", - "e2e:jest-circus:android": "detox test --configuration android.emu.release -o e2e/config-circus.js", - "e2e:jest-circus-timeout:android": "node scripts/assert_timeout.js npm run e2e:jest-circus:android -- --take-screenshots failing e2e/26.timeout.test.js", - "e2e:android-debug": "detox test --configuration android.emu.debug", - "e2e:android-ci": "detox test --configuration android.emu.release --workers 3 --take-screenshots all --record-logs all --record-timeline all --headless --loglevel verbose --jest-report-specs -- --coverage", + "e2e:android-ci": "npm run e2e:android -- --workers 3 --headless --loglevel verbose --jest-report-specs -- --coverage", "e2e:android-timeout-ci": "node scripts/assert_timeout.js npm run e2e:android-ci -- e2e/26.timeout.test.js", - "build:ios": "detox build --configuration ios.sim.release", - "build:android": "detox build --configuration android.emu.release", - "build:android-debug": "detox build --configuration android.emu.debug", + "e2e:legacy-jasmine:ios-timeout-ci": "node scripts/assert_timeout.js npm run e2e:ios -- -o e2e/config-jasmine.js --take-screenshots failing -- --coverage e2e/26.timeout.test.js", + "e2e:legacy-jasmine:android-timeout-ci": "node scripts/assert_timeout.js npm run e2e:android -- -o e2e/config-jasmine.js --take-screenshots failing -- --coverage e2e/26.timeout.test.js", + "build:ios": "detox build -c ios.sim.release", + "build:android": "detox build -c android.emu.release", + "build:android-debug": "detox build -c android.emu.debug", "clean:android": "pushd android && ./gradlew clean && popd", "verify-artifacts:ios": "jest ./scripts/verify_artifacts_are_not_missing.ios.test.js --testEnvironment node", "verify-artifacts:android": "jest ./scripts/verify_artifacts_are_not_missing.android.test.js --testEnvironment node" @@ -38,9 +36,9 @@ "@babel/core": "^7.4.5", "detox": "^16.7.2", "express": "^4.15.3", - "jest": "25.3.x", - "jest-circus": "25.3.x", - "jest-junit": "^8.0.0", + "jest": "26.x.x", + "jest-circus": "26.x.x", + "jest-junit": "^10.0.0", "lodash": "^4.14.1", "nyc": "^14.0.0", "pngjs": "^3.4.0" @@ -64,11 +62,14 @@ }, "artifacts": { "plugins": { + "log": "all", "screenshot": { + "shouldTakeAutomaticSnapshots": true, "takeWhen": { "testDone": true } - } + }, + "timeline": "all" } }, "configurations": { diff --git a/docs/APIRef.Artifacts.md b/docs/APIRef.Artifacts.md index 9c4ffb281a..b49f4e869b 100644 --- a/docs/APIRef.Artifacts.md +++ b/docs/APIRef.Artifacts.md @@ -192,4 +192,9 @@ your app](https://github.com/wix/DetoxInstruments/blob/master/Documentation/Xcod ### Ctrl+C does not terminate Detox+Jest tests correctly -This is a known issue. Video or log recording process under Detox+Jest is apt to keep running even after you press Ctrl+C and stop the tests. Furthermore, some of temporary files won't get erased (e.g. `/sdcard/83541_0.mp4` on Android emulator, or `/private/var/folders/lm/thz8hdxs4v3fppjh0fjc2twhfl_3x2/T/f12a4fcb-0d1f-4d98-866c-e7cea4942ade.png` on your Mac). It cannot be solved on behalf of Detox itself, because the problem has to do with how Jest runner works with its puppet processes. The issue is on our radar, but the ETA for the fix stays unknown. If you feel able to contribute the fix to [Jest](https://github.com/facebook/jest), you are very welcome. +This is a known issue. +Video or log recording process under Detox+Jest is apt to keep running even after you press Ctrl+C and stop the tests. +Furthermore, some of temporary files won't get erased (e.g. `/sdcard/83541_0.mp4` on Android emulator, or `/private/var/folders/lm/thz8hdxs4v3fppjh0fjc2twhfl_3x2/T/f12a4fcb-0d1f-4d98-866c-e7cea4942ade.png` on your Mac). +It cannot be solved on behalf of Detox itself, because the problem has to do with how Jest runner works with its puppet processes. +The issue is on our radar, but the ETA for the fix stays unknown. +If you feel able to contribute the fix to [Jest](https://github.com/facebook/jest), you are very welcome. diff --git a/docs/APIRef.Configuration.md b/docs/APIRef.Configuration.md index badf8f075e..2a91abf226 100644 --- a/docs/APIRef.Configuration.md +++ b/docs/APIRef.Configuration.md @@ -197,30 +197,28 @@ Session can also be set per configuration: ### Test Runner Configuration -##### Optional: setting a test runner (Mocha as default, Jest is supported) +##### Jest (recommended) -##### Mocha ```json "detox": { ... - "test-runner": "mocha", - "runner-config": "path/to/.mocharc.json" + "test-runner": "jest" + "runner-config": "path/to/jest-config" } ``` -`.mocharc.json` refers to `--config` in https://mochajs.org/#-config-path - -##### Jest +`path/to/jest-config` refers to `--config` in https://facebook.github.io/jest/docs/en/configuration.html +##### Mocha ```json "detox": { ... - "test-runner": "jest" - "runner-config": "path/to/config.json" + "test-runner": "mocha", + "runner-config": "path/to/.mocharc.json" } ``` -`config.json` refers to `--config` in https://facebook.github.io/jest/docs/en/configuration.html +`.mocharc.json` refers to `--config` in https://mochajs.org/#-config-path ## detox-cli diff --git a/docs/Guide.Jest.md b/docs/Guide.Jest.md index d6fc38ce52..5991d3b46d 100644 --- a/docs/Guide.Jest.md +++ b/docs/Guide.Jest.md @@ -1,128 +1,145 @@ -# Jest +# Jest setup guide -This guide describes how to install [Jest](http://jestjs.io/) as the test runner to be used by Detox for effectively running the E2E tests (i.e. instead of the default runner, which is Mocha). +> **NOTE: This article previously focused on deprecated `jest-jasmine2` runner setup, and if you nevertheless need to access it, [follow this Git history link](https://github.com/wix/Detox/blob/ef466822129a4befcda71111d02b1a334539889b/docs/Guide.Jest.md).** -## Disclaimer -- The guide describes installing Detox with Jest on a _fresh project_. If you're migrating an existing project, use the guide but please apply some common sense in the process. +This guide describes how to install [Jest](https://jestjs.io) as a test runner to be used by Detox for running the E2E tests. -- The guide has been officially tested only with Jest 24.x.x. We cannot guarantee that everything would work with older versions. +**Disclaimer:** + +1. Here we focus on installing Detox on _new projects_. If you're migrating a project with an existing Detox installation, please apply some common sense while using this guide. +1. These instructions are relevant for `jest-circus@^26.0.1`. They should likely work for the newer `jest-circus` versions too, but for **the older ones** (25.x, 24.x) — **they will not, due to blocking issues.** ## Introduction -As already mentioned in the [Getting Started](Introduction.GettingStarted.md#step-3-create-your-first-test) guide, Detox itself does not effectively run tests logic, but rather delegates that responsibility onto a test runner. Jest is the recommended runner for projects with test suites that have become large enough so as to require parallel execution. +As already mentioned in the [Getting Started](Introduction.GettingStarted.md#step-3-create-your-first-test) guide, Detox itself does not effectively run tests logic, but rather delegates that responsibility onto a test runner. At the moment, Jest is the only recommended choice, for many reasons, including but not limited to parallel test suite execution capability, and complete integration with Detox API. -Do note that in turn, Jest itself - much like Detox, also does not effectively run any tests; Rather, it is more of a dispatcher and orchestrator of multiple instances of a delegated runner, capable of running in parallel (for more info, refer to [this video](https://youtu.be/3YDiloj8_d0?t=2127); source: [Jest architecture](https://jestjs.io/docs/en/architecture)). Currently, by default, the concrete runner is `jasmine v2`, but Jest's own project called [jest-circus](https://github.com/facebook/jest/tree/master/packages/jest-circus) is becoming more and more stable. In a way, we even recommend using it over `jasmine` because of bugs in `jasmine` that are not maintained ([this one](https://github.com/facebook/jest/issues/6755) in particular). +By the way, Jest itself — much like Detox, also does not effectively run any tests. Instead, it is more of a dispatcher and orchestrator of multiple instances of a delegated runner capable of running in parallel. For more info, refer to [this video](https://youtu.be/3YDiloj8_d0?t=2127) (source: [Jest architecture](https://jestjs.io/docs/en/architecture)). -**The guide initially describes the setup of Jest in its default settings (i.e. using `jasmine`).** For applying `jest-circus`, refer to the end of the installation section. +For its part, Detox supports only one Jest's concrete runner, which is [`jest-circus`](https://www.npmjs.com/package/jest-circus). The former runner, `jest-jasmine2`, is deprecated due to specific bugs in the past, and architectural limitations at present. Moreover, Jest team plans to deprecate `jest-jasmine2` in the upcoming major release 27.0.0 ([see blog post](https://jestjs.io/blog/2020/05/05/jest-26)). ## Installation -### 0. Set up Detox' fundamentals +### 1. Install Jest -Before starting out with Jest, please be sure to go over the fundamentals of the [Getting Started](Introduction.GettingStarted.md) guide. +Before starting with Jest setup, please go over [Getting Started](Introduction.GettingStarted.md) guide, +especially **steps 1 and 2**. -### 1. Install Jest +Afterward, install the respective npm packages: ```sh -npm install --save-dev jest +npm install --save-dev jest jest-circus ``` +If you are already using Jest in your project, +make sure that `jest` and `jest-circus` package versions match (e.g., both are `26.0.1`). + ### 2. Set up test-code scaffolds -#### Run an automated init script: +Run the automated init script: ```sh detox init -r jest ``` +> **Note:** errors occurring in the process may appear in red. -> Errors occurring in the process may appear in red. - -#### Fix & Verify - -Even if `detox init` goes well and everything is green, we recommend going over some fundamental things, using our homebrewed [`demo-react-native-jest`](https://github.com/wix/Detox/tree/master/examples/demo-react-native-jest) example project as a reference. - -##### a. Fix/verify this list of JSON properties: +### 3. Fix / Verify +Even if `detox init` passes well, and everything is green, we still recommend going over the checklist below. You can also use our example project, [`demo-react-native-jest`](https://github.com/wix/Detox/tree/master/examples/demo-react-native-jest), as a reference in case of ambiguities. -| File | Property | Value | Description | -| ------------------------------------------------------------ | ---------------------- | ---------------------------------------------- | ------------------------------------------------------------ | -| [**`package.json`**](https://github.com/wix/Detox/blob/master/examples/demo-react-native-jest/package.json#L25) | `detox.test-runner` | `"jest"` | *Required.* Should be `"jest"` for the proper `detox test` CLI functioning. | -| | `detox.runner-config ` | (optional path to Jest config file) | *Optional.* This field tells `detox test` CLI where to look for Jest's config file. If omitted, the path defaults to "e2e/config.json" (a file generated by `detox init -r jest`). | -| [**`e2e/config.json`**](https://github.com/wix/Detox/blob/master/examples/demo-react-native-jest/e2e/config.json) | `testEnvironment ` | `"node"` | *Required*. Needed for the proper functioning of Jest and Detox. | -| | `setupFilesAfterEnv ` | `["./init.js"]` | *Required*. Indicates which files to run before each test suite. The field was [introduced in Jest 24](https://jestjs.io/docs/en/configuration#setupfilesafterenv-array). | -| | `reporters` | ["detox/runners/
jest/streamlineReporter"] | *Optional.* Available since Detox `12.7.0`. Sets up our highly recommended `streamline-reporter` [Jest reporter](https://jestjs.io/docs/en/configuration#reporters-array-modulename-modulename-options), tailored for running end-to-end tests in Jest - which in itself was mostly intended for running unit tests. For more details, [see the migration guide](Guide.Migration.md#migrating-to-1270-from-older-nonbreaking). | -| | `verbose` | `true` | Must be `true` if you have replaced Jest's `default` reporter with Detox's `streamlineReporter`. Optional otherwise. | +#### .detoxrc.json -A typical detox configuration in a `package.json`file: +| Property | Value | Description | +| ---------------------- | ---------------------------------------------- | ------------------------------------------------------------ | +| `test-runner` | `"jest"` | *Required.* Should be `"jest"` for the proper `detox test` CLI functioning. | +| `runner-config ` | (optional path to Jest config file) | *Optional.* This field tells `detox test` CLI where to look for Jest's config file. If omitted, the default value is `e2e/config.json`. | -![package.json](img/jest-guide/package_json.png) +A typical Detox configuration in `.detoxrc.json` file looks like: -##### b. Fix/verify the custom Jest init script (i.e. [`e2e/init.js`](https://github.com/wix/Detox/blob/master/examples/demo-react-native-jest/e2e/init.js)): +```json +{ + "test-runner": "jest", + "runner-config": "e2e/config.json", + "configurations": { + "ios.sim.release": { + "type": "ios.simulator", + "binaryPath": "ios/build/Build/Products/Release-iphonesimulator/example.app", + "build": "...", + "device": { + "type": "iPhone 11 Pro" + } + } + } +} +``` -- `beforeAll()`, `beforeEach()`, `afterAll()` should be registered as hooks - for invoking `detox` and/or a custom adapter. -- The custom Detox-Jest adapter must be registered as a `jasmine` reporter (`jasmine.getEnv().addReporter()`). It is required for the [artifacts subsystem](APIRef.Artifacts.md) to properly work. -- (Recommended) Starting Detox `12.7.0`, an additional, custom `spec-reporter` should be registered as a `jasmine` reporter, as well. This one takes care of logging on a per-spec basis (i.e. when `it`'s start and end) — which Jest does not do by default. - Should be used in conjunction with the Detox-Jest adapter. +#### e2e/config.json -A typical Jest log output, having set up `streamline-reporter` in `config.json` and `spec-reporter` in `init.js`: +| Property | Value | Description | +| ---------------------- | ---------------------------------------------- | ------------------------------------------------------------ | +| `testEnvironment ` | `"./environment"` | *Required.* Needed for the proper functioning of Jest and Detox. See [Jest documentation](https://jestjs.io/docs/en/configuration#testenvironment-string) for more details. | +| `testRunner ` | `"jest-circus/runner"` | *Required.* Needed for the proper functioning of Jest and Detox. See [Jest documentation](https://jestjs.io/docs/en/configuration#testrunner-string) for more details. | +| `testTimeout ` | `120000` | *Required*. Overrides the default timeout (5 seconds), which is usually too short to complete a single end-to-end test. | +| `reporters` | `["detox/runners/jest/streamlineReporter"]` | *Recommended.* Sets up our streamline replacement for [Jest's default reporter](https://jestjs.io/docs/en/configuration#reporters-array-modulename-modulename-options), which removes Jest's default buffering of `console.log()` output. That is helpful for end-to-end tests since log messages appear on the screen without any artificial delays. For more context, [read Detox 12.7.0 migration guide](Guide.Migration.md#migrating-to-1270-from-older-nonbreaking). | +| `verbose` | `true` | *Conditional.* Must be `true` if above you have replaced Jest's default reporter with Detox's `streamlineReporter`. Optional otherwise. | -![Streamlined output](img/jest-guide/streamlined_logging.png) +A typical `jest-circus` configuration in `e2e/config.json` file would look like: -### 3. Applying `jest-circus` (optional) - -> * **Experimental;** Frequent breaking changes are expected in upcoming versions! Known issues: -> * Video recording causes iOS simulator to freeze -> * Requires Detox >= 14.3.0 !!! -> * Tested on Jest & jest-circus version >= 24.8.0 !!! +```json +{ + "testRunner": "jest-circus/runner", + "testEnvironment": "./environment", + "testTimeout": 120000, + "reporters": ["detox/runners/jest/streamlineReporter"], + "verbose": true +} +``` -By reaching this point you effectively have Jest set up and ready to launch in its default settings - i.e. with `jasmine` as its default test runner. However, `jasmine` can be switched with `jest-circus` as a drop-in replacement. To do so, apply these changes: +#### e2e/environment.js -##### a. Install jest-circus +If you are not familiar with Environment concept in Jest, you could check [their documentation](https://jestjs.io/docs/en/configuration#testenvironment-string). -```sh -npm install --save-dev jest-circus -``` +For Detox, having a `CustomDetoxEnvironment` class derived from `NodeEnvironment` enables implementing cross-cutting concerns such as taking screenshots the exact moment a test function (it/test) or a hook (e.g., beforeEach) fails, skip adding tests if they have `:ios:` or `:android:` within their title, starting device log recordings before test starts and so on. -> Make sure jest and jest-circus' versions match (e.g. both are 24.9.0) +API of `CustomDetoxEnvironment` is not entirely public in a sense that there's no guide on how to write custom `DetoxCircusListeners` and override `initDetox()` and `cleanupDetox()` protected methods, since this is not likely to be needed for typical projects, but this is under consideration if there appears specific demand. -##### b. Update jest's config +See [an example](https://github.com/wix/Detox/blob/master/examples/demo-react-native-jest/e2e/init.js) of a custom Detox environment for Jest. -In `e2e/config.json`, apply this change: +```js +const { + DetoxCircusEnvironment, + SpecReporter, + WorkerAssignReporter, +} = require('detox/runners/jest-circus'); -```diff -// e2e/config.json +class CustomDetoxEnvironment extends DetoxCircusEnvironment { + constructor(config) { + super(config); -{ -- "testEnvironment": "node" -+ "testEnvironment": "detox/runners/jest/JestCircusEnvironment", -+ "testRunner": "jest-circus/runner" + // Can be safely removed, if you are content with the default value (=300000ms) + this.initTimeout = 300000; -... + // This takes care of generating status logs on a per-spec basis. By default, Jest only reports at file-level. + // This is strictly optional. + this.registerListeners({ + SpecReporter, + WorkerAssignReporter, + }); + } } -``` - -##### c. Update init script -In `e2e/init.js`, as explained in the previous section, we typically register the main adapter and various reporters directly do `jasmine`. Since `jest-circus` **replaces** `jasmine`, we've set up a custom circus-associated API to replace `jasmine.getEnv().addReporter()` calls, namely: `detoxCircus.getEnv().addEventsListener()`. You must have all associated calls replaced. Example: +module.exports = CustomDetoxEnvironment; +``` -```diff -// e2e/init.js +**Notes:** -const adapter = require('detox/runners/jest/adapter'); -const specReporter = require('detox/runners/jest/specReporter'); -const assignReporter = require('detox/runners/jest/assignReporter'); +- The custom `SpecReporter` is recommended to be registered as a listener. It takes care of logging on a per-spec basis (i.e. when `it('...')` functions start and end) — which Jest does not do by default. +- The custom `WorkerAssignReporter` prints for every next test suite which device is assigned to its execution. --jasmine.getEnv().addReporter(adapter); --jasmine.getEnv().addReporter(specReporter); --jasmine.getEnv().addReporter(assignReporter); -+detoxCircus.getEnv().addEventsListener(adapter); -+detoxCircus.getEnv().addEventsListener(specReporter); -+detoxCircus.getEnv().addEventsListener(assignReporter); -``` +This is how a typical Jest log output looks when `SpecReporter` and `WorkerAssignReporter` are enabled in `streamline-reporter` is set up in `config.json` and +`SpecReporter` added in `e2e/environment.js`: -Once again, use our [jest demo-suite's init.js](https://github.com/wix/Detox/blob/jest-circus/examples/demo-react-native-jest/e2e/init-circus.js#L10) as a reference. +![Streamlined output](img/jest-guide/streamlined_logging.png) ## Writing Tests @@ -133,11 +150,25 @@ There are some things you should notice: ## Parallel Test Execution -Through Detox' cli, Jest can be started with [multiple workers](Guide.ParallelTestExecution.md) that run tests simultaneously. In this mode, Jest effectively assigns one worker per each test file (invoking Jasmine over it). In this mode, the per-spec logging offered by the `spec-reporter` mentioned earlier, does not necessarily make sense, as the workers' outputs get mixed up. +Through Detox' CLI, Jest can be started with [multiple workers](Guide.ParallelTestExecution.md) that run tests simultaneously, e.g.: + +```bash +detox test --configuration --workers 2 +``` -By default, we disable `spec-reporter` in a multi-workers environment. If you wish to force-enable it nonetheless, the [`--jest-report-specs`](APIRef.DetoxCLI.md#test) CLI option can be used. +In this mode, Jest effectively assigns one worker per each test file. +Per-spec logging offered by the `SpecReporter` mentioned earlier, does not necessarily make sense, as the workers' outputs get mixed up. + +By default, we disable `SpecReporter` in a multi-workers environment. +If you wish to force-enable it nonetheless, the [`--jest-report-specs`](APIRef.DetoxCLI.md#test) CLI option can be used with `detox test`, e.g.: + +```bash +detox test --configuration --workers 2 --jest-report-specs +``` -## How to run unit test and E2E tests in the same project +## How to run unit and E2E tests in the same project -- If you have a setup file for the unit tests pass `./jest/setup` implementation into your unit setup. -- Call your E2E tests using `detox-cli`: `detox test` \ No newline at end of file +- Create different Jest configs for unit and E2E tests, e.g. in `e2e/config.json` (for Detox) and `jest.config.js` +(for unit tests). For example, in Jest's E2E config you can set `testRegex` to look for `\.e2e.js$` regexp, + and this way avoid accidental triggering of unit tests with `.test.js` extension. +- To run your E2E tests, use `detox test` command (or `npx detox test`, if you haven't installed `detox-cli`). diff --git a/docs/Guide.ParallelTestExecution.md b/docs/Guide.ParallelTestExecution.md index 4f7a346010..f3c796c11b 100644 --- a/docs/Guide.ParallelTestExecution.md +++ b/docs/Guide.ParallelTestExecution.md @@ -1,13 +1,16 @@ # Parallel Test Execution + Detox can leverage multi worker support of JS test runners ([Jest](http://jestjs.io/docs/en/cli#maxworkers-num), [AVA](https://github.com/avajs/ava#process-isolation), etc.). By default `detox test` will run the test runner with one worker (it will pass `--maxWorkers=1` to Jest cli, Mocha is unaffected). Worker count can be controlled by adding `--workers n` to `detox test`, read more in [detox-cli section](APIRef.DetoxCLI.md#test). ## Device Creation + While running with multiple workers, Detox might not have an available simulator for every worker. If no simulator is available for that worker, the worker will create one with the name `{name}-Detox`. ## Lock File + Simulators/emulators run on a different process, outside of node, and require some sort of lock mechanism to make sure only one process controlls a simulator in a given time. Therefore, Detox 7.4.0 introduced `device.registry.state.lock`, a lock file controlled by Detox, that registers all in-use simulators. > **Note:** Each worker is responsible of removing the deviceId from the list in `device.registry.state.lock`. Exiting a test runner abruptly (using ctrl+c / ⌘+c) will not give the worker a chance to deregister the device from the lock file, resulting in an inconsistent state, which can result in creation of unnecessary new simulators. @@ -22,14 +25,9 @@ Simulators/emulators run on a different process, outside of node, and require so The lock file location is determined by the OS, and [defined here](https://github.com/wix/detox/blob/master/detox/src/utils/appdatapath.js). -#### MacOS -`~/Library/Detox/device.registry.state.lock` -#### Linux -`~/.local/share/Detox/device.registry.state.lock` -#### Windows -`$LOCALAPPDATA/data/Detox/device.registry.state.lock` -or -`$USERPROFILE/Application Data/Detox/device.registry.state.lock` +* **MacOS**: `~/Library/Detox/device.registry.state.lock` +* **Linux**: `~/.local/share/Detox/device.registry.state.lock` +* **Windows**: `%LOCALAPPDATA%/data/Detox/device.registry.state.lock` or `%USERPROFILE%/Application Data/Detox/device.registry.state.lock` ### Persisting the Lock File diff --git a/docs/Introduction.GettingStarted.md b/docs/Introduction.GettingStarted.md index 8b727485be..1e0d69889c 100644 --- a/docs/Introduction.GettingStarted.md +++ b/docs/Introduction.GettingStarted.md @@ -106,7 +106,7 @@ config file names above. To get some help with creating your first Detox config file, you can try running Detox CLI: `detox init -r jest` (or `-r mocha`). -Either way the config should look like this: +Either way the config should look like this: ##### in package.json @@ -154,50 +154,41 @@ Also make sure the simulator model specified under the key `device.type` (e.g. ` Detox CLI supports Jest and Mocha out of the box. You need to choose one now, but it *is* possible to replace it later on. -> Do note that: -> -> * Jest is more complex to set up, but it's the only one that supports parallel tests execution. In Detox `12.7.0`, we've made Jest more suitable for e2e testing in terms of logging and usability. -> * Mocha is easy to set up and is lightweight. +* **Jest is the recommended choice**, since it provides parallel test execution and complete lifecycle integration for Detox. +* Mocha, albeit its integration is less complete, is still lightweight, and a bit easier to set up. -[Jest](http://jestjs.io/): +**Note:** Detox is coupled neither with Mocha or Jest nor with a specific directory structure. Both runners are just a recommendation — with some effort, they can be replaced without touching the internal implementation of Detox itself. -```sh -npm install jest --save-dev -``` +##### [Jest](https://jestjs.io/) + +Follow the [Guide.Jest.md](Guide.Jest.md) documentation. -[Mocha](https://mochajs.org/): +##### [Mocha](https://mochajs.org/) ```sh npm install mocha --save-dev ``` -> Tip: Detox is not tightly coupled to Mocha and Jest, neither to this specific directory structure. Both are just a recommendation and are easy to replace without touching the internal implementation of Detox itself. - #### 2. Set up test-code scaffolds (automated) :building_construction: The Detox CLI has a `detox init` convenience method to automate a setup for your first test. Depending on your test runner of choice, run one of these commands: -Jest: +Note: `detox init` runs these steps, which you can reproduce manually: -```sh -detox init -r jest -``` +- Creates an `e2e/` folder in your project root +- Inside `e2e` folder, creates `mocha.opts` (for `mocha`) or `config.json` (for `jest`). See examples: [mocha.opts](/examples/demo-react-native/e2e/mocha.opts), [config.json](/examples/demo-react-native-jest/e2e/config.json) +- Inside `e2e` folder, creates `init.js` file. See examples for [Mocha](/examples/demo-react-native/e2e/init.js) and [Jest](/examples/demo-react-native-jest/e2e/init.js). +- Inside `e2e` folder, creates `firstTest.e2e.js` with content similar to [this](/examples/demo-react-native-jest/e2e/app-hello.e2e.js). -Mocha: +##### Mocha ```sh detox init -r mocha ``` -**For a Jest-based environment, please pause and run through the comprehensive [Jest setup guide](Guide.Jest.md).** +##### Jest -> Note: `detox init` runs these steps, which you can reproduce manually: -> -> - Creates `.detoxrc.json` file in your project root -> - Creates an `e2e/` folder in your project root -> - Inside `e2e` folder, creates `.mocharc.json` (for `mocha`) or `config.json` (for `jest`). See examples: [mocha.opts](/examples/demo-react-native/e2e/.mocharc), [config.json](/examples/demo-react-native-jest/e2e/config.json) -> - Inside `e2e` folder, creates `init.js` file. See examples for [Mocha](/examples/demo-react-native/e2e/init.js) and [Jest](/examples/demo-react-native-jest/e2e/init.js). -> - Inside `e2e` folder, creates `firstTest.spec.js` with content similar to [this](/examples/demo-react-native/e2e/example.spec.js). +Follow the [Guide.Jest.md](Guide.Jest.md) documentation. ## Step 4: Build your app and run Detox tests diff --git a/docs/img/jest-guide/streamlined_logging.png b/docs/img/jest-guide/streamlined_logging.png index bf26207dce..cac3a2c05d 100644 Binary files a/docs/img/jest-guide/streamlined_logging.png and b/docs/img/jest-guide/streamlined_logging.png differ diff --git a/examples/demo-plugin/package.json b/examples/demo-plugin/package.json index db9f049204..697265420e 100644 --- a/examples/demo-plugin/package.json +++ b/examples/demo-plugin/package.json @@ -7,7 +7,7 @@ }, "devDependencies": { "detox": "^16.7.2", - "jest": "24.8.x" + "jest": "26.x.x" }, "detox": { "test-runner": "jest", diff --git a/examples/demo-react-native-jest/.detoxrc.json b/examples/demo-react-native-jest/.detoxrc.json new file mode 100644 index 0000000000..60f9eaaa76 --- /dev/null +++ b/examples/demo-react-native-jest/.detoxrc.json @@ -0,0 +1,26 @@ +{ + "testRunner": "jest", + "runnerConfig": "e2e/config.json", + "configurations": { + "ios.sim.release": { + "type": "ios.simulator", + "binaryPath": "../demo-react-native/ios/build/Build/Products/Release-iphonesimulator/example.app", + "device": { + "type": "iPhone 11 Pro" + }, + "artifacts": { + "pathBuilder": "./e2e/detox.pathbuilder.ios.js" + } + }, + "android.emu.release": { + "type": "android.emulator", + "binaryPath": "../demo-react-native/android/app/build/outputs/apk/release/app-release.apk", + "device": { + "avdName": "Pixel_API_28" + }, + "artifacts": { + "pathBuilder": "./e2e/detox.pathbuilder.android.js" + } + } + } +} diff --git a/examples/demo-react-native-jest/e2e/app-goodbye.test.js b/examples/demo-react-native-jest/e2e/app-goodbye.e2e.js similarity index 100% rename from examples/demo-react-native-jest/e2e/app-goodbye.test.js rename to examples/demo-react-native-jest/e2e/app-goodbye.e2e.js diff --git a/examples/demo-react-native-jest/e2e/app-hello.test.js b/examples/demo-react-native-jest/e2e/app-hello.e2e.js similarity index 100% rename from examples/demo-react-native-jest/e2e/app-hello.test.js rename to examples/demo-react-native-jest/e2e/app-hello.e2e.js diff --git a/examples/demo-react-native-jest/e2e/config-circus.json b/examples/demo-react-native-jest/e2e/config-circus.json deleted file mode 100644 index a433ea2e83..0000000000 --- a/examples/demo-react-native-jest/e2e/config-circus.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "setupFilesAfterEnv": ["./init-circus.js"], - "testEnvironment": "detox/runners/jest/JestCircusEnvironment", - "testRunner": "jest-circus/runner", - "reporters": ["detox/runners/jest/streamlineReporter"], - "verbose": true -} diff --git a/examples/demo-react-native-jest/e2e/config.json b/examples/demo-react-native-jest/e2e/config.json index 523800e2d2..5467a93030 100644 --- a/examples/demo-react-native-jest/e2e/config.json +++ b/examples/demo-react-native-jest/e2e/config.json @@ -1,6 +1,8 @@ { - "setupFilesAfterEnv": ["./init.js"], - "testEnvironment": "node", - "reporters": ["detox/runners/jest/streamlineReporter"], - "verbose": true + "testEnvironment": "./environment", + "testRunner": "jest-circus/runner", + "testTimeout": 120000, + "testRegex": "\\.e2e\\.js$", + "reporters": ["detox/runners/jest/streamlineReporter"], + "verbose": true } diff --git a/examples/demo-react-native-jest/e2e/environment.js b/examples/demo-react-native-jest/e2e/environment.js new file mode 100644 index 0000000000..ad46bec5f3 --- /dev/null +++ b/examples/demo-react-native-jest/e2e/environment.js @@ -0,0 +1,23 @@ +const { + DetoxCircusEnvironment, + SpecReporter, + WorkerAssignReporter, +} = require('detox/runners/jest-circus'); + +class CustomDetoxEnvironment extends DetoxCircusEnvironment { + constructor(config) { + super(config); + + // Can be safely removed, if you are content with the default value (=300000ms) + this.initTimeout = 300000; + + // This takes care of generating status logs on a per-spec basis. By default, Jest only reports at file-level. + // This is strictly optional. + this.registerListeners({ + SpecReporter, + WorkerAssignReporter, + }); + } +} + +module.exports = CustomDetoxEnvironment; diff --git a/examples/demo-react-native-jest/e2e/init-circus.js b/examples/demo-react-native-jest/e2e/init-circus.js deleted file mode 100644 index e312f2c17e..0000000000 --- a/examples/demo-react-native-jest/e2e/init-circus.js +++ /dev/null @@ -1,24 +0,0 @@ -const detox = require('detox'); -const adapter = require('detox/runners/jest/adapter'); -const specReporter = require('detox/runners/jest/specReporter'); -const assignReporter = require('detox/runners/jest/assignReporter'); - -detoxCircus.getEnv().addEventsListener(adapter); -detoxCircus.getEnv().addEventsListener(assignReporter); -detoxCircus.getEnv().addEventsListener(specReporter); - -// Set the default timeout -jest.setTimeout(90000); - -beforeAll(async () => { - await detox.init(); -}, 300000); - -beforeEach(async () => { - await adapter.beforeEach(); -}); - -afterAll(async () => { - await adapter.afterAll(); - await detox.cleanup(); -}); diff --git a/examples/demo-react-native-jest/e2e/init.js b/examples/demo-react-native-jest/e2e/init.js deleted file mode 100644 index 8007485c62..0000000000 --- a/examples/demo-react-native-jest/e2e/init.js +++ /dev/null @@ -1,30 +0,0 @@ -const detox = require('detox'); -const adapter = require('detox/runners/jest/adapter'); -const specReporter = require('detox/runners/jest/specReporter'); -const assignReporter = require('detox/runners/jest/assignReporter'); - -jasmine.getEnv().addReporter(adapter); - -// This takes care of generating status logs on a per-spec basis. By default, jest only reports at file-level. -// This is strictly optional. -jasmine.getEnv().addReporter(specReporter); - -// This will post which device has assigned to run a suite, which can be useful in a multiple-worker tests run. -// This is strictly optional. -jasmine.getEnv().addReporter(assignReporter); - -// Set the default timeout -jest.setTimeout(90000); - -beforeAll(async () => { - await detox.init(); -}, 300000); - -beforeEach(async () => { - await adapter.beforeEach(); -}); - -afterAll(async () => { - await adapter.afterAll(); - await detox.cleanup(); -}); diff --git a/examples/demo-react-native-jest/package.json b/examples/demo-react-native-jest/package.json index a272434a0b..b5f4e2778b 100644 --- a/examples/demo-react-native-jest/package.json +++ b/examples/demo-react-native-jest/package.json @@ -5,42 +5,13 @@ "scripts": { "test:ios-release": "detox test --configuration ios.sim.release -l verbose", "test:ios-release-ci": "detox test --configuration ios.sim.release -l verbose --workers 2", - "test:jest-circus:ios-release": "detox test --configuration ios.sim.release -l verbose -o e2e/config-circus.json", - "test:jest-circus:ios-release-ci": "detox test --configuration ios.sim.release -l verbose -o e2e/config-circus.json --workers 2", "test:android-release": "detox test --configuration android.emu.release", - "test:android-release-ci": "detox test --configuration android.emu.release -l verbose --workers 2 --headless --record-logs all --take-screenshots all", - "test:jest-circus:android-release": "detox test --configuration android.emu.release -o e2e/config-circus.json", - "test:jest-circus:android-release-ci": "detox test --configuration android.emu.release -o e2e/config-circus.json -l verbose --workers 2 --headless --record-logs all --take-screenshots all" + "test:android-release-ci": "detox test --configuration android.emu.release -l verbose --workers 2 --headless --record-logs all --take-screenshots all" }, "devDependencies": { "detox": "^16.7.2", - "jest": "25.1.x", - "jest-circus": "25.1.x", + "jest": "26.x.x", + "jest-circus": "26.x.x", "sanitize-filename": "^1.6.1" - }, - "detox": { - "test-runner": "jest", - "configurations": { - "ios.sim.release": { - "artifacts": { - "pathBuilder": "./e2e/detox.pathbuilder.ios.js" - }, - "binaryPath": "../demo-react-native/ios/build/Build/Products/Release-iphonesimulator/example.app", - "type": "ios.simulator", - "device": { - "type": "iPhone 11 Pro" - } - }, - "android.emu.release": { - "artifacts": { - "pathBuilder": "./e2e/detox.pathbuilder.android.js" - }, - "binaryPath": "../demo-react-native/android/app/build/outputs/apk/release/app-release.apk", - "type": "android.emulator", - "device": { - "avdName": "Pixel_API_28" - } - } - } } } diff --git a/generation/package.json b/generation/package.json index 2b8440dbcd..56926e7802 100644 --- a/generation/package.json +++ b/generation/package.json @@ -30,7 +30,7 @@ "uuid": "^3.2.1" }, "devDependencies": { - "jest": "^25.1.0", + "jest": "^26.0.0", "lint-staged": "^6.0.0", "prettier": "^1.8.2" }, diff --git a/scripts/ci.android.sh b/scripts/ci.android.sh index 857926dfe1..23cc74ab2f 100755 --- a/scripts/ci.android.sh +++ b/scripts/ci.android.sh @@ -26,8 +26,8 @@ cp coverage/lcov.info ../../coverage/e2e-android-ci.lcov run_f "npm run e2e:android-timeout-ci" cp coverage/lcov.info ../../coverage/e2e-android-timeout-ci.lcov -run_f "npm run e2e:jest-circus-timeout:android" -cp coverage/lcov.info ../../coverage/e2e-jest-circus-timeout-android.lcov +run_f "npm run e2e:legacy-jasmine:android-timeout-ci" +cp coverage/lcov.info ../../coverage/e2e-legacy-jasmine-android-timeout-ci.lcov # run_f "npm run verify-artifacts:android" popd diff --git a/scripts/ci.ios.sh b/scripts/ci.ios.sh index 24f68aebc3..824822dfed 100755 --- a/scripts/ci.ios.sh +++ b/scripts/ci.ios.sh @@ -17,8 +17,8 @@ cp coverage/lcov.info ../../coverage/e2e-ios-ci.lcov run_f "npm run e2e:ios-timeout-ci" cp coverage/lcov.info ../../coverage/e2e-ios-timeout-ci.lcov -run_f "npm run e2e:jest-circus-timeout:ios" -cp coverage/lcov.info ../../coverage/e2e-jest-circus-timeout-ios.lcov +run_f "npm run e2e:legacy-jasmine:ios-timeout-ci" +cp coverage/lcov.info ../../coverage/e2e-legacy-jasmine-ios-timeout-ci.lcov # run_f "npm run verify-artifacts:ios" popd diff --git a/scripts/demo-projects.android.sh b/scripts/demo-projects.android.sh index 6738ab5a05..3807d1a1c1 100755 --- a/scripts/demo-projects.android.sh +++ b/scripts/demo-projects.android.sh @@ -17,7 +17,6 @@ popd # as it runs tests in parallel. pushd examples/demo-react-native-jest run_f "npm run test:android-release-ci" -run_f "npm run test:jest-circus:android-release-ci" popd pushd examples/demo-react-native diff --git a/scripts/demo-projects.ios.sh b/scripts/demo-projects.ios.sh index aa77d5784d..28b08ce080 100755 --- a/scripts/demo-projects.ios.sh +++ b/scripts/demo-projects.ios.sh @@ -16,7 +16,6 @@ popd pushd examples/demo-react-native-jest run_f "npm run test:ios-release-ci" -run_f "npm run test:jest-circus:ios-release-ci" popd pushd examples/demo-react-native-detox-instruments