Timeouts (#3113)

* updated changelog

* implemented timeouts

* fixed tests

* removed retry to test

Author: DavertMik

CHANGELOG.md

@@ -1,3 +1,5 @@
+## 3.2.0
+
 ## 3.1.3
 
 🛩️ Features:

docs/advanced.md

@@ -100,7 +100,7 @@ Scenario('update user profile', ({  }) => {
 All tests with `@tag` could be executed with `--grep @tag` option.
 
 ```sh
-codeceptjs run --grep @slow
+npx codeceptjs run --grep @slow
 ```
 
 Use regex for more flexible filtering:
@@ -119,24 +119,30 @@ CodeceptJS provides a debug mode in which additional information is printed.
 It can be turned on with `--debug` flag.
 
 ```sh
-codeceptjs run --debug
+npx codeceptjs run --debug
 ```
 
 to receive even more information turn on `--verbose` flag:
 
 ```sh
-codeceptjs run --verbose
+npx codeceptjs run --verbose
 ```
 
-And don't forget that you can pause execution and enter **interactive console** mode by calling `pause()` inside your test.
+> You can pause execution and enter **interactive console** mode by calling `pause()` inside your test.
 
-For advanced debugging use NodeJS debugger. In WebStorm IDE:
+To see a complete internal debug of CodeceptJS use `DEBUG` env variable:
+
+```sh
+DEBUG=codeceptjs:* npx codeceptjs run
+```
+
+For an interactive debugging use NodeJS debugger. In **WebStorm**:
 
 ```sh
 node $NODE_DEBUG_OPTION ./node_modules/.bin/codeceptjs run
 ```
 
-For Visual Studio Code, add the following configuration in launch.json:
+For **Visual Studio Code**, add the following configuration in launch.json:
 
 ```json
 {
@@ -180,27 +186,54 @@ You can use this options for build your own [plugins](https://codecept.io/hooks/
   });
 ```
 
-### Timeout
+### Timeout <Badge text="Updated in 3.2" type="warning"/>
+
+Tests can get stuck due to various reasons such as network connection issues, crashed browser, etc.
+This can make tests process hang. To prevent these situations timeouts can be used. Timeouts can be set explicitly for flaky parts of code, or implicitly in a config.
 
-By default there is no timeout for tests, however you can change this value for a specific suite:
+> Previous timeout implementation was disabled as it had no effect when dealing with steps and promises. 
+
+### Steps Timeout
+
+It is possible to limit a step execution to specified time with `I.limitTime` command. 
+It will set timeout in seconds for the next executed step:
 
 ```js
-Feature('Stop me').timeout(5000); // set timeout to 5s
+// limit clicking to 5 seconds
+I.limitTime(5).click('Link')
 ```
 
-or for the test:
+It is possible to set a timeout for all steps implicitly (except waiters) using [stepTimeout plugin](/plugins/#steptimeout).
+
+### Tests Timeout
+
+Test timeout can be set in seconds via Scenario options:
 
 ```js
-// set timeout to 1s
-Scenario("Stop me faster",({ I }) => {
-  // test goes here
-}).timeout(1000);
+// limit test to 20 seconds
+Scenario('slow test that should be stopped', { timeout: 20 }, ({ I }) => {
+  // ...
+})
+```
+
+This timeout can be set globally in `codecept.conf.js` in seconds:
+
+```js
+exports.config = {
+
+  // each test must not run longer than 5 mins
+  timeout: 300,
 
-// alternative
-Scenario("Stop me faster", {timeout: 1000},({ I }) => {});
+}
+```
+
+### Suites Timeout
 
-// disable timeout for this scenario
-Scenario("Don't stop me", {timeout: 0},({ I }) => {});
+A timeout for a group of tests can be set on Feature level via options. 
+
+```js
+// limit all tests in this suite to 30 seconds
+Feature('flaky tests', { timeout: 30 })
 ```
 
 
@@ -249,45 +282,3 @@ Please note that some config changes can't be applied on the fly. For instance,
 
 Configuration changes will be reverted after a test or a suite.
 
-
-### Rerunning Flaky Tests Multiple Times <Badge text="Since 2.4" type="warning"/>
-
-End to end tests can be flaky for various reasons. Even when we can't do anything to solve this problem it we can do next two things:
-
-* Detect flaky tests in our suite
-* Fix flaky tests by rerunning them.
-
-Both tasks can be achieved with [`run-rerun` command](/commands/#run-rerun) which runs tests multiple times until all tests are passed.
-
-You should set min and max runs boundaries so when few tests fail in a row you can rerun them until they are succeeded.
-
-```js
-// inside to codecept.conf.js
-exports.config = { // ...
-  rerun: {
-    // run 4 times until 1st success
-    minSuccess: 1,
-    maxReruns: 4,
-  }
-}
-```
-
-If you want to check all your tests for stability you can set high boundaries for minimal success:
-
-```js
-// inside to codecept.conf.js
-exports.config = { // ...
-  rerun: {
-    // run all tests must pass exactly 5 times
-    minSuccess: 5,
-    maxReruns: 5,
-  }
-}
-```
-
-Now execute tests with `run-rerun` command:
-
-```
-npx codeceptjs run-rerun
-```
-

docs/plugins.md

@@ -626,6 +626,73 @@ Scenario('scenario tite', () => {
 
 -   `config`  
 
+## retryTo
+
+Adds global `retryTo` which retries steps a few times before failing.
+
+Enable this plugin in `codecept.conf.js` (enabled by default for new setups):
+
+```js
+plugins: {
+  retryTo: {
+    enabled: true
+  }
+}
+```
+
+Use it in your tests:
+
+```js
+// retry these steps 5 times before failing
+await retryTo((tryNum) => {
+  I.click('Open');
+  I.see('Opened')
+}, 5);
+```
+
+Set polling interval as 3rd argument (200ms by default):
+
+```js
+// retry these steps 5 times before failing
+await retryTo((tryNum) => {
+  I.click('Open');
+  I.see('Opened')
+}, 5, 100);
+```
+
+Default polling interval can be changed in a config:
+
+```js
+plugins: {
+  retryTo: {
+    enabled: true,
+    pollingInterval: 500,
+  }
+}
+```
+
+Disables retryFailedStep plugin for steps inside a block;
+
+Use this plugin if:
+
+-   you need repeat a set of actions in flaky tests
+-   iframe was not rendered and you need to retry switching to it
+
+#### Configuration
+
+-   `pollingInterval` - default interval between retries in ms. 200 by default.
+-   `registerGlobal` - to register `retryTo` function globally, true by default
+
+If `registerGlobal` is false you can use retryTo from the plugin:
+
+```js
+const retryTo = codeceptjs.container.plugins('retryTo');
+```
+
+### Parameters
+
+-   `config`  
+
 ## screenshotOnFail
 
 Creates screenshot on failure. Screenshot is saved into `output` directory.

examples/github_test.js

@@ -1,5 +1,5 @@
 // / <reference path="./steps.d.ts" />
-Feature('GitHub');
+Feature('GitHub', { timeout: 6 });
 
 Before(({ Smth }) => {
   Smth.openGitHub();
@@ -12,7 +12,7 @@ Scenario('Visit Home Page @retry', async ({ I }) => {
   I.retry(2).see('IMAGES');
 });
 
-Scenario('search @grop', ({ I }) => {
+Scenario('search @grop', { timeout: 6 }, ({ I }) => {
   I.amOnPage('https://github.com/search');
   const a = {
     b: {
@@ -32,20 +32,20 @@ Scenario('search @grop', ({ I }) => {
     urls: {},
   };
   I.fillField('Search GitHub', 'CodeceptJS');
-  pause({ a, b });
+  // pause({ a, b });
   I.pressKey('Enter');
-  I.wait(1);
-  pause();
+  I.wait(3);
+  // pause();
   I.see('Codeception/CodeceptJS', locate('.repo-list .repo-list-item').first());
 });
 
-Scenario('signin', ({ I, loginPage }) => {
+Scenario('signin @sign', { timeout: 6 }, ({ I, loginPage }) => {
   I.say('it should not enter');
   loginPage.login('something@totest.com', '123456');
   I.see('Incorrect username or password.', '.flash-error');
 }).tag('normal').tag('important').tag('@slow');
 
-Scenario('signin2', ({ I, Smth }) => {
+Scenario('signin2', { timeout: 1 }, ({ I, Smth }) => {
   Smth.openAndLogin();
   I.see('Incorrect username or password.', '.flash-error');
 });

examples/yahoo_test.js

@@ -1,9 +1,9 @@
 Feature('Yahoo test');
 
-Scenario('Nightmare basic test', ({ I }) => {
+Scenario('Nightmare basic test', { timeout: 3 }, ({ I }) => {
   I.amOnPage('http://yahoo.com');
   I.fillField('p', 'github nightmare');
-  I.click('Search Web');
+  I.click('Search');
   I.waitForElement('#main', 2);
   I.seeElement('#main .searchCenterMiddle li a');
   // I.seeElement("//a[contains(@href,'github.com/segmentio/nightmare')]");

lib/actor.js

@@ -26,30 +26,18 @@ class Actor {
     });
   }
 
-  /**
-   * @function
-   * @return {number}
-   * @inner
-   */
-  popTimeout() {
-    const nextStepTimeout = this.nextStepTimeout;
-    this.nextStepTimeout = undefined;
-    return nextStepTimeout || 0;
-  }
-
   /**
    * set the maximum execution time for the next step
    * @function
    * @param {number} timeout - step timeout in seconds
-   * @param {boolean} force - whether to update timeout when it is already set for the step, default is true
    * @return {this}
    * @inner
    */
-  limitTime(timeout, force = true) {
-    // give priority to timeout set from test code
-    if (this.nextStepTimeout === undefined || force) {
-      this.nextStepTimeout = timeout * 1000;
-    }
+  limitTime(timeout) {
+    event.dispatcher.once(event.step.before, (step) => {
+      output.log(`Timeout to ${step}: ${timeout}s`);
+      step.totalTimeout = timeout * 1000;
+    });
     return this;
   }
 
@@ -115,7 +103,7 @@ module.exports = function (obj = {}) {
                 step.actor = translation.I;
               }
               // add methods to promise chain
-              return recordStep(step, Array.from(arguments), actor);
+              return recordStep(step, Array.from(arguments));
             };
           }
         });
@@ -124,7 +112,7 @@ module.exports = function (obj = {}) {
   return actor;
 };
 
-function recordStep(step, args, actor) {
+function recordStep(step, args) {
   step.status = 'queued';
   step.setArguments(args);
 
@@ -141,7 +129,7 @@ function recordStep(step, args, actor) {
       step.startTime = Date.now();
     }
     return val = step.run(...args);
-  }, false, undefined, actor.popTimeout());
+  }, false, undefined, step.totalTimeout);
 
   event.emit(event.step.after, step);
 

lib/codecept.js

@@ -95,6 +95,7 @@ class Codecept {
     runHook(require('./listener/steps'));
     runHook(require('./listener/config'));
     runHook(require('./listener/helpers'));
+    runHook(require('./listener/timeout'));
     runHook(require('./listener/exit'));
 
     // custom hooks

lib/config.js

@@ -13,6 +13,7 @@ const defaultConfig = {
   include: {},
   mocha: {},
   bootstrap: null,
+  timeout: null,
   teardown: null,
   hooks: [],
   gherkin: {},

lib/interfaces/featureConfig.js

@@ -21,6 +21,9 @@ class FeatureConfig {
    * @returns {this}
    */
   timeout(timeout) {
+    console.log(`Feature('${this.suite.title}').timeout(${timeout}) is deprecated!`);
+    console.log(`Please use Feature('${this.suite.title}', { timeout: ${timeout / 1000} }) instead`);
+    console.log('Timeout should be set in seconds');
     this.suite.timeout(timeout);
     return this;
   }

lib/interfaces/scenarioConfig.js

@@ -45,6 +45,10 @@ class ScenarioConfig {
    * @returns {this}
    */
   timeout(timeout) {
+    console.log(`Scenario('${this.test.title}', () => {}).timeout(${timeout}) is deprecated!`);
+    console.log(`Please use Scenario('${this.test.title}', { timeout: ${timeout / 1000} }, () => {}) instead`);
+    console.log('Timeout should be set in seconds');
+
     this.test.timeout(timeout);
     return this;
   }