Feat/analyze result (#4753)

* added result object, improved transport of tests

* fixed setting name for test

* fixed workers tests

* improved handling information between workers and main process

* fixed tests

* added result to typings

* fixed printing steps

* fixed def & failing tests

* fixing timeout errors

* fixed mocha hooks, analyze plugin, added custom reporter tests

* improved prompt for analyze plugin

* refactored timeouts to fix tests

* fixed saving screenshot file

* fixed step sections

* fixed tests for timeout errors

* added change fot pageInfo plugin

* fixed timeout tests

* fixed timeout tests with retries

---------

Co-authored-by: DavertMik <davert@testomat.io>

Author: DavertMik

.github/workflows/webdriver.yml

@@ -38,7 +38,7 @@ jobs:
           PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
       - name: start a server
         run: 'php -S 127.0.0.1:8000 -t test/data/app &'
-      - name: Check CodeceptJS can be started
+      - name: check
         run: './bin/codecept.js check -c test/acceptance/codecept.WebDriver.js'
       - name: run unit tests
         run: ./node_modules/.bin/mocha test/helper/WebDriver_test.js --exit

docs/plugins.md

@@ -7,6 +7,12 @@ title: Plugins
 
 <!-- Generated by documentation.js. Update this documentation by updating the source code. -->
 
+## analyze
+
+### Parameters
+
+- `config` **any** (optional, default `{}`)
+
 ## autoDelay
 
 Sometimes it takes some time for a page to respond to user's actions.
@@ -521,29 +527,13 @@ I.click('=sign-up') // matches => [data-qa=sign-up],[data-test=sign-up]
 
 - `config` &#x20;
 
-## debugErrors
-
-Prints errors found in HTML code after each failed test.
+## customReporter
 
-It scans HTML and searches for elements with error classes.
-If an element found prints a text from it to console and adds as artifact to the test.
-
-Enable this plugin in config:
-
-```js
-plugins: {
- debugErrors: {
-  enabled: true,
-}
-```
-
-Additional config options:
-
-- `errorClasses` - list of classes to search for errors (default: `['error', 'warning', 'alert', 'danger']`)
+Sample custom reporter for CodeceptJS.
 
 ### Parameters
 
-- `config` (optional, default `{}`)
+- `config` &#x20;
 
 ## eachElement
 
@@ -672,6 +662,32 @@ More config options are available:
 
 - `config` (optional, default `{}`)
 
+## pageInfo
+
+Collects information from web page after each failed test and adds it to the test as an artifact.
+It is suggested to enable this plugin if you run tests on CI and you need to debug failed tests.
+This plugin can be paired with `analyze` plugin to provide more context.
+
+It collects URL, HTML errors (by classes), and browser logs.
+
+Enable this plugin in config:
+
+```js
+plugins: {
+ pageInfo: {
+  enabled: true,
+}
+```
+
+Additional config options:
+
+- `errorClasses` - list of classes to search for errors (default: `['error', 'warning', 'alert', 'danger']`)
+- `browserLogs` - list of types of errors to search for in browser logs (default: `['error']`)
+
+### Parameters
+
+- `config` (optional, default `{}`)
+
 ## pauseOnFail
 
 Automatically launches [interactive pause][11] when a test fails.

examples/codecept.config.js

@@ -1,4 +1,4 @@
-require('./heal_recipes');
+require('./heal_recipes')
 
 exports.config = {
   output: './output',
@@ -34,22 +34,21 @@ exports.config = {
   },
   gherkin: {
     features: './features/*.feature',
-    steps: [
-      './step_definitions/steps.js',
-    ],
+    steps: ['./step_definitions/steps.js'],
   },
   plugins: {
-    tryTo: {
-      enabled: true,
-    },
-    heal: {
+    analyze: {
       enabled: true,
     },
+    // heal: {
+    //   enabled: true,
+    // },
+    // customReporter: {
+    //   enabled: true,
+    // },
     wdio: {
       enabled: false,
-      services: [
-        'selenium-standalone',
-      ],
+      services: ['selenium-standalone'],
     },
     stepByStepReport: {},
     autoDelay: {
@@ -61,10 +60,8 @@ exports.config = {
     subtitles: {
       enabled: true,
     },
-    retryTo: {
-      enabled: true,
-    },
   },
+
   tests: './*_test.js',
   // timeout: 100,
   multiple: {
@@ -73,11 +70,8 @@ exports.config = {
     },
     default: {
       grep: 'signin',
-      browsers: [
-        'chrome',
-        'firefox',
-      ],
+      browsers: ['chrome', 'firefox'],
     },
   },
   name: 'tests',
-};
+}

examples/github_test.js

@@ -1,36 +1,36 @@
 // / <reference path="./steps.d.ts" />
-Feature('GitHub');
+Feature('GitHub')
 
 Before(({ I }) => {
-  I.amOnPage('https://github.com');
-});
+  I.amOnPage('https://github.com')
+  I.see('GitLab')
+})
 
 xScenario('test ai features', ({ I }) => {
-  I.amOnPage('https://getbootstrap.com/docs/5.1/examples/checkout/');
-  pause();
-});
+  I.amOnPage('https://getbootstrap.com/docs/5.1/examples/checkout/')
+})
 
 Scenario('Incorrect search for Codeceptjs', ({ I }) => {
-  I.fillField('.search-input', 'CodeceptJS');
-  I.pressKey('Enter');
-  I.waitForElement('[data-testid=search-sub-header]', 10);
-  I.see('Supercharged End 2 End Testing');
-});
+  I.fillField('.search-input', 'CodeceptJS')
+  I.pressKey('Enter')
+  I.waitForElement('[data-testid=search-sub-header]', 10)
+  I.see('Supercharged End 2 End Testing')
+})
 
 Scenario('Visit Home Page @retry', async ({ I }) => {
   // .retry({ retries: 3, minTimeout: 1000 })
-  I.retry(2).see('GitHub');
-  I.retry(3).see('ALL');
-  I.retry(2).see('IMAGES');
-});
+  I.retry(2).see('GitHub')
+  I.retry(3).see('ALL')
+  I.retry(2).see('IMAGES')
+})
 
 Scenario('search @grop', { timeout: 6 }, ({ I }) => {
-  I.amOnPage('https://github.com/search');
+  I.amOnPage('https://github.com/search')
   const a = {
     b: {
       c: 'asdasdasd',
     },
-  };
+  }
   const b = {
     users: {
       admin: {
@@ -42,35 +42,38 @@ Scenario('search @grop', { timeout: 6 }, ({ I }) => {
       other: (world = '') => `Hello ${world}`,
     },
     urls: {},
-  };
-  I.fillField('Search GitHub', 'CodeceptJS');
+  }
+  I.fillField('Search GitHub', 'CodeceptJS')
   // pause({ a, b });
-  I.pressKey('Enter');
-  I.wait(3);
+  I.pressKey('Enter')
+  I.wait(3)
   // pause();
-  I.see('Codeception/CodeceptJS', locate('.repo-list .repo-list-item').first());
-});
+  I.see('Codeception/CodeceptJS', locate('.repo-list .repo-list-item').first())
+})
 
 Scenario('signin @sign', { timeout: 6 }, ({ I, loginPage }) => {
-  I.say('it should not enter');
-  loginPage.login('something@totest.com', '123456');
-  I.see('Incorrect username or password.', '.flash-error');
-}).tag('normal').tag('important').tag('@slow');
+  I.say('it should not enter')
+  loginPage.login('something@totest.com', '123456')
+  I.see('Incorrect username or password.', '.flash-error')
+})
+  .tag('normal')
+  .tag('important')
+  .tag('@slow')
 
 Scenario('signin2', { timeout: 1 }, ({ I, Smth }) => {
-  Smth.openAndLogin();
-  I.see('Incorrect username or password.', '.flash-error');
-});
+  Smth.openAndLogin()
+  I.see('Incorrect username or password.', '.flash-error')
+})
 
 Scenario('register', ({ I }) => {
   within('.js-signup-form', () => {
-    I.fillField('user[login]', 'User');
-    I.fillField('user[email]', 'user@user.com');
-    I.fillField('user[password]', 'user@user.com');
-    I.fillField('q', 'aaa');
-    I.click('button');
-  });
-  I.see('There were problems creating your account.');
-  I.click('Explore');
-  I.seeInCurrentUrl('/explore');
-});
+    I.fillField('user[login]', 'User')
+    I.fillField('user[email]', 'user@user.com')
+    I.fillField('user[password]', 'user@user.com')
+    I.fillField('q', 'aaa')
+    I.click('button')
+  })
+  I.see('There were problems creating your account.')
+  I.click('Explore')
+  I.seeInCurrentUrl('/explore')
+})

examples/selenoid-example/browsers.json

@@ -3,8 +3,7 @@ const MetaStep = require('./step/meta')
 const recordStep = require('./step/record')
 const container = require('./container')
 const { methodsOfObject } = require('./utils')
-const { TIMEOUT_ORDER } = require('./step/timeout')
-const recorder = require('./recorder')
+const { TIMEOUT_ORDER } = require('./timeout')
 const event = require('./event')
 const store = require('./store')
 const output = require('./output')