[Security Solution] ~200 ways to decrease flakiness in Cypress (#157387)

This PR addresses Cypress parallelisation in Security Solution scope. It is a first step in improving e2e tests reliability and gives us a solid foundation for further work on flakiness and run times. With this PR each CI job starts 3 instances of ES, kibana and Cypress runner in isolation. Other issues addressed in this PR: - Use click() instead of trigger('click') in all Cypress tests. - Use testIsolation in all Cypress tests. - login before each test to ensure that the correct user is always logged in. Also, with test isolation enabled, login should be required before each test. - use visit() instead of cy.visit() in some cases so that tests wait for the page to fully load before executing. - Cypress e2e tests are no longer compatible with Firefox (or any non-Chromium based browser) due to using cypress-real-events - Removes uses of cypress-pipe. Tests that used cypress-pipe in order to retry click events will no longer retry click events. - Fixes an error related to test files being overwritten by Webpack file processing --------- Co-authored-by: Patryk Kopycinski <contact@patrykkopycinski.com> Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
elastic · May 30, 2023 · c9658bf · c9658bf
1 parent 3d05fc6
commit c9658bf
Show file tree

Hide file tree

Showing 214 changed files with 2,485 additions and 1,526 deletions.
diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml
@@ -38,13 +38,11 @@ disabled:
   - x-pack/test/security_solution_cypress/cases_cli_config.ts
   - x-pack/test/security_solution_cypress/ccs_config.ts
   - x-pack/test/security_solution_cypress/cli_config.ts
-  - x-pack/test/security_solution_cypress/cli_config_parallel.ts
   - x-pack/test/security_solution_cypress/cli_config_investigations_parallel.ts
   - x-pack/test/security_solution_cypress/config.firefox.ts
   - x-pack/test/security_solution_cypress/config.ts
   - x-pack/test/security_solution_cypress/response_ops_cli_config.ts
   - x-pack/test/security_solution_cypress/upgrade_config.ts
-  - x-pack/test/security_solution_cypress/visual_config.ts
   - x-pack/test/threat_intelligence_cypress/visual_config.ts
   - x-pack/test/threat_intelligence_cypress/cli_config_parallel.ts
   - x-pack/test/threat_intelligence_cypress/config.ts

diff --git a/.buildkite/pipelines/pull_request/defend_workflows.yml b/.buildkite/pipelines/pull_request/defend_workflows.yml
@@ -5,6 +5,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '-1'

diff --git a/.buildkite/pipelines/pull_request/response_ops.yml b/.buildkite/pipelines/pull_request/response_ops.yml
@@ -5,6 +5,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '-1'

diff --git a/.buildkite/pipelines/pull_request/security_solution.yml b/.buildkite/pipelines/pull_request/security_solution.yml
@@ -5,7 +5,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
-    parallelism: 4
+    parallelism: 15
     retry:
       automatic:
         - exit_status: '-1'

diff --git a/.buildkite/scripts/steps/functional/defend_workflows.sh b/.buildkite/scripts/steps/functional/defend_workflows.sh
@@ -2,16 +2,15 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
-
-.buildkite/scripts/bootstrap.sh
-node scripts/build_kibana_platform_plugins.js
+source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-defend-workflows-cypress
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-echo "--- Defend Workflows Cypress tests"
+Xvfb -screen 0 1680x946x24 :99 &
 
-node scripts/functional_tests \
-  --debug --bail \
-  --config x-pack/test/defend_workflows_cypress/cli_config.ts
+export DISPLAY=:99
+
+echo "--- Defend Workflows Cypress tests"
 
+yarn --cwd x-pack/plugins/security_solution cypress:dw:run
diff --git a/.buildkite/scripts/steps/functional/response_ops.sh b/.buildkite/scripts/steps/functional/response_ops.sh
@@ -5,10 +5,12 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+Xvfb -screen 0 1680x946x24 :99 &
+
+export DISPLAY=:99
 
 echo "--- Response Ops Cypress Tests on Security Solution"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  --config x-pack/test/security_solution_cypress/response_ops_cli_config.ts
+yarn --cwd x-pack/plugins/security_solution cypress:run:respops
diff --git a/.buildkite/scripts/steps/functional/response_ops_cases.sh b/.buildkite/scripts/steps/functional/response_ops_cases.sh
@@ -5,10 +5,12 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+Xvfb -screen 0 1680x946x24 :99 &
+
+export DISPLAY=:99
 
 echo "--- Response Ops Cases Cypress Tests on Security Solution"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  --config x-pack/test/security_solution_cypress/cases_cli_config.ts
+yarn --cwd x-pack/plugins/security_solution cypress:run:cases
diff --git a/.buildkite/scripts/steps/functional/security_solution.sh b/.buildkite/scripts/steps/functional/security_solution.sh
@@ -5,12 +5,12 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
-export CLI_NUMBER=${CLI_NUMBER:-$((BUILDKITE_PARALLEL_JOB+1))}
-export CLI_COUNT=${CLI_COUNT:-$BUILDKITE_PARALLEL_JOB_COUNT}
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+Xvfb :99 -screen 0 1600x1200x24 &
+
+export DISPLAY=:99
 
 echo "--- Security Solution tests (Chrome)"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  --config x-pack/test/security_solution_cypress/cli_config_parallel.ts
+yarn --cwd x-pack/plugins/security_solution cypress:run
diff --git a/.buildkite/scripts/steps/functional/security_solution_investigations.sh b/.buildkite/scripts/steps/functional/security_solution_investigations.sh
@@ -5,12 +5,13 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
-export CLI_NUMBER=${CLI_NUMBER:-$((BUILDKITE_PARALLEL_JOB+1))}
-export CLI_COUNT=${CLI_COUNT:-$BUILDKITE_PARALLEL_JOB_COUNT}
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-echo "--- Security Solution tests (Chrome)"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  --config x-pack/test/security_solution_cypress/cli_config_investigations_parallel.ts
+Xvfb :99 -screen 0 1600x1200x24 &
+
+export DISPLAY=:99
+
+echo "--- Investigations Cypress Tests on Security Solution"
+
+yarn --cwd x-pack/plugins/security_solution cypress:investigations:run
diff --git a/package.json b/package.json
@@ -1000,6 +1000,7 @@
     "@bazel/ibazel": "^0.16.2",
     "@bazel/typescript": "4.6.2",
     "@cypress/code-coverage": "^3.10.0",
+    "@cypress/grep": "^3.1.5",
     "@cypress/snapshot": "^2.1.7",
     "@cypress/webpack-preprocessor": "^5.12.2",
     "@elastic/eslint-plugin-eui": "0.0.2",
@@ -1349,6 +1350,7 @@
     "chance": "1.0.18",
     "chromedriver": "^113.0.0",
     "clean-webpack-plugin": "^3.0.0",
+    "cli-table3": "^0.6.1",
     "compression-webpack-plugin": "^4.0.0",
     "copy-webpack-plugin": "^6.0.2",
     "cpy": "^8.1.1",
@@ -1360,10 +1362,9 @@
     "cypress-axe": "^1.4.0",
     "cypress-file-upload": "^5.0.8",
     "cypress-multi-reporters": "^1.6.3",
-    "cypress-pipe": "^2.0.0",
     "cypress-react-selector": "^3.0.0",
-    "cypress-real-events": "^1.7.6",
-    "cypress-recurse": "^1.31.2",
+    "cypress-real-events": "^1.8.1",
+    "cypress-recurse": "^1.35.1",
     "date-fns": "^2.29.3",
     "debug": "^2.6.9",
     "delete-empty": "^2.0.0",
@@ -1439,8 +1440,8 @@
     "minimist": "^1.2.6",
     "mocha": "^10.1.0",
     "mocha-junit-reporter": "^2.0.2",
-    "mochawesome": "^7.0.1",
-    "mochawesome-merge": "^4.2.1",
+    "mochawesome": "^7.1.3",
+    "mochawesome-merge": "^4.3.0",
     "mock-fs": "^5.1.2",
     "ms-chromium-edge-driver": "^0.5.1",
     "multistream": "^4.1.0",

diff --git a/packages/kbn-cypress-config/index.ts b/packages/kbn-cypress-config/index.ts
@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { v4 as uuid } from 'uuid';
 import { defineConfig } from 'cypress';
 import wp from '@cypress/webpack-preprocessor';
 
@@ -15,9 +16,12 @@ export function defineCypressConfig(options?: Cypress.ConfigOptions<any>) {
     e2e: {
       ...options?.e2e,
       setupNodeEvents(on, config) {
-        on(
-          'file:preprocessor',
-          wp({
+        on('file:preprocessor', (file) => {
+          const id = uuid();
+          // Fix an issue with running Cypress parallel
+          file.outputPath = file.outputPath.replace(/^(.*\/)(.*?)(\..*)$/, `$1$2.${id}$3`);
+
+          return wp({
             webpackOptions: {
               resolve: {
                 extensions: ['.ts', '.tsx', '.js'],
@@ -39,8 +43,8 @@ export function defineCypressConfig(options?: Cypress.ConfigOptions<any>) {
                 ],
               },
             },
-          })
-        );
+          })(file);
+        });
 
         const external = options?.e2e?.setupNodeEvents;
         if (external) {

diff --git a/packages/kbn-dev-proc-runner/src/with_proc_runner.ts b/packages/kbn-dev-proc-runner/src/with_proc_runner.ts
@@ -19,9 +19,9 @@ import { ProcRunner } from './proc_runner';
  *  @param  {async Function} fn
  *  @return {Promise<undefined>}
  */
-export async function withProcRunner(
+export async function withProcRunner<T = void>(
   log: ToolingLog,
-  fn: (procs: ProcRunner) => Promise<void>
+  fn: (procs: ProcRunner) => Promise<T>
 ): Promise<void> {
   const procs = new ProcRunner(log);
 

diff --git a/packages/kbn-test/index.ts b/packages/kbn-test/index.ts
@@ -14,6 +14,7 @@ export { startServersCli, startServers } from './src/functional_tests/start_serv
 // @internal
 export { runTestsCli, runTests } from './src/functional_tests/run_tests';
 
+export { runElasticsearch, runKibanaServer } from './src/functional_tests/lib';
 export { getKibanaCliArg, getKibanaCliLoggers } from './src/functional_tests/lib/kibana_cli_args';
 
 export type {

diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts
@@ -192,6 +192,7 @@ export const schema = Joi.object()
         elasticsearch: urlPartsSchema({
           requiredKeys: ['port'],
         }),
+        fleetserver: urlPartsSchema(),
       })
       .default(),
 

diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts
@@ -19,6 +19,7 @@ interface RunElasticsearchOptions {
   config: Config;
   onEarlyExit?: (msg: string) => void;
   logsDir?: string;
+  name?: string;
 }
 
 interface CcsConfig {
@@ -63,13 +64,13 @@ function getEsConfig({
 export async function runElasticsearch(
   options: RunElasticsearchOptions
 ): Promise<() => Promise<void>> {
-  const { log, logsDir } = options;
+  const { log, logsDir, name } = options;
   const config = getEsConfig(options);
 
   if (!config.ccsConfig) {
     const node = await startEsNode({
       log,
-      name: 'ftr',
+      name: name ?? 'ftr',
       logsDir,
       config,
     });
@@ -81,7 +82,7 @@ export async function runElasticsearch(
   const remotePort = await getPort();
   const remoteNode = await startEsNode({
     log,
-    name: 'ftr-remote',
+    name: name ?? 'ftr-remote',
     logsDir,
     config: {
       ...config,
@@ -92,7 +93,7 @@ export async function runElasticsearch(
 
   const localNode = await startEsNode({
     log,
-    name: 'ftr-local',
+    name: name ?? 'ftr-local',
     logsDir,
     config: {
       ...config,

diff --git a/x-pack/plugins/security_solution/cypress/.eslintrc.json b/x-pack/plugins/security_solution/cypress/.eslintrc.json
@@ -1,9 +1,13 @@
 {
   "plugins": ["cypress"],
+  "extends": [
+    "plugin:cypress/recommended"
+  ],
   "env": {
     "cypress/globals": true
   },
   "rules": {
+    "cypress/no-force": "warn",
     "import/no-extraneous-dependencies": "off"
   }
 }
diff --git a/x-pack/plugins/security_solution/cypress/README.md b/x-pack/plugins/security_solution/cypress/README.md
@@ -440,9 +440,6 @@ taken into consideration until another solution is implemented:
 
 Remember that minimizing the number of times the web page is loaded, we minimize as well the execution time.
 
-### Cypress-pipe
-It is very common in the code to don't have click handlers regitered. In this specific case, please use [Cypress pipe](https://www.cypress.io/blog/2019/01/22/when-can-the-test-click/). 
-
 ### CCS test specific
 When testing CCS we want to put our focus in making sure that our `Source` instance is receiving properly the data that comes from the `Remote` instances, as well as the data is displayed as we expect on the `Source`.
 

diff --git a/x-pack/plugins/security_solution/cypress/cypress.config.ts b/x-pack/plugins/security_solution/cypress/cypress.config.ts
@@ -11,13 +11,16 @@ export default defineCypressConfig({
   defaultCommandTimeout: 60000,
   execTimeout: 60000,
   pageLoadTimeout: 60000,
+  responseTimeout: 60000,
   screenshotsFolder: '../../../target/kibana-security-solution/cypress/screenshots',
   trashAssetsBeforeRuns: false,
   video: false,
   videosFolder: '../../../target/kibana-security-solution/cypress/videos',
   viewportHeight: 946,
   viewportWidth: 1680,
+  numTestsKeptInMemory: 10,
   e2e: {
-    baseUrl: 'http://localhost:5601',
+    experimentalRunAllSpecs: true,
+    experimentalMemoryManagement: true,
   },
 });
diff --git a/x-pack/plugins/security_solution/cypress/cypress_ci.config.ts b/x-pack/plugins/security_solution/cypress/cypress_ci.config.ts
@@ -14,7 +14,7 @@ export default defineCypressConfig({
   pageLoadTimeout: 150000,
   numTestsKeptInMemory: 0,
   retries: {
-    runMode: 2,
+    runMode: 1,
   },
   screenshotsFolder: '../../../target/kibana-security-solution/cypress/screenshots',
   trashAssetsBeforeRuns: false,
@@ -24,5 +24,7 @@ export default defineCypressConfig({
   viewportWidth: 1680,
   e2e: {
     baseUrl: 'http://localhost:5601',
+    experimentalMemoryManagement: true,
+    specPattern: './cypress/e2e/**/*.cy.ts',
   },
 });
diff --git a/x-pack/plugins/security_solution/cypress/e2e/cases/attach_alert_to_case.cy.ts b/x-pack/plugins/security_solution/cypress/e2e/cases/attach_alert_to_case.cy.ts
@@ -19,6 +19,7 @@ import { ATTACH_ALERT_TO_CASE_BUTTON, ATTACH_TO_NEW_CASE_BUTTON } from '../../sc
 import { LOADING_INDICATOR } from '../../screens/security_header';
 
 const loadDetectionsPage = (role: ROLES) => {
+  login(role);
   waitForPageWithoutDateRange(ALERTS_URL, role);
   waitForAlertsToPopulate();
 };
@@ -35,7 +36,6 @@ describe('Alerts timeline', () => {
 
   context('Privileges: read only', () => {
     beforeEach(() => {
-      login(ROLES.reader);
       loadDetectionsPage(ROLES.reader);
     });
 
@@ -54,7 +54,6 @@ describe('Alerts timeline', () => {
 
   context('Privileges: can crud', () => {
     beforeEach(() => {
-      login(ROLES.platform_engineer);
       loadDetectionsPage(ROLES.platform_engineer);
       cy.get(LOADING_INDICATOR).should('not.exist'); // on CI, waitForPageToBeLoaded fails because the loading icon can't be found
     });

diff --git a/x-pack/plugins/security_solution/cypress/e2e/cases/attach_timeline.cy.ts b/x-pack/plugins/security_solution/cypress/e2e/cases/attach_timeline.cy.ts
@@ -23,9 +23,10 @@ describe('attach timeline to case', () => {
   context('without cases created', () => {
     before(() => {
       cleanKibana();
-      login();
     });
+
     beforeEach(() => {
+      login();
       deleteTimelines();
       createTimeline(getTimeline()).then((response) => {
         cy.wrap(response.body.data.persistTimeline.timeline).as('myTimeline');
@@ -60,13 +61,18 @@ describe('attach timeline to case', () => {
 
   context('with cases created', () => {
     before(() => {
+      login();
       deleteTimelines();
       createTimeline(getTimeline()).then((response) =>
         cy.wrap(response.body.data.persistTimeline.timeline.savedObjectId).as('timelineId')
       );
       createCase(getCase1()).then((response) => cy.wrap(response.body.id).as('caseId'));
     });
 
+    beforeEach(() => {
+      login();
+    });
+
     it('attach timeline to an existing case', function () {
       visitTimeline(this.timelineId);
       attachTimelineToExistingCase();