[7.17] [security_solution] Cypress flaky tests catcher (#162376) (#16…

…3992)

# Backport

This will backport the following commits from `main` to `7.17`:
- [[security_solution] Cypress flaky tests catcher
(#162376)](#162376)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Patryk
Kopyciński","email":"contact@patrykkopycinski.com"},"sourceCommit":{"committedDate":"2023-08-09T21:58:04Z","message":"[security_solution]
Cypress flaky tests catcher (#162376)\n\n## Summary\r\n\r\nInspired by
https://glebbahmutov.com/blog/burning-tests/\r\n\r\nImplements the idea
presented
here\r\nhttps://glebbahmutov.com/blog/burning-tests/#bonus-3-burning-new-or-changed-specs\r\n\r\nIn
short, on PR that is changing/adding new Cypress spec files we
will\r\ntry to \"burn\" them, it means we will try to run each `it` `2`
times to\r\nmake sure tests are written in a way that gives Cypress a
chance to\r\nrecover from the failed test.\r\nAlso adding a command that
allows to \"burn\" tests locally\r\n```\r\nyarn cypress:burn --spec
\"<>\"\r\n```\r\n\r\nRight now the job is set to `soft_fail`, so it is
not going to block the\r\nPR from merging, but hopefully will help the
Team to recognize potential\r\nflakiness before it is merged to
`main`","sha":"09aaecb59d5e82e17bf6f274de3cedd394bbe25b","branchLabelMapping":{"^v8.10.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:skip","v8.10.0"],"number":162376,"url":"#162376
Cypress flaky tests catcher (#162376)\n\n## Summary\r\n\r\nInspired by
https://glebbahmutov.com/blog/burning-tests/\r\n\r\nImplements the idea
presented
here\r\nhttps://glebbahmutov.com/blog/burning-tests/#bonus-3-burning-new-or-changed-specs\r\n\r\nIn
short, on PR that is changing/adding new Cypress spec files we
will\r\ntry to \"burn\" them, it means we will try to run each `it` `2`
times to\r\nmake sure tests are written in a way that gives Cypress a
chance to\r\nrecover from the failed test.\r\nAlso adding a command that
allows to \"burn\" tests locally\r\n```\r\nyarn cypress:burn --spec
\"<>\"\r\n```\r\n\r\nRight now the job is set to `soft_fail`, so it is
not going to block the\r\nPR from merging, but hopefully will help the
Team to recognize potential\r\nflakiness before it is merged to
`main`","sha":"09aaecb59d5e82e17bf6f274de3cedd394bbe25b"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.10.0","labelRegex":"^v8.10.0$","isSourceBranch":true,"state":"MERGED","url":"#162376
Cypress flaky tests catcher (#162376)\n\n## Summary\r\n\r\nInspired by
https://glebbahmutov.com/blog/burning-tests/\r\n\r\nImplements the idea
presented
here\r\nhttps://glebbahmutov.com/blog/burning-tests/#bonus-3-burning-new-or-changed-specs\r\n\r\nIn
short, on PR that is changing/adding new Cypress spec files we
will\r\ntry to \"burn\" them, it means we will try to run each `it` `2`
times to\r\nmake sure tests are written in a way that gives Cypress a
chance to\r\nrecover from the failed test.\r\nAlso adding a command that
allows to \"burn\" tests locally\r\n```\r\nyarn cypress:burn --spec
\"<>\"\r\n```\r\n\r\nRight now the job is set to `soft_fail`, so it is
not going to block the\r\nPR from merging, but hopefully will help the
Team to recognize potential\r\nflakiness before it is merged to
`main`","sha":"09aaecb59d5e82e17bf6f274de3cedd394bbe25b"}}]}]
BACKPORT-->

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Tiago Costa <tiago.costa@elastic.co>

.buildkite/pipelines/pull_request/security_solution.yml

@@ -12,3 +12,14 @@ steps:
           limit: 1
     artifact_paths:
       - "target/kibana-security-solution/**/*"
+
+  - command: .buildkite/scripts/steps/functional/security_solution_burn.sh
+    label: 'Security Solution Cypress tests, burning changed specs'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 1
+    soft_fail: true
+    artifact_paths:
+      - "target/kibana-security-solution/**/*"

.buildkite/scripts/steps/functional/security_solution_burn.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'false'
+
+echo "--- Security Solution Cypress tests, burning changed specs (Chrome)"
+
+yarn --cwd x-pack/plugins/security_solution cypress:changed-specs-only

package.json

@@ -439,6 +439,7 @@
     "@babel/types": "^7.21.2",
     "@bazel/ibazel": "^0.16.2",
     "@bazel/typescript": "^3.8.0",
+    "@cypress/grep": "^3.1.5",
     "@cypress/snapshot": "^2.1.7",
     "@cypress/webpack-preprocessor": "^5.12.2",
     "@elastic/eslint-plugin-eui": "0.0.2",
@@ -720,6 +721,7 @@
     "fast-glob": "2.2.7",
     "fetch-mock": "^7.3.9",
     "file-loader": "^4.2.0",
+    "find-cypress-specs": "^1.35.1",
     "form-data": "^4.0.0",
     "geckodriver": "^4.0.0",
     "glob-watcher": "5.0.3",

x-pack/plugins/security_solution/cypress/e2e/timelines/creation.cy.ts

@@ -106,7 +106,7 @@ describe('Timelines', (): void => {
   });
 });
 
-describe('Create a timeline from a template', () => {
+describe.skip('Create a timeline from a template', () => {
   before(() => {
     cy.intercept('/api/timeline*').as('timeline');
     cleanKibana();

x-pack/plugins/security_solution/cypress/scripts/run_cypress/parallel.ts

@@ -11,6 +11,9 @@ import _ from 'lodash';
 import globby from 'globby';
 import pMap from 'p-map';
 import cypress from 'cypress';
+import { findChangedFiles } from 'find-cypress-specs';
+import minimatch from 'minimatch';
+import path from 'path';
 
 import {
   EsVersion,
@@ -62,13 +65,39 @@ const retrieveIntegrations = (
 export const cli = () => {
   run(
     async () => {
-      const { argv } = yargs(process.argv.slice(2));
+      const { argv } = yargs(process.argv.slice(2)).coerce('env', (arg: string) =>
+        arg.split(',').reduce((acc, curr) => {
+          const [key, value] = curr.split('=');
+          if (key === 'burn') {
+            acc[key] = parseInt(value, 10);
+          } else {
+            acc[key] = value;
+          }
+          return acc;
+        }, {} as Record<string, string | number>)
+      );
 
       const isOpen = argv._[0] === 'open';
       const cypressConfigFilePath = require.resolve(`../../../${argv.configFile}`) as string;
       const cypressConfigFile = await import(require.resolve(`../../../${argv.configFile}`));
       const spec: string | undefined = argv?.spec as string;
-      const files = retrieveIntegrations(spec ? [spec] : cypressConfigFile?.e2e?.specPattern);
+      let files = retrieveIntegrations(spec ? [spec] : cypressConfigFile?.e2e?.specPattern);
+
+      if (argv.changedSpecsOnly) {
+        const basePath = process.cwd().split('kibana/')[1];
+        files = findChangedFiles('7.17', false)
+          .filter(
+            minimatch.filter(path.join(basePath, cypressConfigFile?.e2e?.specPattern), {
+              matchBase: true,
+            })
+          )
+          .map((filePath: string) => filePath.replace(basePath, '.'));
+
+        if (!files?.length) {
+          // eslint-disable-next-line no-process-exit
+          return process.exit(0);
+        }
+      }
 
       if (!files?.length) {
         throw new Error('No files found');
@@ -360,7 +389,7 @@ ${JSON.stringify(config.getAll(), null, 2)}
               ...ftrEnv,
 
               // NOTE:
-              // ELASTICSEARCH_URL needs to be crated here with auth because SIEM cypress setup depends on it. At some
+              // ELASTICSEARCH_URL needs to be created here with auth because SIEM cypress setup depends on it. At some
               // points we should probably try to refactor that code to use `ELASTICSEARCH_URL_WITH_AUTH` instead
               ELASTICSEARCH_URL:
                 ftrEnv.ELASTICSEARCH_URL ?? createUrlFromFtrConfig('elasticsearch', true),
@@ -376,6 +405,8 @@ ${JSON.stringify(config.getAll(), null, 2)}
               KIBANA_URL_WITH_AUTH: createUrlFromFtrConfig('kibana', true),
               KIBANA_USERNAME: config.get('servers.kibana.username'),
               KIBANA_PASSWORD: config.get('servers.kibana.password'),
+
+              ...argv.env,
             };
 
             log.info(`
@@ -406,7 +437,7 @@ ${JSON.stringify(cyCustomEnv, null, 2)}
                   configFile: cypressConfigFilePath,
                   reporter: argv.reporter as string,
                   reporterOptions: argv.reporterOptions,
-                  headed: true,
+                  headed: argv.headed as boolean,
                   config: {
                     e2e: {
                       baseUrl,

x-pack/plugins/security_solution/cypress/support/e2e.js

@@ -5,29 +5,13 @@
  * 2.0.
  */
 
-// ***********************************************************
-// This example support/index.js is processed and
-// loaded automatically before your test files.
-//
-// This is a great place to put global configuration and
-// behavior that modifies Cypress.
-//
-// You can change the location of this file or turn off
-// automatically serving support files with the
-// 'supportFile' configuration option.
-//
-// You can read more here:
-// https://on.cypress.io/configuration
-// ***********************************************************
-
-// Import commands.js using ES2015 syntax:
 import './commands';
 import 'cypress-real-events/support';
 import 'cypress-pipe';
+import registerCypressGrep from '@cypress/grep';
+
+registerCypressGrep();
 
 Cypress.on('uncaught:exception', () => {
   return false;
 });
-
-// Alternatively you can use CommonJS syntax:
-// require('./commands')

x-pack/plugins/security_solution/cypress/support/index.d.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+declare module 'find-cypress-specs';
 declare namespace Cypress {
   interface Chainable<Subject> {
     promisify(): Promise<Subject>;

x-pack/plugins/security_solution/package.json

@@ -8,6 +8,8 @@
     "extract-mitre-attacks": "node scripts/extract_tactics_techniques_mitre.js && node ../../../scripts/eslint ./public/detections/mitre/mitre_tactics_techniques.ts --fix",
     "build-beat-doc": "node scripts/beat_docs/build.js && node ../../../scripts/eslint ./server/utils/beat_schema/fields.ts --fix",
     "cypress": "../../../node_modules/.bin/cypress",
+    "cypress:burn": "yarn cypress:run:reporter --env burn=2 --concurrency=1 --headed",
+    "cypress:changed-specs-only": "yarn cypress:run:reporter --changed-specs-only --env burn=2",
     "cypress:open": "TZ=UTC node ./scripts/start_cypress_parallel open --spec './cypress/e2e/**/*.cy.ts' --config-file ./cypress/cypress.config.ts --ftr-config-file ../../../../../../../x-pack/test/security_solution_cypress/cli_config",
     "cypress:run": "yarn cypress:run:reporter --spec './cypress/e2e/**/*.cy.ts'; status=$?; yarn junit:merge && exit $status",
     "cypress:run:reporter": "TZ=UTC node ./scripts/start_cypress_parallel run --config-file ./cypress/cypress_ci.config.ts --ftr-config-file ../../../../../../../x-pack/test/security_solution_cypress/cli_config --reporter ../../../node_modules/cypress-multi-reporters --reporter-options configFile=./cypress/reporter_config.json",