Jest flakiness testing

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -12,7 +12,7 @@ import * as globby from 'globby';
 import minimatch from 'minimatch';
 import { load as loadYaml } from 'js-yaml';
 
-import { BuildkiteClient, BuildkiteStep } from '../buildkite';
+import { BuildkiteClient } from '../buildkite';
 import { CiStatsClient, TestGroupRunOrderResponse } from './client';
 
 import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
@@ -200,6 +200,12 @@ export async function pickTestGroupRunOrder() {
         .filter(Boolean)
     : undefined;
 
+  const JEST_UNIT_CONFIG_PATTERNS = process.env.JEST_UNIT_CONFIG_PATTERNS
+    ? process.env.JEST_UNIT_CONFIG_PATTERNS.split(',')
+        .map((t) => t.trim())
+        .filter(Boolean)
+    : undefined;
+
   const FUNCTIONAL_MINIMUM_ISOLATION_MIN = process.env.FUNCTIONAL_MINIMUM_ISOLATION_MIN
     ? parseFloat(process.env.FUNCTIONAL_MINIMUM_ISOLATION_MIN)
     : undefined;
@@ -225,25 +231,26 @@ export async function pickTestGroupRunOrder() {
     throw new Error(`invalid JEST_CONFIGS_RETRY_COUNT: ${process.env.JEST_CONFIGS_RETRY_COUNT}`);
   }
 
-  const FTR_CONFIGS_DEPS =
-    process.env.FTR_CONFIGS_DEPS !== undefined
-      ? process.env.FTR_CONFIGS_DEPS.split(',')
-          .map((t) => t.trim())
-          .filter(Boolean)
-      : ['build'];
+  // const FTR_CONFIGS_DEPS =
+  //   process.env.FTR_CONFIGS_DEPS !== undefined
+  //     ? process.env.FTR_CONFIGS_DEPS.split(',')
+  //         .map((t) => t.trim())
+  //         .filter(Boolean)
+  //     : ['build'];
 
-  const FTR_EXTRA_ARGS: Record<string, string> = process.env.FTR_EXTRA_ARGS
-    ? { FTR_EXTRA_ARGS: process.env.FTR_EXTRA_ARGS }
-    : {};
+  // const FTR_EXTRA_ARGS: Record<string, string> = process.env.FTR_EXTRA_ARGS
+  //   ? { FTR_EXTRA_ARGS: process.env.FTR_EXTRA_ARGS }
+  //   : {};
 
   const { defaultQueue, ftrConfigsByQueue } = getEnabledFtrConfigs(FTR_CONFIG_PATTERNS);
 
   const ftrConfigsIncluded = LIMIT_CONFIG_TYPE.includes('functional');
 
   if (!ftrConfigsIncluded) ftrConfigsByQueue.clear();
 
+  const jestUnitPattern = JEST_UNIT_CONFIG_PATTERNS || '**';
   const jestUnitConfigs = LIMIT_CONFIG_TYPE.includes('unit')
-    ? globby.sync(['**/jest.config.js', '!**/__fixtures__/**'], {
+    ? globby.sync([`${jestUnitPattern}/jest.config.js`, '!**/__fixtures__/**'], {
         cwd: process.cwd(),
         absolute: false,
         ignore: DISABLED_JEST_CONFIGS,
@@ -415,7 +422,7 @@ export async function pickTestGroupRunOrder() {
         ? {
             label: 'Jest Tests',
             command: getRequiredEnv('JEST_UNIT_SCRIPT'),
-            parallelism: unit.count,
+            parallelism: 25,
             timeout_in_minutes: 120,
             key: 'jest',
             agents: {
@@ -431,68 +438,68 @@ export async function pickTestGroupRunOrder() {
             },
           }
         : [],
-      integration.count > 0
-        ? {
-            label: 'Jest Integration Tests',
-            command: getRequiredEnv('JEST_INTEGRATION_SCRIPT'),
-            parallelism: integration.count,
-            timeout_in_minutes: 120,
-            key: 'jest-integration',
-            agents: {
-              queue: 'n2-4-spot',
-            },
-            retry: {
-              automatic: [
-                { exit_status: '-1', limit: 3 },
-                ...(JEST_CONFIGS_RETRY_COUNT > 0
-                  ? [{ exit_status: '*', limit: JEST_CONFIGS_RETRY_COUNT }]
-                  : []),
-              ],
-            },
-          }
-        : [],
-      functionalGroups.length
-        ? {
-            group: 'FTR Configs',
-            key: 'ftr-configs',
-            depends_on: FTR_CONFIGS_DEPS,
-            steps: functionalGroups
-              .sort((a, b) =>
-                // if both groups are sorted by number then sort by that
-                typeof a.sortBy === 'number' && typeof b.sortBy === 'number'
-                  ? a.sortBy - b.sortBy
-                  : // if both groups are sorted by string, sort by that
-                  typeof a.sortBy === 'string' && typeof b.sortBy === 'string'
-                  ? a.sortBy.localeCompare(b.sortBy)
-                  : // if a is sorted by number then order it later than b
-                  typeof a.sortBy === 'number'
-                  ? 1
-                  : -1
-              )
-              .map(
-                ({ title, key, queue = defaultQueue }): BuildkiteStep => ({
-                  label: title,
-                  command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
-                  timeout_in_minutes: 90,
-                  agents: {
-                    queue,
-                  },
-                  env: {
-                    FTR_CONFIG_GROUP_KEY: key,
-                    ...FTR_EXTRA_ARGS,
-                  },
-                  retry: {
-                    automatic: [
-                      { exit_status: '-1', limit: 3 },
-                      ...(FTR_CONFIGS_RETRY_COUNT > 0
-                        ? [{ exit_status: '*', limit: FTR_CONFIGS_RETRY_COUNT }]
-                        : []),
-                    ],
-                  },
-                })
-              ),
-          }
-        : [],
+      // integration.count > 0
+      //   ? {
+      //       label: 'Jest Integration Tests',
+      //       command: getRequiredEnv('JEST_INTEGRATION_SCRIPT'),
+      //       parallelism: integration.count,
+      //       timeout_in_minutes: 120,
+      //       key: 'jest-integration',
+      //       agents: {
+      //         queue: 'n2-4-spot',
+      //       },
+      //       retry: {
+      //         automatic: [
+      //           { exit_status: '-1', limit: 3 },
+      //           ...(JEST_CONFIGS_RETRY_COUNT > 0
+      //             ? [{ exit_status: '*', limit: JEST_CONFIGS_RETRY_COUNT }]
+      //             : []),
+      //         ],
+      //       },
+      //     }
+      //   : [],
+      // functionalGroups.length
+      //   ? {
+      //       group: 'FTR Configs',
+      //       key: 'ftr-configs',
+      //       depends_on: FTR_CONFIGS_DEPS,
+      //       steps: functionalGroups
+      //         .sort((a, b) =>
+      //           // if both groups are sorted by number then sort by that
+      //           typeof a.sortBy === 'number' && typeof b.sortBy === 'number'
+      //             ? a.sortBy - b.sortBy
+      //             : // if both groups are sorted by string, sort by that
+      //             typeof a.sortBy === 'string' && typeof b.sortBy === 'string'
+      //             ? a.sortBy.localeCompare(b.sortBy)
+      //             : // if a is sorted by number then order it later than b
+      //             typeof a.sortBy === 'number'
+      //             ? 1
+      //             : -1
+      //         )
+      //         .map(
+      //           ({ title, key, queue = defaultQueue }): BuildkiteStep => ({
+      //             label: title,
+      //             command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
+      //             timeout_in_minutes: 90,
+      //             agents: {
+      //               queue,
+      //             },
+      //             env: {
+      //               FTR_CONFIG_GROUP_KEY: key,
+      //               ...FTR_EXTRA_ARGS,
+      //             },
+      //             retry: {
+      //               automatic: [
+      //                 { exit_status: '-1', limit: 3 },
+      //                 ...(FTR_CONFIGS_RETRY_COUNT > 0
+      //                   ? [{ exit_status: '*', limit: FTR_CONFIGS_RETRY_COUNT }]
+      //                   : []),
+      //               ],
+      //             },
+      //           })
+      //         ),
+      //     }
+      //   : [],
     ].flat()
   );
 }

.buildkite/pipelines/pull_request/base.yml

@@ -7,30 +7,30 @@ steps:
 
   - wait
 
-  - command: .buildkite/scripts/steps/build_kibana.sh
-    label: Build Kibana Distribution and Plugins
-    agents:
-      queue: n2-16-spot
-    key: build
-    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
+  # - command: .buildkite/scripts/steps/build_kibana.sh
+  #   label: Build Kibana Distribution and Plugins
+  #   agents:
+  #     queue: n2-16-spot
+  #   key: build
+  #   if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+  #   timeout_in_minutes: 60
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
 
-  - command: .buildkite/scripts/steps/quick_checks.sh
-    label: 'Quick Checks'
-    agents:
-      queue: n2-2-spot
-    key: quick_checks
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
+  # - command: .buildkite/scripts/steps/quick_checks.sh
+  #   label: 'Quick Checks'
+  #   agents:
+  #     queue: n2-2-spot
+  #   key: quick_checks
+  #   timeout_in_minutes: 60
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
 
-  - wait
+  # - wait
 
   - command: .buildkite/scripts/steps/ci_stats_ready.sh
     label: Mark CI Stats as ready
@@ -52,52 +52,53 @@ steps:
     timeout_in_minutes: 10
     env:
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_UNIT_CONFIG_PATTERNS: 'x-pack/plugins/security_solution/public/common'
       JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
       FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/lint.sh
-    label: 'Linting'
-    agents:
-      queue: n2-8-spot
-    key: linting
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
+  # - command: .buildkite/scripts/steps/lint.sh
+  #   label: 'Linting'
+  #   agents:
+  #     queue: n2-8-spot
+  #   key: linting
+  #   timeout_in_minutes: 60
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
 
-  - command: .buildkite/scripts/steps/lint_with_types.sh
-    label: 'Linting (with types)'
-    agents:
-      queue: n2-16-spot
-    key: linting_with_types
-    timeout_in_minutes: 90
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
+  # - command: .buildkite/scripts/steps/lint_with_types.sh
+  #   label: 'Linting (with types)'
+  #   agents:
+  #     queue: n2-16-spot
+  #   key: linting_with_types
+  #   timeout_in_minutes: 90
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
 
-  - command: .buildkite/scripts/steps/checks.sh
-    label: 'Checks'
-    agents:
-      queue: n2-2-spot
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
+  # - command: .buildkite/scripts/steps/checks.sh
+  #   label: 'Checks'
+  #   agents:
+  #     queue: n2-2-spot
+  #   timeout_in_minutes: 60
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
 
-  - command: .buildkite/scripts/steps/build_api_docs.sh
-    label: 'Check Types and Build API Docs'
-    agents:
-      queue: n2-4-spot
-    key: build_api_docs
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
+  # - command: .buildkite/scripts/steps/build_api_docs.sh
+  #   label: 'Check Types and Build API Docs'
+  #   agents:
+  #     queue: n2-4-spot
+  #   key: build_api_docs
+  #   timeout_in_minutes: 60
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3

.buildkite/scripts/steps/code_coverage/jest_parallel.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 source "$(dirname "$0")/../../common/util.sh"
 source .buildkite/scripts/steps/code_coverage/util.sh
 
-export JOB=$BUILDKITE_PARALLEL_JOB
+export JOB=0
 
 # a jest failure will result in the script returning an exit code of 10
 exitCode=0

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 source "$(dirname "$0")/../../common/util.sh"
-export JOB=${BUILDKITE_PARALLEL_JOB:-0}
+export JOB=0
 
 # a jest failure will result in the script returning an exit code of 10
 exitCode=0

x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx

@@ -13,7 +13,7 @@ import { API_ERROR } from '../../translations';
 jest.mock('@kbn/core-http-browser');
 
 const mockHttp = {
-  get: jest.fn(),
+  fetch: jest.fn(),
 } as unknown as HttpSetup;
 
 describe('Capabilities API tests', () => {
@@ -25,14 +25,15 @@ describe('Capabilities API tests', () => {
     it('calls the internal assistant API for fetching assistant capabilities', async () => {
       await getCapabilities({ http: mockHttp });
 
-      expect(mockHttp.get).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', {
+      expect(mockHttp.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', {
+        method: 'GET',
         signal: undefined,
         version: '1',
       });
     });
 
     it('returns API_ERROR when the response status is error', async () => {
-      (mockHttp.get as jest.Mock).mockResolvedValue({ status: API_ERROR });
+      (mockHttp.fetch as jest.Mock).mockResolvedValue({ status: API_ERROR });
 
       const result = await getCapabilities({ http: mockHttp });