Merge branch 'main' into security/bugfix/158110-exception-stuck-loading

.buildkite/ftr_configs.yml

@@ -38,13 +38,11 @@ disabled:
   - x-pack/test/security_solution_cypress/cases_cli_config.ts
   - x-pack/test/security_solution_cypress/ccs_config.ts
   - x-pack/test/security_solution_cypress/cli_config.ts
-  - x-pack/test/security_solution_cypress/cli_config_parallel.ts
   - x-pack/test/security_solution_cypress/cli_config_investigations_parallel.ts
   - x-pack/test/security_solution_cypress/config.firefox.ts
   - x-pack/test/security_solution_cypress/config.ts
   - x-pack/test/security_solution_cypress/response_ops_cli_config.ts
   - x-pack/test/security_solution_cypress/upgrade_config.ts
-  - x-pack/test/security_solution_cypress/visual_config.ts
   - x-pack/test/threat_intelligence_cypress/visual_config.ts
   - x-pack/test/threat_intelligence_cypress/cli_config_parallel.ts
   - x-pack/test/threat_intelligence_cypress/config.ts

.buildkite/pipelines/pull_request/defend_workflows.yml

@@ -5,6 +5,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/response_ops.yml

@@ -5,6 +5,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/security_solution.yml

@@ -5,7 +5,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
-    parallelism: 4
+    parallelism: 15
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/scripts/steps/functional/defend_workflows.sh

@@ -2,16 +2,15 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
-
-.buildkite/scripts/bootstrap.sh
-node scripts/build_kibana_platform_plugins.js
+source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-defend-workflows-cypress
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-echo "--- Defend Workflows Cypress tests"
+Xvfb -screen 0 1680x946x24 :99 &
 
-node scripts/functional_tests \
-  --debug --bail \
-  --config x-pack/test/defend_workflows_cypress/cli_config.ts
+export DISPLAY=:99
+
+echo "--- Defend Workflows Cypress tests"
 
+yarn --cwd x-pack/plugins/security_solution cypress:dw:run

.buildkite/scripts/steps/functional/response_ops.sh

@@ -5,10 +5,12 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+Xvfb -screen 0 1680x946x24 :99 &
+
+export DISPLAY=:99
 
 echo "--- Response Ops Cypress Tests on Security Solution"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  --config x-pack/test/security_solution_cypress/response_ops_cli_config.ts
+yarn --cwd x-pack/plugins/security_solution cypress:run:respops

.buildkite/scripts/steps/functional/response_ops_cases.sh

@@ -5,10 +5,12 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+Xvfb -screen 0 1680x946x24 :99 &
+
+export DISPLAY=:99
 
 echo "--- Response Ops Cases Cypress Tests on Security Solution"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  --config x-pack/test/security_solution_cypress/cases_cli_config.ts
+yarn --cwd x-pack/plugins/security_solution cypress:run:cases

.buildkite/scripts/steps/functional/security_solution.sh

@@ -5,12 +5,12 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
-export CLI_NUMBER=${CLI_NUMBER:-$((BUILDKITE_PARALLEL_JOB+1))}
-export CLI_COUNT=${CLI_COUNT:-$BUILDKITE_PARALLEL_JOB_COUNT}
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+Xvfb :99 -screen 0 1600x1200x24 &
+
+export DISPLAY=:99
 
 echo "--- Security Solution tests (Chrome)"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  --config x-pack/test/security_solution_cypress/cli_config_parallel.ts
+yarn --cwd x-pack/plugins/security_solution cypress:run

.buildkite/scripts/steps/functional/security_solution_investigations.sh

@@ -5,12 +5,13 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
-export CLI_NUMBER=${CLI_NUMBER:-$((BUILDKITE_PARALLEL_JOB+1))}
-export CLI_COUNT=${CLI_COUNT:-$BUILDKITE_PARALLEL_JOB_COUNT}
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-echo "--- Security Solution tests (Chrome)"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  --config x-pack/test/security_solution_cypress/cli_config_investigations_parallel.ts
+Xvfb :99 -screen 0 1600x1200x24 &
+
+export DISPLAY=:99
+
+echo "--- Investigations Cypress Tests on Security Solution"
+
+yarn --cwd x-pack/plugins/security_solution cypress:investigations:run

docs/CHANGELOG.asciidoc

@@ -478,8 +478,6 @@ Uptime::
 [[release-notes-8.7.1]]
 == {kib} 8.7.1
 
-coming::[8.7.1]
-
 Review the following information about the {kib} 8.7.1 release.
 
 [float]
@@ -1089,6 +1087,18 @@ For each {kibana-ref}/xpack-spaces.html[space], complete the following to change
 . Enter `100`, then click *Save changes*.
 ====
 
+[discrete]
+[[breaking-158338]]
+.CSV reports use PIT instead of Scroll
+[%collapsible]
+====
+*Details* +
+CSV reports now use PIT instead of Scroll. Previously generated CSV reports that used an index alias with alias-only privileges, but without privileges on the alias referenced-indices will no longer generate. For more information, refer to {kibana-pull}158338[#158338].
+
+*Impact* +
+To generate CSV reports, grant `read` privileges to the underlying indices.
+====
+
 To review the breaking changes in previous versions, refer to the following:
 
 {kibana-ref-all}/8.5/release-notes-8.5.0.html#breaking-changes-8.5.0[8.5.0] | {kibana-ref-all}/8.4/release-notes-8.4.0.html#breaking-changes-8.4.0[8.4.0] | {kibana-ref-all}/8.3/release-notes-8.3.0.html#breaking-changes-8.3.0[8.3.0] | {kibana-ref-all}/8.2/release-notes-8.2.0.html#breaking-changes-8.2.0[8.2.0] | {kibana-ref-all}/8.1/release-notes-8.1.0.html#breaking-changes-8.1.0[8.1.0] | {kibana-ref-all}/8.0/release-notes-8.0.0.html#breaking-changes-8.0.0[8.0.0] | {kibana-ref-all}/8.0/release-notes-8.0.0-rc2.html#breaking-changes-8.0.0-rc2[8.0.0-rc2] | {kibana-ref-all}/8.0/release-notes-8.0.0-rc1.html#breaking-changes-8.0.0-rc1[8.0.0-rc1] | {kibana-ref-all}/8.0/release-notes-8.0.0-beta1.html#breaking-changes-8.0.0-beta1[8.0.0-beta1] | {kibana-ref-all}/8.0/release-notes-8.0.0-alpha2.html#breaking-changes-8.0.0-alpha2[8.0.0-alpha2] | {kibana-ref-all}/8.0/release-notes-8.0.0-alpha1.html#breaking-changes-8.0.0-alpha1[8.0.0-alpha1]

package.json

@@ -96,7 +96,7 @@
     "@elastic/apm-rum-react": "^1.4.2",
     "@elastic/charts": "55.0.0",
     "@elastic/datemath": "5.0.3",
-    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@8.6.0-canary.3",
+    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@8.8.0-canary.2",
     "@elastic/ems-client": "8.4.0",
     "@elastic/eui": "80.0.0",
     "@elastic/filesaver": "1.1.2",
@@ -1000,6 +1000,7 @@
     "@bazel/ibazel": "^0.16.2",
     "@bazel/typescript": "4.6.2",
     "@cypress/code-coverage": "^3.10.0",
+    "@cypress/grep": "^3.1.5",
     "@cypress/snapshot": "^2.1.7",
     "@cypress/webpack-preprocessor": "^5.12.2",
     "@elastic/eslint-plugin-eui": "0.0.2",
@@ -1349,6 +1350,7 @@
     "chance": "1.0.18",
     "chromedriver": "^113.0.0",
     "clean-webpack-plugin": "^3.0.0",
+    "cli-table3": "^0.6.1",
     "compression-webpack-plugin": "^4.0.0",
     "copy-webpack-plugin": "^6.0.2",
     "cpy": "^8.1.1",
@@ -1360,10 +1362,9 @@
     "cypress-axe": "^1.4.0",
     "cypress-file-upload": "^5.0.8",
     "cypress-multi-reporters": "^1.6.3",
-    "cypress-pipe": "^2.0.0",
     "cypress-react-selector": "^3.0.0",
-    "cypress-real-events": "^1.7.6",
-    "cypress-recurse": "^1.31.2",
+    "cypress-real-events": "^1.8.1",
+    "cypress-recurse": "^1.35.1",
     "date-fns": "^2.29.3",
     "debug": "^2.6.9",
     "delete-empty": "^2.0.0",
@@ -1439,8 +1440,8 @@
     "minimist": "^1.2.6",
     "mocha": "^10.1.0",
     "mocha-junit-reporter": "^2.0.2",
-    "mochawesome": "^7.0.1",
-    "mochawesome-merge": "^4.2.1",
+    "mochawesome": "^7.1.3",
+    "mochawesome-merge": "^4.3.0",
     "mock-fs": "^5.1.2",
     "ms-chromium-edge-driver": "^0.5.1",
     "multistream": "^4.1.0",

packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts

@@ -9,13 +9,18 @@
 import { valid } from 'semver';
 import { schema, TypeOf } from '@kbn/config-schema';
 import type { ServiceConfigDescriptor } from '@kbn/core-base-server-internal';
+import buffer from 'buffer';
 
 const migrationSchema = schema.object({
   algorithm: schema.oneOf([schema.literal('v2'), schema.literal('zdt')], {
     defaultValue: 'v2',
   }),
   batchSize: schema.number({ defaultValue: 1_000 }),
   maxBatchSizeBytes: schema.byteSize({ defaultValue: '100mb' }), // 100mb is the default http.max_content_length Elasticsearch config value
+  maxReadBatchSizeBytes: schema.byteSize({
+    defaultValue: buffer.constants.MAX_STRING_LENGTH,
+    max: buffer.constants.MAX_STRING_LENGTH,
+  }),
   discardUnknownObjects: schema.maybe(
     schema.string({
       validate: (value: string) =>

packages/core/saved-objects/core-saved-objects-migration-server-internal/index.ts

@@ -47,6 +47,7 @@ export type {
   ReindexResponse,
   UpdateByQueryResponse,
   UpdateAndPickupMappingsResponse,
+  EsResponseTooLargeError,
 } from './src/actions';
 export {
   isClusterShardLimitExceeded,

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts

@@ -72,7 +72,7 @@ describe('catchRetryableEsClientErrors', () => {
       });
     });
     it('ResponseError with retryable status code', async () => {
-      const statusCodes = [503, 401, 403, 408, 410];
+      const statusCodes = [503, 401, 403, 408, 410, 429];
       return Promise.all(
         statusCodes.map(async (status) => {
           const error = new esErrors.ResponseError(

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/catch_retryable_es_client_errors.ts

@@ -15,6 +15,7 @@ const retryResponseStatuses = [
   403, // AuthenticationException
   408, // RequestTimeout
   410, // Gone
+  429, // TooManyRequests -> ES circuit breaker
 ];
 
 export interface RetryableEsClientError {

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts

@@ -146,6 +146,11 @@ export interface RequestEntityTooLargeException {
   type: 'request_entity_too_large_exception';
 }
 
+export interface EsResponseTooLargeError {
+  type: 'es_response_too_large';
+  contentLength: number;
+}
+
 /** @internal */
 export interface AcknowledgeResponse {
   acknowledged: boolean;
@@ -168,6 +173,7 @@ export interface ActionErrorTypeMap {
   index_not_green_timeout: IndexNotGreenTimeout;
   index_not_yellow_timeout: IndexNotYellowTimeout;
   cluster_shard_limit_exceeded: ClusterShardLimitExceeded;
+  es_response_too_large: EsResponseTooLargeError;
 }
 
 /**