Merge branch 'main' into 130713-add-labels-to-agent-listing-ui

.buildkite/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "elastic/kibana-buildkite-library"
+    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#0c02e8e8461f3bd5b6d686efe401d579d622395d"
   }
 }

.buildkite/pipelines/pull_request/base.yml

@@ -123,29 +123,14 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/jest.sh
-    label: 'Jest Tests'
-    parallelism: 8
+  - command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
+    label: 'Pick Jest Config Run Order'
     agents:
-      queue: n2-4-spot
-    timeout_in_minutes: 90
-    key: jest
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - command: .buildkite/scripts/steps/test/jest_integration.sh
-    label: 'Jest Integration Tests'
-    parallelism: 3
-    agents:
-      queue: n2-4-spot
-    timeout_in_minutes: 120
-    key: jest-integration
+      queue: kibana-default
     retry:
       automatic:
-        - exit_status: '-1'
-          limit: 3
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'

.buildkite/scripts/lifecycle/pre_command.sh

@@ -9,21 +9,7 @@ export BUILDKITE_TOKEN
 
 echo '--- Install buildkite dependencies'
 cd '.buildkite'
-
-# If this yarn install is terminated early, e.g. if the build is cancelled in buildkite,
-# A node module could end up in a bad state that can cause all future builds to fail
-# So, let's cache clean and try again to make sure that's not what caused the error
-install_deps() {
-  yarn install --production --pure-lockfile
-  EXIT=$?
-  if [[ "$EXIT" != "0" ]]; then
-    yarn cache clean
-  fi
-  return $EXIT
-}
-
-retry 5 15 install_deps
-
+retry 5 15 npm ci
 cd ..
 
 echo '--- Agent Debug/SSH Info'

.buildkite/scripts/steps/test/jest_env.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# keys used to associate test group data in ci-stats with Jest execution order
+export TEST_GROUP_TYPE_UNIT="Jest Unit Tests"
+export TEST_GROUP_TYPE_INTEGRATION="Jest Integration Tests"

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -2,36 +2,38 @@
 
 set -uo pipefail
 
-JOB=$BUILDKITE_PARALLEL_JOB
-JOB_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT
+source .buildkite/scripts/steps/test/jest_env.sh
 
-# a jest failure will result in the script returning an exit code of 10
+export JOB=$BUILDKITE_PARALLEL_JOB
 
-i=0
+# a jest failure will result in the script returning an exit code of 10
 exitCode=0
 
-# run unit tests in parallel
 if [[ "$1" == 'jest.config.js' ]]; then
+  # run unit tests in parallel
   parallelism="-w2"
+  TEST_TYPE="unit"
 else
+  # run integration tests in-band
   parallelism="--runInBand"
+  TEST_TYPE="integration"
 fi
 
+export TEST_TYPE
+echo "--- downloading integration test run order"
+buildkite-agent artifact download jest_run_order.json .
+configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)
+
 while read -r config; do
-  if [ "$((i % JOB_COUNT))" -eq "$JOB" ]; then
-    echo "--- $ node scripts/jest --config $config"
-    node --max-old-space-size=14336 ./scripts/jest --config="$config" "$parallelism" --coverage=false --passWithNoTests
-    lastCode=$?
-
-    if [ $lastCode -ne 0 ]; then
-      exitCode=10
-      echo "Jest exited with code $lastCode"
-      echo "^^^ +++"
-    fi
+  echo "--- $ node scripts/jest --config $config"
+  NODE_OPTIONS="--max-old-space-size=14336" node ./scripts/jest --config="$config" "$parallelism" --coverage=false --passWithNoTests
+  lastCode=$?
+
+  if [ $lastCode -ne 0 ]; then
+    exitCode=10
+    echo "Jest exited with code $lastCode"
+    echo "^^^ +++"
   fi
-
-  ((i=i+1))
-# uses heredoc to avoid the while loop being in a sub-shell thus unable to overwrite exitCode
-done <<< "$(find src x-pack packages -name "$1" -not -path "*/__fixtures__/*" | sort)"
+done <<< "$configs"
 
 exit $exitCode

.buildkite/scripts/steps/test/pick_jest_config_run_order.js

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const { CiStats } = require('kibana-buildkite-library');
+
+(async () => {
+  try {
+    await CiStats.pickTestGroupRunOrder();
+  } catch (ex) {
+    console.error('CI Stats Error', ex.message);
+    if (ex.response) {
+      console.error('HTTP Error Response Status', ex.response.status);
+      console.error('HTTP Error Response Body', ex.response.data);
+    }
+    process.exit(1);
+  }
+})();

.buildkite/scripts/steps/test/pick_jest_config_run_order.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/test/jest_env.sh
+
+echo '--- Pick Jest Config Run Order'
+node "$(dirname "${0}")/pick_jest_config_run_order.js"

.gitignore

@@ -57,6 +57,7 @@ selenium
 *.swo
 *.out
 package-lock.json
+!/.buildkite/package-lock.json
 .yo-rc.json
 .vscode
 *.sublime-*

docs/api/machine-learning/sync.asciidoc

@@ -47,9 +47,9 @@ it is added. This list contains the {dfeed} identifiers and indicates whether
 the synchronization was successful.
 
 `datafeedsRemoved`::
-(array) If saved objects exist for {dfeeds} that no longer exist, they are
-deleted. This list contains the {dfeed} identifiers and indicates whether the
-synchronization was successful.
+(array) If a saved object for an anomaly detection job references a datafeed
+that no longer exists, it is deleted. This list contains the {dfeed} identifiers 
+and indicates whether the synchronization was successful.
 
 `savedObjectsCreated`::
 (array) If saved objects are missing for {ml} jobs or trained models, they are

docs/settings/general-infra-logs-ui-settings.asciidoc

@@ -1,28 +1,7 @@
 
-`xpack.infra.sources.default.logAlias`::
-Index pattern for matching indices that contain log data. Defaults to `filebeat-*,kibana_sample_data_logs*`. To match multiple wildcard patterns, use a comma to separate the names, with no space after the comma. For example, `logstash-app1-*,default-logs-*`.
-
-`xpack.infra.sources.default.metricAlias`::
-Index pattern for matching indices that contain Metricbeat data. Defaults to `metricbeat-*`. To match multiple wildcard patterns, use a comma to separate the names, with no space after the comma. For example, `logstash-app1-*,default-logs-*`.
-
-`xpack.infra.sources.default.fields.timestamp`::
-Timestamp used to sort log entries. Defaults to `@timestamp`.
-
 `xpack.infra.sources.default.fields.message`::
 Fields used to display messages in the Logs app. Defaults to `['message', '@message']`.
 
-`xpack.infra.sources.default.fields.tiebreaker`::
-Field used to break ties between two entries with the same timestamp. Defaults to `_doc`.
-
-`xpack.infra.sources.default.fields.host`::
-Field used to identify hosts. Defaults to `host.name`.
-
-`xpack.infra.sources.default.fields.container`::
-Field used to identify Docker containers. Defaults to `container.id`.
-
-`xpack.infra.sources.default.fields.pod`::
-Field used to identify Kubernetes pods. Defaults to `kubernetes.pod.uid`.
-
 `xpack.infra.alerting.inventory_threshold.group_by_page_size`::
 Controls the size of the composite aggregations used by the Inventory Threshold to retrieve all the hosts. Defaults to `10_000`.