Merge branch 'main' into exception-xcontent-depth

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.17", "8.11.4", "8.12.0", "8.13.0"]
+            BWC_VERSION: ["7.17.17", "8.11.5", "8.12.0", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1793,6 +1793,22 @@ steps:
         env:
           BWC_VERSION: 8.11.4
 
+      - label: "{{matrix.image}} / 8.11.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.5
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.5
+
       - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic-platform-support.yml

@@ -89,6 +89,7 @@ steps:
           setup:
             image:
               - amazonlinux-2023
+              - amazonlinux-2
         agents:
           provider: aws
           imagePrefix: elasticsearch-{{matrix.image}}

.buildkite/pipelines/periodic.template.yml

@@ -2,7 +2,7 @@ steps:
   - group: bwc
     steps: $BWC_STEPS
   - label: concurrent-search-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp
@@ -97,7 +97,7 @@ steps:
       diskSizeGb: 350
       machineType: custom-32-98304
   - label: single-processor-node-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp

.buildkite/pipelines/periodic.yml

@@ -1102,6 +1102,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.11.4
+      - label: 8.11.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.5#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.5
       - label: 8.12.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
         timeout_in_minutes: 300
@@ -1123,7 +1133,7 @@ steps:
         env:
           BWC_VERSION: 8.13.0
   - label: concurrent-search-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp
@@ -1218,7 +1228,7 @@ steps:
       diskSizeGb: 350
       machineType: custom-32-98304
   - label: single-processor-node-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp

.buildkite/scripts/encryption-at-rest.sh

@@ -22,4 +22,4 @@ touch .output.log
 rm -Rf "$WORKSPACE"
 ln -s "$PWD" "$WORKSPACE"
 
-.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
+.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests

.buildkite/scripts/release-tests.sh

@@ -20,4 +20,4 @@ curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/m
 curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT.zip
 
 .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false -Dbuild.ml_cpp.repo=file://${ML_IVY_REPO} \
-  -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef build
+  -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef assemble functionalTests

.ci/bwcVersions

@@ -109,5 +109,6 @@ BWC_VERSION:
   - "8.11.2"
   - "8.11.3"
   - "8.11.4"
+  - "8.11.5"
   - "8.12.0"
   - "8.13.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.17"
-  - "8.11.4"
+  - "8.11.5"
   - "8.12.0"
   - "8.13.0"

build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle

@@ -47,15 +47,6 @@ configure(allprojects) {
             }
         }
     }
-
-    project.plugins.withType(RestTestBasePlugin) {
-      tasks.withType(StandaloneRestIntegTestTask).configureEach {
-        if (BuildParams.getIsRuntimeJavaHomeSet() == false) {
-          nonInputProperties.systemProperty("tests.runtime.java", "${-> launcher.map { it.metadata.installationPath.asFile.path }.get()}")
-        }
-      }
-    }
-
     project.plugins.withType(ThirdPartyAuditPrecommitPlugin) {
       project.getTasks().withType(ThirdPartyAuditTask.class).configureEach {
         if (BuildParams.getIsRuntimeJavaHomeSet() == false) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -63,7 +63,6 @@
 public class RestTestBasePlugin implements Plugin<Project> {
 
     private static final String TESTS_MAX_PARALLEL_FORKS_SYSPROP = "tests.max.parallel.forks";
-    private static final String TESTS_RUNTIME_JAVA_SYSPROP = "tests.runtime.java";
     private static final String DEFAULT_DISTRIBUTION_SYSPROP = "tests.default.distribution";
     private static final String INTEG_TEST_DISTRIBUTION_SYSPROP = "tests.integ-test.distribution";
     private static final String BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX = "tests.snapshot.distribution.";
@@ -189,7 +188,6 @@ public void apply(Project project) {
             // Wire up integ-test distribution by default for all test tasks
             FileCollection extracted = integTestDistro.getExtracted();
             nonInputSystemProperties.systemProperty(INTEG_TEST_DISTRIBUTION_SYSPROP, () -> extracted.getSingleFile().getPath());
-            nonInputSystemProperties.systemProperty(TESTS_RUNTIME_JAVA_SYSPROP, BuildParams.getRuntimeJavaHome());
 
             // Add `usesDefaultDistribution()` extension method to test tasks to indicate they require the default distro
             task.getExtensions().getExtraProperties().set("usesDefaultDistribution", new Closure<Void>(task) {

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -47,6 +48,7 @@ public class NoopPlugin extends Plugin implements ActionPlugin {
     @Override
     public List<RestHandler> getRestHandlers(
         Settings settings,
+        NamedWriteableRegistry namedWriteableRegistry,
         RestController restController,
         ClusterSettings clusterSettings,
         IndexScopedSettings indexScopedSettings,

docs/changelog/100813.yaml

@@ -0,0 +1,6 @@
+pr: 100813
+summary: Make `ParentTaskAssigningClient.getRemoteClusterClient` method also return
+  `ParentTaskAssigningClient`
+area: Infra/Transport API
+type: enhancement
+issues: []

docs/changelog/103949.yaml

@@ -0,0 +1,5 @@
+pr: 103949
+summary: "ESQL: Introduce mode setting for ENRICH"
+area: ES|QL
+type: feature
+issues: []

docs/changelog/103959.yaml

@@ -0,0 +1,5 @@
+pr: 103959
+summary: Add `ApiKey` expiration time to audit log
+area: Security
+type: enhancement
+issues: []

docs/changelog/104026.yaml

@@ -0,0 +1,5 @@
+pr: 104026
+summary: Include user's privileges actions in IdP plugin `_has_privileges` request
+area: IdentityProvider
+type: enhancement
+issues: []

docs/changelog/104043.yaml

@@ -0,0 +1,5 @@
+pr: 104043
+summary: Expose service account authentication metrics
+area: Authentication
+type: enhancement
+issues: []

docs/changelog/104092.yaml

@@ -0,0 +1,5 @@
+pr: 104092
+summary: Ingest geoip processor cache 'no results' from the database
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/104113.yaml

@@ -0,0 +1,5 @@
+pr: 104113
+summary: "X-pack/plugin/apm-data: fix `@custom` pipeline support"
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/104142.yaml

@@ -0,0 +1,5 @@
+pr: 104142
+summary: Expose token authentication metrics
+area: Authentication
+type: enhancement
+issues: []

docs/changelog/104150.yaml

@@ -0,0 +1,5 @@
+pr: 104150
+summary: Correct profiled rewrite time for knn with a pre-filter
+area: Search
+type: bug
+issues: []

docs/changelog/104155.yaml

@@ -0,0 +1,6 @@
+pr: 104155
+summary: "Updated `missingTrainedModel` message to include: you may need to create\
+  \ it"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/104198.yaml

@@ -0,0 +1,5 @@
+pr: 104198
+summary: "[Connector API] Fix bug in configuration validation parser"
+area: Application
+type: bug
+issues: []

docs/changelog/104230.yaml

@@ -0,0 +1,5 @@
+pr: 104230
+summary: Undeploy elser when inference model deleted
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/104259.yaml

@@ -0,0 +1,6 @@
+pr: 104259
+summary: Fix cancellation order in `CancellableRateLimitedFluxIterator`
+area: Snapshot/Restore
+type: bug
+issues:
+ - 103054

docs/reference/ccr/apis/follow/get-follow-stats.asciidoc

@@ -75,6 +75,9 @@ task. In this situation, the following task must be resumed manually with the
 `index`::
 (string) The name of the follower index.
 
+`total_global_checkpoint_lag`::
+(long) Indication of how much the follower is lagging the leader. This is the sum of the difference between the `leader_global_checkpoint` and the `follower_global_checkpoint` for all shards.
+
 //Begin shards
 `shards`::
 (array) An array of shard-level following task statistics.
@@ -219,6 +222,7 @@ The API returns the following results:
   "indices" : [
     {
       "index" : "follower_index",
+      "total_global_checkpoint_lag" : 256,
       "shards" : [
         {
           "remote_cluster" : "remote_cluster",
@@ -255,6 +259,7 @@ The API returns the following results:
   ]
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"total_global_checkpoint_lag" : 256/"total_global_checkpoint_lag" : 0/]
 // TESTRESPONSE[s/"leader_global_checkpoint" : 1024/"leader_global_checkpoint" : $body.indices.0.shards.0.leader_global_checkpoint/]
 // TESTRESPONSE[s/"leader_max_seq_no" : 1536/"leader_max_seq_no" : $body.indices.0.shards.0.leader_max_seq_no/]
 // TESTRESPONSE[s/"follower_global_checkpoint" : 768/"follower_global_checkpoint" : $body.indices.0.shards.0.follower_global_checkpoint/]

docs/reference/ccr/apis/get-ccr-stats.asciidoc

@@ -112,6 +112,7 @@ The API returns the following results:
     "indices" : [
       {
         "index" : "follower_index",
+        "total_global_checkpoint_lag" : 256,
         "shards" : [
           {
             "remote_cluster" : "remote_cluster",
@@ -149,6 +150,7 @@ The API returns the following results:
   }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"total_global_checkpoint_lag" : 256/"total_global_checkpoint_lag" : 0/]
 // TESTRESPONSE[s/"number_of_failed_follow_indices" : 0/"number_of_failed_follow_indices" : $body.auto_follow_stats.number_of_failed_follow_indices/]
 // TESTRESPONSE[s/"number_of_failed_remote_cluster_state_requests" : 0/"number_of_failed_remote_cluster_state_requests" : $body.auto_follow_stats.number_of_failed_remote_cluster_state_requests/]
 // TESTRESPONSE[s/"number_of_successful_follow_indices" : 1/"number_of_successful_follow_indices" : $body.auto_follow_stats.number_of_successful_follow_indices/]

docs/reference/data-streams/tsds-index-settings.asciidoc

@@ -28,7 +28,7 @@ value (exclusive) accepted by the index. Only indices with an `index.mode` of
 `index.look_ahead_time`::
 (<<_static_index_settings,Static>>, <<time-units,time units>>)
 Interval used to calculate the `index.time_series.end_time` for a TSDS's write
-index. Defaults to `2h` (2 hours). Accepts `1m` (one minute) to `2h` (two
+index. Defaults to `30m` (30 minutes). Accepts `1m` (one minute) to `2h` (two
 hours). Only indices with an `index.mode` of `time_series` support this setting.
 For more information, refer to <<tsds-look-ahead-time>>. Additionally this setting
 can not be less than `time_series.poll_interval` cluster setting.