Merge branch 'main' into exception-xcontent-depth

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.17", "8.11.5", "8.12.0", "8.13.0"]
+            BWC_VERSION: ["7.17.17", "8.12.1", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1793,8 +1793,8 @@ steps:
         env:
           BWC_VERSION: 8.11.4
 
-      - label: "{{matrix.image}} / 8.11.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.5
+      - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -1807,10 +1807,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.11.5
+          BWC_VERSION: 8.12.0
 
-      - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
+      - label: "{{matrix.image}} / 8.12.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -1823,7 +1823,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.12.0
+          BWC_VERSION: 8.12.1
 
       - label: "{{matrix.image}} / 8.13.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.0

.buildkite/pipelines/periodic.yml

@@ -1102,26 +1102,26 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.11.4
-      - label: 8.11.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.5#bwcTest
+      - label: 8.12.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.11.5
-      - label: 8.12.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
+          BWC_VERSION: 8.12.0
+      - label: 8.12.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.12.0
+          BWC_VERSION: 8.12.1
       - label: 8.13.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.0#bwcTest
         timeout_in_minutes: 300

.ci/bwcVersions

@@ -109,6 +109,6 @@ BWC_VERSION:
   - "8.11.2"
   - "8.11.3"
   - "8.11.4"
-  - "8.11.5"
   - "8.12.0"
+  - "8.12.1"
   - "8.13.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,4 @@
 BWC_VERSION:
   - "7.17.17"
-  - "8.11.5"
-  - "8.12.0"
+  - "8.12.1"
   - "8.13.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java

@@ -720,7 +720,7 @@ private static long computeBytesRefCheckSum(BytesRefBlock block, int[] traversal
     }
 
     private static long computeDoubleCheckSum(DoubleBlock block, int[] traversalOrder) {
-        double sum = 0;
+        long sum = 0;
 
         for (int position : traversalOrder) {
             if (block.isNull(position)) {
@@ -729,11 +729,12 @@ private static long computeDoubleCheckSum(DoubleBlock block, int[] traversalOrde
             int start = block.getFirstValueIndex(position);
             int end = start + block.getValueCount(position);
             for (int i = start; i < end; i++) {
-                sum += block.getDouble(i);
+                // Use an operation that is not affected by rounding errors. Otherwise, the result may depend on the traversalOrder.
+                sum += (long) block.getDouble(i);
             }
         }
 
-        return (long) sum;
+        return sum;
     }
 
     private static long computeIntCheckSum(IntBlock block, int[] traversalOrder) {

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -166,7 +167,7 @@ public void setUp() throws Exception {
             .build();
 
         Settings settings = Settings.builder().put("node.name", ShardsAvailabilityHealthIndicatorBenchmark.class.getSimpleName()).build();
-        ThreadPool threadPool = new ThreadPool(settings);
+        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
 
         ClusterService clusterService = new ClusterService(
             Settings.EMPTY,

branches.json

@@ -7,9 +7,6 @@
     {
       "branch": "8.12"
     },
-    {
-      "branch": "8.11"
-    },
     {
       "branch": "7.17"
     }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarFuncTest.groovy

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.gradle.internal
 
+import spock.lang.Ignore
+
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
@@ -36,6 +38,11 @@ class SymbolicLinkPreservingTarFuncTest extends AbstractGradleFuncTest {
         final Path linkToRealFolder = archiveSourceRoot.resolve("link-to-real-folder");
         Files.createSymbolicLink(linkToRealFolder, Paths.get("./real-folder"));
 
+        final Path realFolder2 = testProjectDir.getRoot().toPath().resolve("real-folder2")
+        final Path realFolderSub = realFolder2.resolve("sub")
+        Files.createDirectory(realFolder2);
+        Files.createDirectory(realFolderSub);
+
         buildFile << """
 import org.elasticsearch.gradle.internal.SymbolicLinkPreservingTar
 
@@ -56,6 +63,12 @@ tasks.register("buildBZip2Tar", SymbolicLinkPreservingTar) { SymbolicLinkPreserv
   tar.compression = Compression.BZIP2
   tar.preserveFileTimestamps = ${preserverTimestamp}
   from fileTree("archiveRoot")
+
+  into('config') {
+    dirMode 0750
+    fileMode 0660
+    from "real-folder2"
+  }
 }
 """
         when:
@@ -118,14 +131,20 @@ tasks.register("buildTar", SymbolicLinkPreservingTar) { SymbolicLinkPreservingTa
                 if (entry.getName().equals("real-folder/")) {
                     assert entry.isDirectory()
                     realFolderEntry = true
-                } else if (entry.getName().equals("real-folder/file")) {
+                }  else if (entry.getName().equals("real-folder/file")) {
                     assert entry.isFile()
                     fileEntry = true
                 } else if (entry.getName().equals("real-folder/link-to-file")) {
                     assert entry.isSymbolicLink()
                     assert normalized(entry.getLinkName()) == "./file"
                     linkToFileEntry = true
-                } else if (entry.getName().equals("link-in-folder/")) {
+                } else if (entry.getName().equals("config/")) {
+                    assert entry.isDirectory()
+                    assert entry.getMode() == 16877
+                } else if (entry.getName().equals("config/sub/")) {
+                    assert entry.isDirectory()
+                    assert entry.getMode() == 16872
+                }else if (entry.getName().equals("link-in-folder/")) {
                     assert entry.isDirectory()
                     linkInFolderEntry = true
                 } else if (entry.getName().equals("link-in-folder/link-to-file")) {

build-tools-internal/src/main/groovy/elasticsearch.fips.gradle

@@ -79,7 +79,6 @@ if (BuildParams.inFipsJvm) {
           // with no x-pack. Tests having security explicitly enabled/disabled will override this setting
           setting 'xpack.security.enabled', 'false'
           setting 'xpack.security.fips_mode.enabled', 'true'
-          setting 'xpack.security.fips_mode.required_providers', '["BCFIPS", "BCJSSE"]'
           setting 'xpack.license.self_generated.type', 'trial'
           setting 'xpack.security.authc.password_hashing.algorithm', 'pbkdf2_stretch'
           keystorePassword 'keystore-password'

build-tools-internal/src/main/resources/fips_java.policy

@@ -18,3 +18,8 @@ grant {
      permission org.bouncycastle.crypto.CryptoServicesPermission "exportPrivateKey";
      permission java.io.FilePermission "${javax.net.ssl.trustStore}", "read";
 };
+
+// rely on the caller's socket permissions, the JSSE TLS implementation here is always allowed to connect
+grant codeBase "file:${jdk.module.path}/bctls-fips-1.0.17.jar" {
+  permission java.net.SocketPermission "*", "connect";
+};

build-tools-internal/version.properties

@@ -2,7 +2,7 @@ elasticsearch     = 8.13.0
 lucene            = 9.9.1
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 21.0.1+12@415e3f918a1f4062a0074a2794853d0d
+bundled_jdk = 21.0.2+13@f2283984656d49d69e91c558476027ac
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -201,7 +201,7 @@ public void beforeStart() {
                     try {
                         mockServer.start();
                         node.setting("telemetry.metrics.enabled", "true");
-                        node.setting("tracing.apm.agent.enabled", "true");
+                        node.setting("tracing.apm.enabled", "true");
                         node.setting("tracing.apm.agent.transaction_sample_rate", "0.10");
                         node.setting("tracing.apm.agent.metrics_interval", "10s");
                         node.setting("tracing.apm.agent.server_url", "http://127.0.0.1:" + mockServer.getPort());
@@ -213,8 +213,8 @@ public void beforeStart() {
                 // if metrics were not enabled explicitly for gradlew run we should disable them
                 else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) { // metrics
                     node.setting("telemetry.metrics.enabled", "false");
-                } else if (node.getSettingKeys().contains("tracing.apm.agent.enabled") == false) { // tracing
-                    node.setting("tracing.apm.agent.enable", "false");
+                } else if (node.getSettingKeys().contains("tracing.apm.enabled") == false) { // tracing
+                    node.setting("tracing.apm.enable", "false");
                 }
 
             }

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java

@@ -54,6 +54,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         String defaultRouting = request.param("routing");
         String defaultPipeline = request.param("pipeline");
         Boolean defaultRequireAlias = request.paramAsBoolean("require_alias", null);
+        Boolean defaultRequireDataStream = request.paramAsBoolean("require_data_stream", null);
         Boolean defaultListExecutedPipelines = request.paramAsBoolean("list_executed_pipelines", null);
 
         String waitForActiveShards = request.param("wait_for_active_shards");
@@ -69,6 +70,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             null,
             defaultPipeline,
             defaultRequireAlias,
+            defaultRequireDataStream,
             defaultListExecutedPipelines,
             true,
             request.getXContentType(),

client/rest/src/main/java/org/elasticsearch/client/Request.java

@@ -67,7 +67,7 @@ public String getEndpoint() {
     /**
      * Add a query string parameter.
      * @param name the name of the url parameter. Must not be null.
-     * @param value the value of the url url parameter. If {@code null} then
+     * @param value the value of the url parameter. If {@code null} then
      *      the parameter is sent as {@code name} rather than {@code name=value}
      * @throws IllegalArgumentException if a parameter with that name has
      *      already been set

client/test/build.gradle

@@ -27,9 +27,9 @@ dependencies {
   api "org.hamcrest:hamcrest:${versions.hamcrest}"
 
   // mockito
-  api 'org.mockito:mockito-core:5.4.0'
-  api 'org.mockito:mockito-subclass:5.4.0'
-  api 'net.bytebuddy:byte-buddy:1.14.5'
+  api 'org.mockito:mockito-core:5.9.0'
+  api 'org.mockito:mockito-subclass:5.9.0'
+  api 'net.bytebuddy:byte-buddy:1.14.11'
   api 'org.objenesis:objenesis:3.3'
 }
 

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java

@@ -79,7 +79,8 @@ class APMJvmOptions {
         "application_packages", "org.elasticsearch,org.apache.lucene",
         "metrics_interval", "120s",
         "breakdown_metrics", "false",
-        "central_config", "false"
+        "central_config", "false",
+        "transaction_sample_rate", "0.2"
         );
     // end::noformat