Merge branch 'main' into 2024/05/31/update-complete-marker-on-progress

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.22", "8.13.5", "8.14.0", "8.15.0"]
+            BWC_VERSION: ["7.17.22", "8.14.1", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -529,8 +529,8 @@ steps:
         env:
           BWC_VERSION: 8.12.2
 
-      - label: "{{matrix.image}} / 8.13.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.5
+      - label: "{{matrix.image}} / 8.13.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -543,10 +543,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.5
+          BWC_VERSION: 8.13.4
 
-      - label: "{{matrix.image}} / 8.14.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.0
+      - label: "{{matrix.image}} / 8.14.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -559,7 +559,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.14.0
+          BWC_VERSION: 8.14.1
 
       - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0

.buildkite/pipelines/periodic.yml

@@ -591,8 +591,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.13.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.5#bwcTest
+      - label: 8.13.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -601,7 +601,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.13.5
+          BWC_VERSION: 8.13.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -610,8 +610,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.14.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.0#bwcTest
+      - label: 8.14.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -620,7 +620,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.14.0
+          BWC_VERSION: 8.14.1
         retry:
           automatic:
             - exit_status: "-1"
@@ -714,7 +714,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.22", "8.13.5", "8.14.0", "8.15.0"]
+            BWC_VERSION: ["7.17.22", "8.14.1", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -762,7 +762,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.22", "8.13.5", "8.14.0", "8.15.0"]
+            BWC_VERSION: ["7.17.22", "8.14.1", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -30,6 +30,6 @@ BWC_VERSION:
   - "8.10.4"
   - "8.11.4"
   - "8.12.2"
-  - "8.13.5"
-  - "8.14.0"
+  - "8.13.4"
+  - "8.14.1"
   - "8.15.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,4 @@
 BWC_VERSION:
   - "7.17.22"
-  - "8.13.5"
-  - "8.14.0"
+  - "8.14.1"
   - "8.15.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.benchmark.index.mapper;
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.util.Accountable;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -21,10 +22,12 @@
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.LowercaseNormalizer;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.script.Script;
@@ -52,6 +55,13 @@ public static MapperService create(String mappings) {
         MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
 
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {}
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
+        });
         MapperService mapperService = new MapperService(
             () -> TransportVersion.current(),
             indexSettings,
@@ -73,6 +83,7 @@ public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
             },
+            bitsetFilterCache::getBitSetProducer,
             MapperMetrics.NOOP
         );
 

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -189,6 +189,7 @@ public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
             },
+            query -> { throw new UnsupportedOperationException(); },
             MapperMetrics.NOOP
         );
 

branches.json

@@ -7,9 +7,6 @@
     {
       "branch": "8.14"
     },
-    {
-      "branch": "8.13"
-    },
     {
       "branch": "7.17"
     }

build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle

@@ -33,7 +33,8 @@ tasks.register("bwcTest") {
 
 plugins.withType(ElasticsearchTestBasePlugin) {
   tasks.withType(Test).matching { it.name ==~ /v[0-9\.]+#.*/ }.configureEach {
-    onlyIf("BWC tests enabled") { project.bwc_tests_enabled }
+    boolean bwcEnabled = project.bwc_tests_enabled
+    onlyIf("BWC tests enabled") { bwcEnabled }
     nonInputProperties.systemProperty 'tests.bwc', 'true'
   }
 }
@@ -50,5 +51,5 @@ plugins.withType(InternalJavaRestTestPlugin) {
   }
 }
 
-tasks.matching { it.name.equals("check") }.configureEach {dependsOn(bwcTestSnapshots) }
-tasks.matching { it.name.equals("test") }.configureEach {enabled = false}
+tasks.matching { it.name.equals("check") }.configureEach { dependsOn(bwcTestSnapshots) }
+tasks.matching { it.name.equals("test") }.configureEach { enabled = false }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -20,6 +20,9 @@
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.provider.ValueSource;
+import org.gradle.api.provider.ValueSourceParameters;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainService;
@@ -41,6 +44,7 @@ public class BwcSetupExtension {
     private static final Version BUILD_TOOL_MINIMUM_VERSION = Version.fromString("7.14.0");
     private final Project project;
     private final ObjectFactory objectFactory;
+    private final ProviderFactory providerFactory;
     private final JavaToolchainService toolChainService;
     private final Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo;
 
@@ -49,12 +53,14 @@ public class BwcSetupExtension {
     public BwcSetupExtension(
         Project project,
         ObjectFactory objectFactory,
+        ProviderFactory providerFactory,
         JavaToolchainService toolChainService,
         Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo,
         Provider<File> checkoutDir
     ) {
         this.project = project;
         this.objectFactory = objectFactory;
+        this.providerFactory = providerFactory;
         this.toolChainService = toolChainService;
         this.unreleasedVersionInfo = unreleasedVersionInfo;
         this.checkoutDir = checkoutDir;
@@ -65,11 +71,26 @@ TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration)
     }
 
     TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration, boolean useUniqueUserHome) {
-        return createRunBwcGradleTask(project, name, configuration, useUniqueUserHome);
+        return createRunBwcGradleTask(
+            project,
+            checkoutDir,
+            providerFactory,
+            unreleasedVersionInfo,
+            objectFactory,
+            toolChainService,
+            name,
+            configuration,
+            useUniqueUserHome
+        );
     }
 
-    private TaskProvider<LoggedExec> createRunBwcGradleTask(
+    private static TaskProvider<LoggedExec> createRunBwcGradleTask(
         Project project,
+        Provider<File> checkoutDir,
+        ProviderFactory providerFactory,
+        Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo,
+        ObjectFactory objectFactory,
+        JavaToolchainService toolChainService,
         String name,
         Action<LoggedExec> configAction,
         boolean useUniqueUserHome
@@ -78,10 +99,10 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(
             loggedExec.dependsOn("checkoutBwcBranch");
             loggedExec.getWorkingDir().set(checkoutDir.get());
 
-            loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> {
-                String minimumCompilerVersion = readFromFile(new File(checkoutDir, minimumCompilerVersionPath(version.version())));
-                return getJavaHome(Integer.parseInt(minimumCompilerVersion));
-            }));
+            loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", providerFactory.of(JavaHomeValueSource.class, spec -> {
+                spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version()));
+                spec.getParameters().getCheckoutDir().set(checkoutDir);
+            }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s))));
 
             if (BuildParams.isCi() && OS.current() != OS.WINDOWS) {
                 // TODO: Disabled for now until we can figure out why files are getting corrupted
@@ -137,10 +158,13 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(
         });
     }
 
-    private String minimumCompilerVersionPath(Version bwcVersion) {
-        return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION))
-            ? "build-tools-internal/" + MINIMUM_COMPILER_VERSION_PATH
-            : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
+    /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
+    private static Provider<String> getJavaHome(ObjectFactory objectFactory, JavaToolchainService toolChainService, final int version) {
+        Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
+        return toolChainService.launcherFor(javaToolchainSpec -> {
+            javaToolchainSpec.getLanguageVersion().value(value);
+            javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE);
+        }).map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath());
     }
 
     private static String readFromFile(File file) {
@@ -151,13 +175,25 @@ private static String readFromFile(File file) {
         }
     }
 
-    /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
-    public String getJavaHome(final int version) {
-        Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
-        return toolChainService.launcherFor(javaToolchainSpec -> {
-            javaToolchainSpec.getLanguageVersion().value(value);
-            javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE);
-        }).get().getMetadata().getInstallationPath().getAsFile().getAbsolutePath();
-    }
+    public static abstract class JavaHomeValueSource implements ValueSource<String, JavaHomeValueSource.Params> {
+
+        private String minimumCompilerVersionPath(Version bwcVersion) {
+            return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION))
+                ? "build-tools-internal/" + MINIMUM_COMPILER_VERSION_PATH
+                : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
+        }
 
+        @Override
+        public String obtain() {
+            return readFromFile(
+                new File(getParameters().getCheckoutDir().get(), minimumCompilerVersionPath(getParameters().getVersion().get()))
+            );
+        }
+
+        public interface Params extends ValueSourceParameters {
+            Property<Version> getVersion();
+
+            Property<File> getCheckoutDir();
+        }
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -30,6 +30,7 @@
 import org.gradle.api.tasks.testing.Test;
 
 import java.io.File;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.gradle.util.FileUtils.mkdirs;
@@ -100,6 +101,7 @@ public void execute(Task t) {
                 "-Xmx" + System.getProperty("tests.heap.size", "512m"),
                 "-Xms" + System.getProperty("tests.heap.size", "512m"),
                 "-Djava.security.manager=allow",
+                "--add-opens=java.base/java.util=ALL-UNNAMED",
                 // TODO: only open these for mockito when it is modularized
                 "--add-opens=java.base/java.security.cert=ALL-UNNAMED",
                 "--add-opens=java.base/java.nio.channels=ALL-UNNAMED",
@@ -199,5 +201,29 @@ public void execute(Task t) {
                 }
             });
         });
+        configureImmutableCollectionsPatch(project);
+    }
+
+    private void configureImmutableCollectionsPatch(Project project) {
+        String patchProject = ":test:immutable-collections-patch";
+        if (project.findProject(patchProject) == null) {
+            return; // build tests may not have this project, just skip
+        }
+        String configurationName = "immutableCollectionsPatch";
+        FileCollection patchedFileCollection = project.getConfigurations()
+            .create(configurationName, config -> config.setCanBeConsumed(false));
+        var deps = project.getDependencies();
+        deps.add(configurationName, deps.project(Map.of("path", patchProject, "configuration", "patch")));
+        project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
+            test.getInputs().files(patchedFileCollection);
+            test.systemProperty("tests.hackImmutableCollections", "true");
+            test.getJvmArgumentProviders()
+                .add(
+                    () -> List.of(
+                        "--patch-module=java.base=" + patchedFileCollection.getSingleFile() + "/java.base",
+                        "--add-opens=java.base/java.util=ALL-UNNAMED"
+                    )
+                );
+        });
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java

@@ -93,13 +93,6 @@ public void execute(Task task) {
                         String remoteRepo = remote.get();
                         // for testing only we can override the base remote url
                         String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo")
-                            .orElse(
-                                providerFactory.provider(
-                                    () -> addRemote.getExtensions().getExtraProperties().has("remote")
-                                        ? addRemote.getExtensions().getExtraProperties().get("remote").toString()
-                                        : null
-                                )
-                            )
                             .getOrElse("https://github.com/" + remoteRepo + "/" + rootProjectName);
                         spec.commandLine("git", "remote", "add", remoteRepo, remoteRepoUrl);
                     });
@@ -213,6 +206,7 @@ private String maybeAlignedRefSpec(Logger logger, String defaultRefSpec) {
 
     private void writeFile(File file, String content) {
         try {
+            file.getParentFile().mkdirs();
             Files.writeString(file.toPath(), content, CREATE, TRUNCATE_EXISTING);
         } catch (IOException e) {
             throw new UncheckedIOException(e);