Merge branch 'master' into add-time_series-aggregation

* master: (104 commits)
  fix: ordering terms aggregation on top metrics null values (elastic#85774)
  Fix up whitespace error introduced in elastic#85948
  More docs re. removing cluster.initial_master_nodes (elastic#85948)
  [Test] Remove API key methods from HLRC (elastic#85802)
  Remove references to bootstrap.system_call_filter (elastic#85964)
  Move docker cgroup override to SystemJvmOptions (elastic#85960)
  Add connection accounting tests (elastic#85966)
  Remove MacOS from platform support testing matrix
  Remove custom KnnVectorFieldExistsQuery (elastic#85945)
  Relax data path deprecations from critical to warn (elastic#85952)
  Remove hppc from some "common" classes (elastic#85957)
  Move docker env var settings handling out of bash (elastic#85913)
  Remove hppc from task manager (elastic#85889)
  [ML] rename trained model allocations to assignments (elastic#85503)
  Remove hppc from multi*shard request and responses (elastic#85888)
  Consolidating logging initialization in cli launcher (elastic#85920)
  Convert license tools to use unified cli entrypoint (elastic#85919)
  Add noop detection to node shutdown actions (elastic#85914)
  Adjust SQL expended test output
  TSDB: Add timestamp provider to AggregationExecutionContext (elastic#85850)
  ...

# Conflicts:
#	server/src/main/java/org/elasticsearch/search/aggregations/AggregationExecutionContext.java

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml

@@ -3,7 +3,8 @@
     name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm
     display-name: "elastic / elasticsearch # %BRANCH% - arm compatibility"
     description: "Elasticsearch %BRANCH% ARM (aarch64) compatibility testing.\n"
-    child-workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm"
+    # Don't use ramdisk for now as we are exhausting memory and causing oomkiller to trigger
+    # child-workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm"
     project-type: matrix
     node: master
     scm:

.ci/jobs.t/elastic+elasticsearch+periodic+java-matrix.yml

@@ -1,9 +1,41 @@
 ---
-jjbb-template: matrix-gradle-unix.yml
-vars:
-  - job-name: elastic+elasticsearch+%BRANCH%+periodic+java-matrix
-  - job-display-name: "elastic / elasticsearch # %BRANCH% - java compatibility matrix"
-  - job-description: "Testing of the Elasticsearch %BRANCH% branch java compatibility matrix.\n"
-  - matrix-yaml-file: ".ci/matrix-runtime-javas.yml"
-  - matrix-variable: ES_RUNTIME_JAVA
-  - gradle-args: "-Dbwc.checkout.align=true check"
+- job:
+    name: "elastic+elasticsearch+%BRANCH%+periodic+java-matrix"
+    display-name: "elastic / elasticsearch # %BRANCH% - java compatibility matrix"
+    description: "Testing of the Elasticsearch %BRANCH% branch java compatibility matrix.\n"
+    project-type: matrix
+    child-workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+java-matrix
+    node: master
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: slave
+          name: nodes
+          values:
+            - "general-purpose"
+      - axis:
+          type: yaml
+          filename: ".ci/matrix-runtime-javas.yml"
+          name: "ES_RUNTIME_JAVA"
+      # We shred out these jobs to avoid running out of memory given since we use a ramdisk workspace
+      - axis:
+          type: user-defined
+          name: GRADLE_TASK
+          values:
+            - 'checkPart1'
+            - 'checkPart2'
+            - 'bwcTestSnapshots'
+            - 'checkRestCompat'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA16_HOME=$HOME/.java/openjdk16
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $GRADLE_TASK

.ci/jobs.t/elastic+elasticsearch+periodic+platform-support.yml

@@ -13,10 +13,6 @@
       - multijob:
           name: Packaging tests
           projects:
-            - name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-darwin
-              kill-phase-on: NEVER
-              current-parameters: true
-              git-revision: true
             - name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-unix
               kill-phase-on: NEVER
               current-parameters: true

.ci/matrix-runtime-javas.yml

@@ -8,3 +8,4 @@
 ES_RUNTIME_JAVA:
   - openjdk17
   - openjdk18
+  - openjdk19

benchmarks/src/main/resources/org/elasticsearch/benchmark/xcontent/monitor_cluster_stats.json

@@ -262,7 +262,6 @@
             "vm_name" : "OpenJDK 64-Bit Server VM",
             "vm_version" : "11.0.7.7",
             "vm_vendor" : "OpenJDK",
-            "bundled_jdk" : false,
             "using_bundled_jdk" : null,
             "count" : 15
           }

build-tools-internal/build.gradle

@@ -257,7 +257,7 @@ dependencies {
   integTestImplementation("org.junit.jupiter:junit-jupiter") {
     because 'allows to write and run Jupiter tests'
   }
-  integTestImplementation("net.bytebuddy:byte-buddy:1.11.0") {
+  integTestImplementation("net.bytebuddy:byte-buddy:1.12.9") {
     because 'Generating dynamic mocks of internal libraries like JdkJarHell'
   }
   testRuntimeOnly("org.junit.vintage:junit-vintage-engine") {

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -40,10 +40,10 @@ buildScan {
 
     // Link to Jenkins worker logs and system metrics
     if (nodeName) {
-      link 'System logs', "https://infra-stats.elastic.co/app/infra#/logs?" +
+      link 'System logs', "https://ci-stats.elastic.co/app/infra#/logs?" +
         "&logFilter=(expression:'host.name:${nodeName}',kind:kuery)"
       buildFinished {
-        link 'System metrics', "https://infra-stats.elastic.co/app/metrics/detail/host/" +
+        link 'System metrics', "https://ci-stats.elastic.co/app/metrics/detail/host/" +
           "${nodeName}?_g=()&metricTime=(autoReload:!f,refreshInterval:5000," +
           "time:(from:${startTime - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)},interval:%3E%3D1m," +
           "to:${System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)}))"

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -90,7 +90,6 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Builds artifacts needed as dependency for IDE modules'
     dependsOn ':client:rest-high-level:shadowJar',
       ':plugins:repository-hdfs:hadoop-client-api:shadowJar',
-      ':modules:repository-azure:azure-storage-blob:shadowJar',
       ':libs:elasticsearch-x-content:generateProviderImpl'
   }
 

build-tools-internal/version.properties

@@ -2,7 +2,7 @@ elasticsearch     = 8.3.0
 lucene            = 9.1.0
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 17.0.2+8
+bundled_jdk = 18+36
 
 checkstyle = 9.3
 

build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java

@@ -26,7 +26,6 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.UncheckedIOException;
-import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.function.Consumer;
@@ -95,13 +94,7 @@ public void setSpoolOutput(boolean spoolOutput) {
             };
         } else {
             out = new ByteArrayOutputStream();
-            outputLogger = logger -> {
-                try {
-                    logger.error(((ByteArrayOutputStream) out).toString("UTF-8"));
-                } catch (UnsupportedEncodingException e) {
-                    throw new RuntimeException(e);
-                }
-            };
+            outputLogger = logger -> { logger.error(((ByteArrayOutputStream) out).toString(StandardCharsets.UTF_8)); };
         }
         setStandardOutput(out);
         setErrorOutput(out);
@@ -134,13 +127,9 @@ private static <T extends BaseExecSpec> ExecResult genericExec(Function<Action<T
                 }
             });
         } catch (Exception e) {
-            try {
-                if (output.size() != 0) {
-                    LOGGER.error("Exec output and error:");
-                    NEWLINE.splitAsStream(output.toString("UTF-8")).forEach(s -> LOGGER.error("| " + s));
-                }
-            } catch (UnsupportedEncodingException ue) {
-                throw new GradleException("Failed to read exec output", ue);
+            if (output.size() != 0) {
+                LOGGER.error("Exec output and error:");
+                NEWLINE.splitAsStream(output.toString(StandardCharsets.UTF_8)).forEach(s -> LOGGER.error("| " + s));
             }
             throw e;
         }

build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java

@@ -34,9 +34,6 @@
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.provider.Provider;
-import org.gradle.api.publish.PublishingExtension;
-import org.gradle.api.publish.maven.MavenPublication;
-import org.gradle.api.publish.maven.plugins.MavenPublishPlugin;
 import org.gradle.api.tasks.Copy;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.Sync;
@@ -74,42 +71,6 @@ public void apply(final Project project) {
         configureDependencies(project);
 
         final var bundleTask = createBundleTasks(project, extension);
-        project.afterEvaluate(project1 -> {
-            configurePublishing(project1, extension);
-            String name = extension.getName();
-            project1.setProperty("archivesBaseName", name);
-            project1.setDescription(extension.getDescription());
-
-            if (extension.getName() == null) {
-                throw new InvalidUserDataException("name is a required setting for esplugin");
-            }
-
-            if (extension.getDescription() == null) {
-                throw new InvalidUserDataException("description is a required setting for esplugin");
-            }
-
-            if (extension.getType().equals(PluginType.BOOTSTRAP) == false && extension.getClassname() == null) {
-                throw new InvalidUserDataException("classname is a required setting for esplugin");
-            }
-
-            Map<String, Object> map = new LinkedHashMap<>(12);
-            map.put("name", extension.getName());
-            map.put("description", extension.getDescription());
-            map.put("version", extension.getVersion());
-            map.put("elasticsearchVersion", Version.fromString(VersionProperties.getElasticsearch()).toString());
-            map.put("javaVersion", project1.getExtensions().getByType(JavaPluginExtension.class).getTargetCompatibility().toString());
-            map.put("classname", extension.getType().equals(PluginType.BOOTSTRAP) ? "" : extension.getClassname());
-            map.put("extendedPlugins", extension.getExtendedPlugins().stream().collect(Collectors.joining(",")));
-            map.put("hasNativeController", extension.isHasNativeController());
-            map.put("requiresKeystore", extension.isRequiresKeystore());
-            map.put("type", extension.getType().toString());
-            map.put("javaOpts", extension.getJavaOpts());
-            map.put("licensed", extension.isLicensed());
-            project1.getTasks().withType(Copy.class).named("pluginProperties").configure(copy -> {
-                copy.expand(map);
-                copy.getInputs().properties(map);
-            });
-        });
         project.getConfigurations().getByName("default").extendsFrom(project.getConfigurations().getByName("runtimeClasspath"));
 
         // allow running ES with this plugin in the foreground of a build
@@ -134,14 +95,6 @@ private static NamedDomainObjectContainer<ElasticsearchCluster> testClusters(Pro
         return (NamedDomainObjectContainer<ElasticsearchCluster>) project.getExtensions().getByName(extensionName);
     }
 
-    private static void configurePublishing(Project project, PluginPropertiesExtension extension) {
-        if (project.getPlugins().hasPlugin(MavenPublishPlugin.class)) {
-            PublishingExtension publishingExtension = project.getExtensions().getByType(PublishingExtension.class);
-            MavenPublication elastic = publishingExtension.getPublications().maybeCreate("elastic", MavenPublication.class);
-            elastic.setArtifactId(extension.getName());
-        }
-    }
-
     private static void configureDependencies(final Project project) {
         var dependencies = project.getDependencies();
         dependencies.add("compileOnly", "org.elasticsearch:elasticsearch:" + VersionProperties.getElasticsearch());
@@ -189,6 +142,35 @@ public void execute(Task task) {
             copy.dependsOn(copyPluginPropertiesTemplate);
             copy.from(templateFile);
             copy.into(new File(project.getBuildDir(), "generated-resources"));
+
+            if (extension.getName() == null) {
+                throw new InvalidUserDataException("name is a required setting for esplugin");
+            }
+
+            if (extension.getDescription() == null) {
+                throw new InvalidUserDataException("description is a required setting for esplugin");
+            }
+
+            if (extension.getType().equals(PluginType.BOOTSTRAP) == false && extension.getClassname() == null) {
+                throw new InvalidUserDataException("classname is a required setting for esplugin");
+            }
+
+            Map<String, Object> map = new LinkedHashMap<>(12);
+            map.put("name", extension.getName());
+            map.put("description", extension.getDescription());
+            map.put("version", extension.getVersion());
+            map.put("elasticsearchVersion", Version.fromString(VersionProperties.getElasticsearch()).toString());
+            map.put("javaVersion", project.getExtensions().getByType(JavaPluginExtension.class).getTargetCompatibility().toString());
+            map.put("classname", extension.getType().equals(PluginType.BOOTSTRAP) ? "" : extension.getClassname());
+            map.put("extendedPlugins", extension.getExtendedPlugins().stream().collect(Collectors.joining(",")));
+            map.put("hasNativeController", extension.isHasNativeController());
+            map.put("requiresKeystore", extension.isRequiresKeystore());
+            map.put("type", extension.getType().toString());
+            map.put("javaOpts", extension.getJavaOpts());
+            map.put("licensed", extension.isLicensed());
+
+            copy.expand(map);
+            copy.getInputs().properties(map);
         });
         // add the plugin properties and metadata to test resources, so unit tests can
         // know about the plugin (used by test security code to statically initialize the plugin in unit tests)

build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java

@@ -66,6 +66,7 @@ public String getName() {
     }
 
     public void setName(String name) {
+        this.project.setProperty("archivesBaseName", name);
         this.name = name;
     }
 
@@ -82,6 +83,7 @@ public String getDescription() {
     }
 
     public void setDescription(String description) {
+        project.setDescription(description);
         this.description = description;
     }