Merge commit '89a65b2c5ab' into add_config_user_data

Tim-Brooks · Jun 21, 2023 · d6d57a6 · d6d57a6
2 parents 7377639 + 89a65b2
commit d6d57a6
Show file tree

Hide file tree

Showing 193 changed files with 3,349 additions and 1,181 deletions.
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.core.IOUtils;
@@ -138,6 +139,7 @@ protected SearchExecutionContext buildSearchExecutionContext() {
             0,
             0,
             mapperService.getIndexSettings(),
+            ClusterSettings.createBuiltInClusterSettings(),
             null,
             (ft, fdc) -> ft.fielddataBuilder(fdc).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()),
             mapperService,

diff --git a/...a/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java b/...a/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java
@@ -269,6 +269,11 @@ public IndexSettings getIndexSettings() {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        public ClusterSettings getClusterSettings() {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public Optional<SortAndFormats> buildSort(List<SortBuilder<?>> sortBuilders) throws IOException {
             throw new UnsupportedOperationException();

diff --git a/...ternal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/...ternal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java
@@ -58,6 +58,7 @@
 import javax.inject.Inject;
 
 import static org.gradle.api.JavaVersion.VERSION_20;
+import static org.gradle.api.JavaVersion.VERSION_21;
 
 @CacheableTask
 public abstract class ThirdPartyAuditTask extends DefaultTask {
@@ -337,8 +338,8 @@ private String runForbiddenAPIsCli() throws IOException {
                 spec.setExecutable(javaHome.get() + "/bin/java");
             }
             spec.classpath(getForbiddenAPIsClasspath(), classpath);
-            // Enable explicitly for each release as appropriate. Just JDK 20 for now, and just the vector module.
-            if (isJava20()) {
+            // Enable explicitly for each release as appropriate. Just JDK 20/21 for now, and just the vector module.
+            if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21)) {
                 spec.jvmArgs("--add-modules", "jdk.incubator.vector");
             }
             spec.jvmArgs("-Xmx1g");
@@ -363,13 +364,13 @@ private String runForbiddenAPIsCli() throws IOException {
         return forbiddenApisOutput;
     }
 
-    /** Returns true iff the Java version is 20. */
-    private boolean isJava20() {
+    /** Returns true iff the build Java version is the same as the given version. */
+    private boolean isJavaVersion(JavaVersion version) {
         if (BuildParams.getIsRuntimeJavaHomeSet()) {
-            if (VERSION_20.equals(BuildParams.getRuntimeJavaVersion())) {
+            if (version.equals(BuildParams.getRuntimeJavaVersion())) {
                 return true;
             }
-        } else if ("20".equals(VersionProperties.getBundledJdkMajorVersion())) {
+        } else if (version.getMajorVersion().equals(VersionProperties.getBundledJdkMajorVersion())) {
             return true;
         }
         return false;

diff --git a/docs/changelog/95512.yaml b/docs/changelog/95512.yaml
@@ -1,5 +1,5 @@
 pr: 95512
-summary: Adding `manage_dlm` index privilege and expanding `view_index_metadata` for access to data lifecycle APIs
+summary: Adding `manage_data_stream_lifecycle` index privilege and expanding `view_index_metadata` for access to data stream lifecycle APIs
 area: DLM
 type: enhancement
 issues: []
diff --git a/docs/changelog/96551.yaml b/docs/changelog/96551.yaml
@@ -0,0 +1,5 @@
+pr: 96551
+summary: Make cluster health API cancellable
+area: Distributed
+type: bug
+issues: []
diff --git a/docs/changelog/96782.yaml b/docs/changelog/96782.yaml
@@ -0,0 +1,5 @@
+pr: 96782
+summary: Increase concurrent request of opening point-in-time
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/96878.yaml b/docs/changelog/96878.yaml
@@ -0,0 +1,5 @@
+pr: 96878
+summary: Min score for time series
+area: TSDB
+type: bug
+issues: []
diff --git a/docs/changelog/96885.yaml b/docs/changelog/96885.yaml
@@ -0,0 +1,5 @@
+pr: 96885
+summary: Add Search ALC filter index prefix to the enterprise search user
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/96924.yaml b/docs/changelog/96924.yaml
@@ -0,0 +1,6 @@
+pr: 96924
+summary: Interpret microseconds cpu stats from cgroups2 properly as nanos
+area: Infra/Core
+type: bug
+issues:
+ - 96089
diff --git a/docs/changelog/96941.yaml b/docs/changelog/96941.yaml
@@ -0,0 +1,6 @@
+pr: 96941
+summary: Trim field references in reroute processor
+area: Ingest Node
+type: bug
+issues:
+ - 96939
diff --git a/docs/changelog/96943.yaml b/docs/changelog/96943.yaml
@@ -0,0 +1,5 @@
+pr: 96943
+summary: Add cluster setting to `SearchExecutionContext` to configure `TDigestExecutionHint`
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/docs/changelog/96956.yaml b/docs/changelog/96956.yaml
@@ -0,0 +1,6 @@
+pr: 96956
+summary: Error message for misconfigured TSDB index
+area: TSDB
+type: bug
+issues:
+ - 96445
diff --git a/docs/changelog/96958.yaml b/docs/changelog/96958.yaml
@@ -0,0 +1,5 @@
+pr: 96958
+summary: "Ingest: expose reroute inquiry/reset via Elastic-internal API bridge"
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/changelog/96970.yaml b/docs/changelog/96970.yaml
@@ -0,0 +1,5 @@
+pr: 96970
+summary: Ensure checking application privileges work with nested-limited roles
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/reference/dlm/apis/delete-lifecycle.asciidoc b/docs/reference/dlm/apis/delete-lifecycle.asciidoc
@@ -11,7 +11,7 @@ Deletes the lifecycle from a set of data streams.
 [[delete-lifecycle-api-prereqs]]
 ==== {api-prereq-title}
 
-* If the {es} {security-features} are enabled, you must have the `manage_dlm` index privilege or higher to
+* If the {es} {security-features} are enabled, you must have the `manage_data_stream_lifecycle` index privilege or higher to
 use this API. For more information, see <<security-privileges>>.
 
 [[dlm-delete-lifecycle-request]]

diff --git a/docs/reference/dlm/apis/explain-data-lifecycle.asciidoc b/docs/reference/dlm/apis/explain-data-lifecycle.asciidoc
@@ -13,7 +13,7 @@ Retrieves the current data lifecycle status for one or more data stream backing
 
 * Nit: would rephrase as:
 
-If the {es} {security-features} are enabled, you must have at least the `manage_dlm` index privilege or
+If the {es} {security-features} are enabled, you must have at least the `manage_data_stream_lifecycle` index privilege or
 `view_index_metadata` index privilege to use this API. For more information, see <<security-privileges>>.
 
 [[dlm-explain-lifecycle-request]]

diff --git a/docs/reference/dlm/apis/get-lifecycle.asciidoc b/docs/reference/dlm/apis/get-lifecycle.asciidoc
@@ -12,7 +12,7 @@ Gets the lifecycle of a set of data streams.
 ==== {api-prereq-title}
 
 * If the {es} {security-features} are enabled, you must have at least one of the `manage`
-<<privileges-list-indices,index privilege>>, the `manage_dlm` index privilege, or the
+<<privileges-list-indices,index privilege>>, the `manage_data_stream_lifecycle` index privilege, or the
 `view_index_metadata` privilege to use this API. For more information, see <<security-privileges>>.
 
 [[dlm-get-lifecycle-request]]

diff --git a/docs/reference/dlm/apis/put-lifecycle.asciidoc b/docs/reference/dlm/apis/put-lifecycle.asciidoc
@@ -11,7 +11,7 @@ Configures the data lifecycle for the targeted data streams.
 [[put-lifecycle-api-prereqs]]
 ==== {api-prereq-title}
 
-If the {es} {security-features} are enabled, you must have the `manage_dlm` index privilege or higher to use this API.
+If the {es} {security-features} are enabled, you must have the `manage_data_stream_lifecycle` index privilege or higher to use this API.
 For more information, see <<security-privileges>>.
 
 [[dlm-put-lifecycle-request]]

diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc
@@ -51,6 +51,9 @@ of datasets and configurations that we use for our nightly benchmarks.
 [discrete]
 include::search-speed.asciidoc[tag=warm-fs-cache]
 
+The following file extensions are used for the approximate kNN search:
+"vec" (for vector values), "vex" (for HNSW graph), "vem" (for metadata).
+
 [discrete]
 === Reduce vector dimensionality
 

diff --git a/docs/reference/migration/apis/feature-migration.asciidoc b/docs/reference/migration/apis/feature-migration.asciidoc
@@ -56,93 +56,92 @@ Example response:
   "features" : [
     {
       "feature_name" : "async_search",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "enrich",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "ent_search",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "fleet",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "geoip",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "kibana",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "logstash_management",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "machine_learning",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "searchable_snapshots",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "security",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "synonyms",
-      "minimum_index_version" : "8.9.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "tasks",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "transform",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "watcher",
-      "minimum_index_version" : "8.0.0",
+      "minimum_index_version" : "{version}",
       "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     }
   ],
   "migration_status" : "NO_MIGRATION_NEEDED"
 }
 --------------------------------------------------
-// TESTRESPONSE[s/"minimum_index_version" : "8.0.0"/"minimum_index_version" : $body.$_path/]
 
 
 When you submit a POST request to the `_migration/system_features` endpoint to

diff --git a/docs/reference/setup/install/check-running.asciidoc b/docs/reference/setup/install/check-running.asciidoc
@@ -41,7 +41,8 @@ GET /
     "build_snapshot" : false,
     "lucene_version" : "{lucene_version}",
     "minimum_wire_compatibility_version" : "1.2.3",
-    "minimum_index_compatibility_version" : "1.2.3"
+    "minimum_index_compatibility_version" : "1.2.3",
+    "transport_version" : "123456"
   },
   "tagline" : "You Know, for Search"
 }
@@ -54,4 +55,5 @@ GET /
 // TESTRESPONSE[s/"build_snapshot" : false,/"build_snapshot" : $body.version.build_snapshot,/]
 // TESTRESPONSE[s/"minimum_wire_compatibility_version" : "1.2.3"/"minimum_wire_compatibility_version" : $body.version.minimum_wire_compatibility_version/]
 // TESTRESPONSE[s/"minimum_index_compatibility_version" : "1.2.3"/"minimum_index_compatibility_version" : $body.version.minimum_index_compatibility_version/]
+// TESTRESPONSE[s/"transport_version" : "123456"/"transport_version" : $body.version.transport_version/]
 // So much s/// but at least we test that the layout is close to matching....
diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java
@@ -42,10 +42,10 @@ public class SortingDigest extends AbstractTDigest {
     @Override
     public void add(double x, int w) {
         checkValue(x);
+        isSorted = isSorted && (values.isEmpty() || values.get(values.size() - 1) <= x);
         for (int i = 0; i < w; i++) {
             values.add(x);
         }
-        isSorted = false;
         max = Math.max(max, x);
         min = Math.min(min, x);
     }

diff --git a/...mlRestTest/resources/rest-api-spec/test/aggregations/median_absolute_deviation_metric.yml b/...mlRestTest/resources/rest-api-spec/test/aggregations/median_absolute_deviation_metric.yml
@@ -76,6 +76,49 @@ setup:
   - match: { aggregations.mad_int.value: 100 }
   - match: { aggregations.mad_double.value: 100 }
 
+---
+"with execution hint":
+
+  - skip:
+      version: " - 8.8.99"
+      reason: t-digest forked in 8.9.0
+
+  - do:
+      search:
+        body:
+          aggs:
+            mad_int:
+              median_absolute_deviation:
+                field: int_field
+                execution_hint: high_accuracy
+
+  - match: { hits.total.value: 3 }
+  - length: { hits.hits: 3 }
+  - match: { aggregations.mad_int.value: 100 }
+
+  - do:
+      search:
+        body:
+          aggs:
+            mad_int:
+              median_absolute_deviation:
+                field: int_field
+                execution_hint: default
+
+  - match: { hits.total.value: 3 }
+  - length: { hits.hits: 3 }
+  - match: { aggregations.mad_int.value: 100 }
+
+  - do:
+      catch: /Invalid execution_hint \[no_such_hint\], valid values are \[default, high_accuracy\]/
+      search:
+        body:
+          aggs:
+            mad:
+              median_absolute_deviation:
+                field: int_field
+                execution_hint: no_such_hint
+
 ---
 "no documents":