Merge pull request #227 from elastic/main

🤖 ESQL: Merge upstream

.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml

@@ -22,8 +22,14 @@
           export BEATS_DIR=$(pwd)/distribution/docker/build/artifacts/beats
 
           mkdir -p ${BEATS_DIR}
-          curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
-          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
 
-          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false \
+          # Fetch ML artifacts
+          export ML_IVY_REPO=$(mktemp -d)
+          mkdir -p ${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}
+          curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}-deps.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT-deps.zip
+          curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}-nodeps.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT-nodeps.zip
+
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false -Dbuild.ml_cpp.repo=file://${ML_IVY_REPO} \
             -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef build

.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml

@@ -44,8 +44,14 @@
           export BEATS_DIR=$(pwd)/distribution/docker/build/artifacts/beats
 
           mkdir -p ${BEATS_DIR}
-          curl -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
-          curl -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
 
-          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false \
+          # Fetch ML artifacts
+          export ML_IVY_REPO=$(mktemp -d)
+          mkdir -p ${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}
+          curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}-deps.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT-deps.zip
+          curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}-nodeps.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT-nodeps.zip
+
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false -Dbuild.ml_cpp.repo=file://${ML_IVY_REPO} \
             -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef build

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPluginFuncTest.groovy

@@ -172,7 +172,7 @@ class ElasticsearchJavaModulePathPluginFuncTest extends AbstractJavaGradleFuncTe
     def "included build with non module dependencies"() {
         given:
         file(rootBuild.root, 'settings.gradle') << """
-        includeBuild '$projectDir'
+        includeBuild '${projectDir.path.replace('\\', '\\\\')}'
         """
 
         file(rootBuild.root, 'build.gradle') << """
@@ -204,7 +204,7 @@ class ElasticsearchJavaModulePathPluginFuncTest extends AbstractJavaGradleFuncTe
     def "included build with module dependencies"() {
         given:
         file(rootBuild.root, 'settings.gradle') << """
-        includeBuild '$projectDir'
+        includeBuild '${projectDir.path.replace('\\', '\\\\')}'
         """
 
         file(rootBuild.root, 'build.gradle') << """

docs/changelog/86388.yaml

@@ -0,0 +1,5 @@
+pr: 86388
+summary: Do not use recovery from snapshots in searchable snapshots
+area: Recovery
+type: enhancement
+issues: []

docs/changelog/88209.yaml

@@ -0,0 +1,6 @@
+pr: 88209
+summary: Prioritize shard snapshot tasks over file snapshot tasks and limit the number of the concurrently running snapshot tasks
+area: Snapshot/Restore
+type: enhancement
+issues:
+ - 83408

docs/changelog/88241.yaml

@@ -0,0 +1,5 @@
+pr: 88241
+summary: Log unsuccessful attempts to get credentials from web identity tokens
+area: Allocation
+type: enhancement
+issues: []

docs/changelog/89645.yaml

@@ -0,0 +1,5 @@
+pr: 89645
+summary: Add processor autoscaling decider
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/89754.yaml

@@ -0,0 +1,6 @@
+pr: 89754
+summary: Fix segment stats in tsdb
+area: TSDB
+type: bug
+issues:
+ - 89609

docs/changelog/89822.yaml

@@ -0,0 +1,5 @@
+pr: 89822
+summary: Distribute trained model allocations across availability zones
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/89833.yaml

@@ -0,0 +1,5 @@
+pr: 89833
+summary: "Synthetic _source: support histogram field"
+area: TSDB
+type: enhancement
+issues: []

docs/changelog/89840.yaml

@@ -0,0 +1,5 @@
+pr: 89840
+summary: "Synthetic _source: support `dense_vector`"
+area: Vector Search
+type: feature
+issues: []

docs/changelog/89845.yaml

@@ -0,0 +1,5 @@
+pr: 89845
+summary: "Fleet: Add `upgrade_status` attributes to agents"
+area: Infra/Core
+type: enhancement
+issues: []

docs/changelog/89851.yaml

@@ -0,0 +1,6 @@
+pr: 89851
+summary: Distinguish no shutdowns case in `NodeShutdownAllocationDecider`
+area: Infra/Node Lifecycle
+type: enhancement
+issues:
+ - 89823

docs/changelog/89873.yaml

@@ -0,0 +1,5 @@
+pr: 89873
+summary: Fix memory leak when double invoking `RestChannel.sendResponse`
+area: Network
+type: bug
+issues: []

docs/changelog/89889.yaml

@@ -0,0 +1,5 @@
+pr: 89889
+summary: "Script: Write Field API path manipulation"
+area: Infra/Scripting
+type: enhancement
+issues: []

docs/changelog/89934.yaml

@@ -0,0 +1,5 @@
+pr: 89934
+summary: Fix deadlock bug exposed by the test
+area: Infra/Core
+type: bug
+issues: []

docs/changelog/89935.yaml

@@ -0,0 +1,5 @@
+pr: 89935
+summary: Fix memory leak in `TransportDeleteExpiredDataAction`
+area: Machine Learning
+type: bug
+issues: []

docs/reference/health/health.asciidoc

@@ -262,16 +262,18 @@ details have contents and a structure that is unique to each indicator.
 ====
 
 `cluster_formation`::
-    (Optional, map) If there has been no elected master node recently, the node being queried attempts to
+    (Optional, array) If there has been no elected master node recently, the node being queried attempts to
     gather information about why the cluster has been unable to form, or why the node being queried has been
-    unable to join the cluster if it has formed. This could contain each master eligibles
-    node view on the cluster formation. The node id is the key in the map and its 
-    cluster formation information is the value.
+    unable to join the cluster if it has formed. This array could contain any entry for each master eligible
+    node's view of cluster formation.
 +
 .Properties of `cluster_formation`
 [%collapsible%open]
 ====
-`description`::
+`node_id`::
+    (string) The node id of a master-eligible node
+
+`cluster_formation_message`::
     (string) A detailed description explaining what went wrong with cluster formation, or why this node was
     unable to join the cluster if it has formed.
 ====
@@ -342,7 +344,7 @@ details have contents and a structure that is unique to each indicator.
 
 `unhealthy_policies`::
     (map) A detailed view on the policies that are considered unhealthy due to having
-    several consecutive unssuccesful invocations. 
+    several consecutive unssuccesful invocations.
     The `count` key represents the number of unhealthy policies (int).
     The `invocations_since_last_success` key will report a map where the unhealthy policy
     name is the key and it's corresponding number of failed invocations is the value.

docs/reference/how-to/disk-usage.asciidoc

@@ -136,6 +136,6 @@ if fields always occur in the same order.
 [[roll-up-historical-data]]
 === Roll up historical data
 
-Keeping older data can useful for later analysis but is often avoided due to
+Keeping older data can be useful for later analysis but is often avoided due to
 storage costs. You can use data rollups to summarize and store historical data
 at a fraction of the raw data's storage cost. See <<xpack-rollup>>.

docs/reference/index-modules.asciidoc

@@ -39,7 +39,7 @@ specific index module:
 `index.number_of_shards`::
 The number of primary shards that an index should have. Defaults to `1`. This setting can only be set at index creation time. It cannot be changed on a closed index.
 +
-NOTE: The number of shards are limited to `1024` per index. This limitation is a safety limit to prevent accidental creation of indices that can destabilize a cluster due to resource allocation. The limit can be modified by specifying `export CLI_JAVA_OPTS="-Des.index.max_number_of_shards=128"` system property on every node that is part of the cluster.
+NOTE: The number of shards are limited to `1024` per index. This limitation is a safety limit to prevent accidental creation of indices that can destabilize a cluster due to resource allocation. The limit can be modified by specifying `export ES_JAVA_OPTS="-Des.index.max_number_of_shards=128"` system property on every node that is part of the cluster.
 
 // end::index-number-of-shards-tag[]
 

docs/reference/mapping/fields/synthetic-source.asciidoc

@@ -31,10 +31,12 @@ types:
 ** <<aggregate-metric-double-synthetic-source, `aggregate_metric_double`>>
 ** <<boolean-synthetic-source,`boolean`>>
 ** <<numeric-synthetic-source,`byte`>>
+** <<dense-vector-synthetic-source,`dense_vector`>>
 ** <<numeric-synthetic-source,`double`>>
 ** <<numeric-synthetic-source,`float`>>
 ** <<geo-point-synthetic-source,`geo_point`>>
 ** <<numeric-synthetic-source,`half_float`>>
+** <<histogram-synthetic-source,`histogram`>>
 ** <<numeric-synthetic-source,`integer`>>
 ** <<ip-synthetic-source,`ip`>>
 ** <<keyword-synthetic-source,`keyword`>>

docs/reference/mapping/types/dense-vector.asciidoc

@@ -178,3 +178,7 @@ Defaults to `16`.
 The number of candidates to track while assembling the list of nearest
 neighbors for each new node. Defaults to `100`.
 ====
+
+[[dense-vector-synthetic-source]]
+==== Synthetic source preview:[]
+`dense_vector` fields support <<synthetic-source,synthetic `_source`>> .

docs/reference/mapping/types/histogram.asciidoc

@@ -85,6 +85,12 @@ The histogram field is "algorithm agnostic" and does not store data specific to
 means the field can technically be aggregated with either algorithm, in practice the user should chose one algorithm and
 index data in that manner (e.g. centroids for T-Digest or intervals for HDRHistogram) to ensure best accuracy.
 
+[[histogram-synthetic-source]]
+==== Synthetic source preview:[]
+`histogram` fields support <<synthetic-source,synthetic `_source`>> in their
+default configuration. Synthetic `_source` cannot be used together with
+<<ignore-malformed,`ignore_malformed`>> or <<copy-to,`copy_to`>>.
+
 [[histogram-ex]]
 ==== Examples
 

docs/reference/mapping/types/keyword.asciidoc

@@ -173,12 +173,11 @@ Dimension fields have the following constraints:
 ==== Synthetic source preview:[]
 `keyword` fields support <<synthetic-source,synthetic `_source`>> in their
 default configuration. Synthetic `_source` cannot be used together with
-<<ignore-above,`ignore_above`>>, a <<normalizer,`normalizer`>>,
-<<copy-to,`copy_to`>>, or with <<doc-values,`doc_values`>> disabled.
+a <<normalizer,`normalizer`>> or <<copy-to,`copy_to`>>.
 
-Synthetic source always sorts `keyword` fields and removes duplicates. For
-example:
-[source,console,id=synthetic-source-keyword-example]
+By default, synthetic source sorts `keyword` fields and removes duplicates.
+For example:
+[source,console,id=synthetic-source-keyword-example-default]
 ----
 PUT idx
 {
@@ -206,6 +205,67 @@ Will become:
 ----
 // TEST[s/^/{"_source":/ s/\n$/}/]
 
+If a `keyword` field sets `store` to `true` then order and duplicates
+are preserved. For example:
+[source,console,id=synthetic-source-keyword-example-stored]
+----
+PUT idx
+{
+  "mappings": {
+    "_source": { "mode": "synthetic" },
+    "properties": {
+      "kwd": { "type": "keyword", "store": true }
+    }
+  }
+}
+PUT idx/_doc/1
+{
+  "kwd": ["foo", "foo", "bar", "baz"]
+}
+----
+// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/]
+
+Will become:
+
+[source,console-result]
+----
+{
+  "kwd": ["foo", "foo", "bar", "baz"]
+}
+----
+// TEST[s/^/{"_source":/ s/\n$/}/]
+
+Values longer than `ignore_above` are preserved but sorted to the end.
+For example:
+[source,console,id=synthetic-source-keyword-example-ignore-above]
+----
+PUT idx
+{
+  "mappings": {
+    "_source": { "mode": "synthetic" },
+    "properties": {
+      "kwd": { "type": "keyword", "ignore_above": 3 }
+    }
+  }
+}
+PUT idx/_doc/1
+{
+  "kwd": ["foo", "foo", "bang", "bar", "baz"]
+}
+----
+// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/]
+
+Will become:
+
+[source,console-result]
+----
+{
+  "kwd": ["bar", "baz", "foo", "bang"]
+}
+----
+// TEST[s/^/{"_source":/ s/\n$/}/]
+
+
 include::constant-keyword.asciidoc[]
 
 include::wildcard.asciidoc[]

docs/reference/mapping/types/text.asciidoc

@@ -162,12 +162,14 @@ The following parameters are accepted by `text` fields:
 [[text-synthetic-source]]
 ==== Synthetic source preview:[]
 `text` fields support <<synthetic-source,synthetic `_source`>> if they have
-a `keyword` sub-field that supports synthetic `_source` and *do not* have
-<<copy-to,`copy_to`>>.
-
-Synthetic source always sorts `keyword` fields and removes duplicates, so
-`text` fields are sorted based on the sub-`keyword` field. For example:
-[source,console,id=synthetic-source-text-example]
+a <<keyword-synthetic-source, `keyword`>> sub-field that supports synthetic
+`_source` or if the `text` field sets `store` to `true`. Either way, it may
+not have <<copy-to,`copy_to`>>.
+
+If using a sub-`keyword` field then the values are sorted in the same way as
+a `keyword` field's values are sorted. By default, that means sorted with
+duplicates removed. So:
+[source,console,id=synthetic-source-text-example-default]
 ----
 PUT idx
 {
@@ -214,6 +216,43 @@ NOTE: Reordering text fields can have an effect on <<query-dsl-match-query-phras
       can avoid this by making sure the `slop` parameter on the phrase queries
       is lower than the `position_increment_gap`. This is the default.
 
+If the `text` field sets `store` to true then order and duplicates
+are preserved.
+[source,console,id=synthetic-source-text-example-stored]
+----
+PUT idx
+{
+  "mappings": {
+    "_source": { "mode": "synthetic" },
+    "properties": {
+      "text": { "type": "text", "store": true }
+    }
+  }
+}
+PUT idx/_doc/1
+{
+  "text": [
+    "the quick brown fox",
+    "the quick brown fox",
+    "jumped over the lazy dog"
+  ]
+}
+----
+// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/]
+
+Will become:
+[source,console-result]
+----
+{
+  "text": [
+    "the quick brown fox",
+    "the quick brown fox",
+    "jumped over the lazy dog"
+  ]
+}
+----
+// TEST[s/^/{"_source":/ s/\n$/}/]
+
 [[fielddata-mapping-param]]
 ==== `fielddata` mapping parameter