Merge remote-tracking branch 'upstream/main' into combine_global_chec…

…kpoint_with_location_sync

docs/changelog/95512.yaml

@@ -0,0 +1,5 @@
+pr: 95512
+summary: Adding `manage_dlm` index privilege and expanding `view_index_metadata` for access to data lifecycle APIs
+area: DLM
+type: enhancement
+issues: []

docs/changelog/96272.yaml

@@ -0,0 +1,5 @@
+pr: 96272
+summary: "[Profiling] Add status API"
+area: Application
+type: enhancement
+issues: []

docs/changelog/96328.yaml

@@ -0,0 +1,6 @@
+pr: 96328
+summary: Add `ingest` information to the cluster info endpoint
+area: Stats
+type: enhancement
+issues:
+ - 95392

docs/reference/cluster/cluster-info.asciidoc

@@ -33,9 +33,14 @@ You can use the Cluster Info API to retrieve information of a cluster.
 A comma-separated list of the following options:
 +
 --
+`_all`::
+All the information available. Can not be mixed with other targets.
+
 `http`::
 HTTP connection information.
 
+`ingest`::
+Ingest information.
 --
 
 [role="child_attributes"]
@@ -126,17 +131,128 @@ Cumulative size in bytes of all requests from this client.
 ======
 
 
+[[cluster-info-api-response-body-ingest]]
+`ingest`::
+(object)
+Contains ingest information for the cluster.
++
+.Properties of `ingest`
+[%collapsible%open]
+======
+`total`::
+(object)
+Contains information about ingest operations for the cluster.
++
+.Properties of `total`
+[%collapsible%open]
+=======
+`count`::
+(integer)
+Total number of documents ingested across the cluster.
+
+`time`::
+(<<time-units,time value>>)
+Total time spent preprocessing ingest documents across the cluster.
+
+`time_in_millis`::
+(integer)
+Total time, in milliseconds, spent preprocessing ingest documents across the cluster.
+
+`current`::
+(integer)
+Total number of documents currently being ingested.
+
+`failed`::
+(integer)
+Total number of failed ingest operations across the cluster.
+=======
+
+`pipelines`::
+(object)
+Contains information about ingest pipelines for the cluster.
++
+.Properties of `pipelines`
+[%collapsible%open]
+=======
+`<pipeline_id>`::
+(object)
+Contains information about the ingest pipeline.
++
+.Properties of `<pipeline_id>`
+[%collapsible%open]
+========
+`count`::
+(integer)
+Number of documents preprocessed by the ingest pipeline.
+
+`time`::
+(<<time-units,time value>>)
+Total time spent preprocessing documents in the ingest pipeline.
+
+`time_in_millis`::
+(integer)
+Total time, in milliseconds, spent preprocessing documents in the ingest
+pipeline.
+
+`failed`::
+(integer)
+Total number of failed operations for the ingest pipeline.
+
+`processors`::
+(array of objects)
+Contains information for the ingest processors for the ingest pipeline.
++
+.Properties of `processors`
+[%collapsible%open]
+=========
+`<processor>`::
+(object)
+Contains information for the ingest processor.
++
+.Properties of `<processor>`
+[%collapsible%open]
+==========
+`count`::
+(integer)
+Number of documents transformed by the processor.
+
+`time`::
+(<<time-units,time value>>)
+Time spent by the processor transforming documents.
+
+`time_in_millis`::
+(integer)
+Time, in milliseconds, spent by the processor transforming documents.
+
+`current`::
+(integer)
+Number of documents currently being transformed by the processor.
+
+`failed`::
+(integer)
+Number of failed operations for the processor.
+==========
+=========
+========
+=======
+======
+
+
+
 [[cluster-info-api-example]]
 ==== {api-examples-title}
 
 [source,console]
 ----
 # returns all stats info of the cluster
 GET /_info/_all
-----
 
-[source,console]
-----
 # returns the http info of the cluster
 GET /_info/http
+
+# returns the http info of the cluster
+GET /_info/ingest
+
+# returns the http and ingest info of the cluster
+GET /_info/http,ingest
 ----

docs/reference/data-streams/tsds-index-settings.asciidoc

@@ -37,14 +37,15 @@ can not be less than `time_series.poll_interval` cluster setting.
 (<<_static_index_settings,Static>>, string or array of strings) Plain `keyword`
 fields used to route documents in a TSDS to index shards. Supports wildcards
 (`*`). Only indices with an `index.mode` of `time_series` support this setting.
-Defaults to the list of <<time-series-dimension,dimension fields>> with a
-`time_series_dimension` value of `true` defined in your component templates. For
-more information, refer to <<dimension-based-routing>>.
+Defaults to an empty list, except for data streams then defaults to the list
+of <<time-series-dimension,dimension fields>> with a `time_series_dimension`
+value of `true` defined in your component and index templates. For more
+information, refer to <<dimension-based-routing>>.
 
 [[index-mapping-dimension-fields-limit]]
 // tag::dimensions-limit[]
 `index.mapping.dimension_fields.limit`::
 (<<dynamic-index-settings,Dynamic>>, integer)
 Maximum number of <<time-series-dimension,time series dimensions>> for the
-index. Defaults to `16`.
+index. Defaults to `21`.
 // end::dimensions-limit[]

docs/reference/dlm/apis/delete-lifecycle.asciidoc

@@ -8,6 +8,12 @@ experimental::[]
 
 Deletes the lifecycle from a set of data streams.
 
+[[delete-lifecycle-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the `manage_dlm` index privilege or higher to
+use this API. For more information, see <<security-privileges>>.
+
 [[dlm-delete-lifecycle-request]]
 ==== {api-request-title}
 

docs/reference/dlm/apis/explain-data-lifecycle.asciidoc

@@ -8,6 +8,14 @@ experimental::[]
 
 Retrieves the current data lifecycle status for one or more data stream backing indices.
 
+[[explain-lifecycle-api-prereqs]]
+==== {api-prereq-title}
+
+* Nit: would rephrase as:
+
+If the {es} {security-features} are enabled, you must have at least the `manage_dlm` index privilege or
+`view_index_metadata` index privilege to use this API. For more information, see <<security-privileges>>.
+
 [[dlm-explain-lifecycle-request]]
 ==== {api-request-title}
 

docs/reference/dlm/apis/get-lifecycle.asciidoc

@@ -8,6 +8,13 @@ experimental::[]
 
 Gets the lifecycle of a set of data streams.
 
+[[get-lifecycle-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have at least one of the `manage`
+<<privileges-list-indices,index privilege>>, the `manage_dlm` index privilege, or the
+`view_index_metadata` privilege to use this API. For more information, see <<security-privileges>>.
+
 [[dlm-get-lifecycle-request]]
 ==== {api-request-title}
 

docs/reference/dlm/apis/put-lifecycle.asciidoc

@@ -8,6 +8,12 @@ experimental::[]
 
 Configures the data lifecycle for the targeted data streams.
 
+[[put-lifecycle-api-prereqs]]
+==== {api-prereq-title}
+
+If the {es} {security-features} are enabled, you must have the `manage_dlm` index privilege or higher to use this API.
+For more information, see <<security-privileges>>.
+
 [[dlm-put-lifecycle-request]]
 ==== {api-request-title}
 

docs/reference/rest-api/usage.asciidoc

@@ -413,6 +413,9 @@ GET /_xpack/usage
     "enabled": true,
     "search_applications" : {
       "count": 0
+    },
+    "analytics_collections": {
+      "count": 0
     }
   }
 }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java

@@ -31,7 +31,6 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.tasks.Task;
@@ -141,10 +140,6 @@ protected void shardOperation(
         ActionListener.completeWith(listener, () -> {
             IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex());
             IndexShard indexShard = indexService.getShard(shardRouting.shardId().id());
-            // if we don't have the routing entry yet, we need it stats wise, we treat it as if the shard is not ready yet
-            if (indexShard.routingEntry() == null) {
-                throw new ShardNotFoundException(indexShard.shardId());
-            }
             StoreStats storeStats = indexShard.storeStats();
             IndexAbstraction indexAbstraction = clusterService.state().getMetadata().getIndicesLookup().get(shardRouting.getIndexName());
             assert indexAbstraction != null;

modules/dlm/qa/with-security/build.gradle

@@ -0,0 +1,19 @@
+import org.elasticsearch.gradle.Version
+
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.authenticated-testclusters'
+
+dependencies {
+  javaRestTestImplementation project(":client:rest-high-level")
+}
+
+testClusters.configureEach {
+  testDistribution = 'DEFAULT'
+  setting 'xpack.watcher.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+  rolesFile file('roles.yml')
+  user username: "test_dlm", password: "x-pack-test-password", role: "manage_dlm"
+  user username: "test_non_privileged", password: "x-pack-test-password", role: "not_privileged"
+  requiresFeature 'es.dlm_feature_flag_enabled', Version.fromString("8.9.0")
+}

modules/dlm/qa/with-security/roles.yml

@@ -0,0 +1,18 @@
+manage_dlm:
+  cluster:
+    - monitor
+  indices:
+    - names: [ 'dlm-*' ]
+      privileges:
+        - read
+        - write
+        - manage_dlm
+not_privileged:
+  cluster:
+    - monitor
+  indices:
+    - names: [ 'dlm-*' ]
+      privileges:
+        - read
+        - write
+        - view_index_metadata