Update from upstream repository

clients/pkg/promtail/targets/docker/target_test.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
-	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -17,6 +16,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/relabel"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/loki/clients/pkg/promtail/client/fake"
@@ -77,49 +77,56 @@ func Test_DockerTarget(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	require.Eventually(t, func() bool {
-		return len(entryHandler.Received()) >= 5
-	}, 5*time.Second, 100*time.Millisecond)
-
-	received := entryHandler.Received()
-	sort.Slice(received, func(i, j int) bool {
-		return received[i].Timestamp.Before(received[j].Timestamp)
-	})
-
 	expectedLines := []string{
 		"5.3.69.55 - - [09/Dec/2021:09:15:02 +0000] \"HEAD /brand/users/clicks-and-mortar/front-end HTTP/2.0\" 503 27087",
 		"101.54.183.185 - - [09/Dec/2021:09:15:03 +0000] \"POST /next-generation HTTP/1.0\" 416 11468",
 		"69.27.137.160 - runolfsdottir2670 [09/Dec/2021:09:15:03 +0000] \"HEAD /content/visionary/engineer/cultivate HTTP/1.1\" 302 2975",
 		"28.104.242.74 - - [09/Dec/2021:09:15:03 +0000] \"PATCH /value-added/cultivate/systems HTTP/2.0\" 405 11843",
 		"150.187.51.54 - satterfield1852 [09/Dec/2021:09:15:03 +0000] \"GET /incentivize/deliver/innovative/cross-platform HTTP/1.1\" 301 13032",
 	}
-	actualLines := make([]string, 0, 5)
-	for _, entry := range received[:5] {
-		actualLines = append(actualLines, entry.Line)
-	}
-	require.ElementsMatch(t, actualLines, expectedLines)
 
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		assertExpectedLog(c, entryHandler, expectedLines)
+	}, 5*time.Second, 100*time.Millisecond, "Expected log lines were not found within the time limit.")
+
+	target.Stop()
+	entryHandler.Clear()
 	// restart target to simulate container restart
 	target.startIfNotRunning()
-	entryHandler.Clear()
-	require.Eventually(t, func() bool {
-		return len(entryHandler.Received()) >= 5
-	}, 5*time.Second, 100*time.Millisecond)
-
-	receivedAfterRestart := entryHandler.Received()
-	sort.Slice(receivedAfterRestart, func(i, j int) bool {
-		return receivedAfterRestart[i].Timestamp.Before(receivedAfterRestart[j].Timestamp)
-	})
-	actualLinesAfterRestart := make([]string, 0, 5)
-	for _, entry := range receivedAfterRestart[:5] {
-		actualLinesAfterRestart = append(actualLinesAfterRestart, entry.Line)
-	}
 	expectedLinesAfterRestart := []string{
 		"243.115.12.215 - - [09/Dec/2023:09:16:57 +0000] \"DELETE /morph/exploit/granular HTTP/1.0\" 500 26468",
 		"221.41.123.237 - - [09/Dec/2023:09:16:57 +0000] \"DELETE /user-centric/whiteboard HTTP/2.0\" 205 22487",
 		"89.111.144.144 - - [09/Dec/2023:09:16:57 +0000] \"DELETE /open-source/e-commerce HTTP/1.0\" 401 11092",
 		"62.180.191.187 - - [09/Dec/2023:09:16:57 +0000] \"DELETE /cultivate/integrate/technologies HTTP/2.0\" 302 12979",
 		"156.249.2.192 - - [09/Dec/2023:09:16:57 +0000] \"POST /revolutionize/mesh/metrics HTTP/2.0\" 401 5297",
 	}
-	require.ElementsMatch(t, actualLinesAfterRestart, expectedLinesAfterRestart)
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		assertExpectedLog(c, entryHandler, expectedLinesAfterRestart)
+	}, 5*time.Second, 100*time.Millisecond, "Expected log lines after restart were not found within the time limit.")
+}
+
+// assertExpectedLog will verify that all expectedLines were received, in any order, without duplicates.
+func assertExpectedLog(c *assert.CollectT, entryHandler *fake.Client, expectedLines []string) {
+	logLines := entryHandler.Received()
+	testLogLines := make(map[string]int)
+	for _, l := range logLines {
+		if containsString(expectedLines, l.Line) {
+			testLogLines[l.Line]++
+		}
+	}
+	// assert that all log lines were received
+	assert.Len(c, testLogLines, len(expectedLines))
+	// assert that there are no duplicated log lines
+	for _, v := range testLogLines {
+		assert.Equal(c, v, 1)
+	}
+}
+
+func containsString(slice []string, str string) bool {
+	for _, item := range slice {
+		if item == str {
+			return true
+		}
+	}
+	return false
 }

docs/sources/configure/_index.md

@@ -4577,7 +4577,7 @@ chunks:
   [tags: <map of string to string>]
 
 # How many shards will be created. Only used if schema is v10 or greater.
-[row_shards: <int>]
+[row_shards: <int> | default = 16]
 ```
 
 ### aws_storage_config
@@ -5345,7 +5345,8 @@ place in the `limits_config` section:
 configure a runtime configuration file:
 
     ```
-    runtime_config: overrides.yaml
+    runtime_config:
+      file: overrides.yaml
     ```
 
     In the `overrides.yaml` file, add `unordered_writes` for each tenant

docs/sources/configure/index.template

@@ -152,7 +152,8 @@ place in the `limits_config` section:
 configure a runtime configuration file:
 
     ```
-    runtime_config: overrides.yaml
+    runtime_config:
+      file: overrides.yaml
     ```
 
     In the `overrides.yaml` file, add `unordered_writes` for each tenant

docs/sources/get-started/labels/structured-metadata.md

@@ -6,24 +6,22 @@ description: Describes how to enable structure metadata for logs and how to quer
 # What is structured metadata
 
 {{% admonition type="warning" %}}
-Structured metadata is an experimental feature and is subject to change in future releases of Grafana Loki.
+Structured metadata is an experimental feature and is subject to change in future releases of Grafana Loki. This feature is not yet available for Cloud Logs users.
 {{% /admonition %}}
 
 {{% admonition type="warning" %}}
 Structured metadata was added to chunk format V4 which is used if the schema version is greater or equal to `13`. (See [Schema Config]({{< relref "../../storage#schema-config" >}}) for more details about schema versions. )
 {{% /admonition %}}
 
-One of the powerful features of Loki is parsing logs at query time to extract metadata and build labels out of it.
-However, the parsing of logs at query time comes with a cost which can be significantly high for, as an example,
-large JSON blobs or a poorly written query using complex regex patterns.
+Selecting proper, low cardinality labels is critical to operating and querying Loki effectively. Some metadata, especially infrastructure related metadata, can be difficult to embed in log lines, and is too high cardinality to effectively store as indexed labels (and therefore reducing performance of the index).
 
-In addition, the data extracted from logs at query time is usually high cardinality, which can’t be stored
-in the index as it would increase the cardinality too much, and therefore reduce the performance of the index.
-
-Structured metadata is a way to attach metadata to logs without indexing them. Examples of useful metadata are
-trace IDs, user IDs, and any other label that is often used in queries but has high cardinality and is expensive
+Structured metadata is a way to attach metadata to logs without indexing them or including them in the log line content itself. Examples of useful metadata are
+kubernetes pod names, process ID's, or any other label that is often used in queries but has high cardinality and is expensive
 to extract at query time.
 
+Structured metadata can also be used to query commonly needed metadata from log lines without needing to apply a parser at query time. Large json blobs or a poorly written query using complex regex patterns, for example, come with a high performance cost. Examples of useful metadata include trace IDs or user IDs.
+
+
 ## Attaching structured metadata to log lines
 
 You have the option to attach structured metadata to log lines in the push payload along with each log line and the timestamp.
@@ -34,25 +32,37 @@ See the [Promtail: Structured metadata stage]({{< relref "../../send-data/promta
 
 With Loki version 1.2.0, support for structured metadata has been added to the Logstash output plugin. For more information, see [logstash]({{< relref "../../send-data/logstash/_index.md" >}}).
 
+{{% admonition type="warning" %}}
+There are defaults for how much structured metadata can be attached per log line.
+```
+# Maximum size accepted for structured metadata per log line.
+# CLI flag: -limits.max-structured-metadata-size
+[max_structured_metadata_size: <int> | default = 64KB]
+
+# Maximum number of structured metadata entries per log line.
+# CLI flag: -limits.max-structured-metadata-entries-count
+[max_structured_metadata_entries_count: <int> | default = 128]
+```
+{{% /admonition %}}
+
 ## Querying structured metadata
 
 Structured metadata is extracted automatically for each returned log line and added to the labels returned for the query.
 You can use labels of structured metadata to filter log line using a [label filter expression]({{< relref "../../query/log_queries#label-filter-expression" >}}).
 
-For example, if you have a label `trace_id` attached to some of your log lines as structured metadata, you can filter log lines using:
+For example, if you have a label `pod` attached to some of your log lines as structured metadata, you can filter log lines using:
 
 ```logql
-{job="example"} | trace_id="0242ac120002"
+{job="example"} | pod="myservice-abc1234-56789"`
 ```
 
 Of course, you can filter by multiple labels of structured metadata at the same time:
 
 ```logql
-{job="example"} | trace_id="0242ac120002" | user_id="superUser123"
+{job="example"} | pod="myservice-abc1234-56789" | trace_id="0242ac120002"
 ```
 
-Note that since structured metadata is extracted automatically to the results labels, some metric queries might return
-an error like `maximum of series (50000) reached for a single query`. You can use the [Keep]({{< relref "../../query/log_queries#keep-labels-expression" >}}) and [Drop]({{< relref "../../query/log_queries#drop-labels-expression" >}}) stages to filter out labels that you don't need.
+Note that since structured metadata is extracted automatically to the results labels, some metric queries might return an error like `maximum of series (50000) reached for a single query`. You can use the [Keep]({{< relref "../../query/log_queries#keep-labels-expression" >}}) and [Drop]({{< relref "../../query/log_queries#drop-labels-expression" >}}) stages to filter out labels that you don't need.
 For example:
 
 ```logql

docs/sources/send-data/promtail/configuration.md

@@ -1940,7 +1940,7 @@ users with thousands of services it can be more efficient to use the Consul API
 directly which has basic support for filtering nodes (currently by node
 metadata and a single tag).
 
-### docker_sd_config
+### docker_sd_configs
 
 Docker service discovery allows retrieving targets from a Docker daemon.
 It will only watch containers of the Docker daemon referenced with the host parameter. Docker

docs/sources/setup/install/helm/reference.md

@@ -856,7 +856,7 @@ false
 			<td>string</td>
 			<td></td>
 			<td><pre lang="json">
-"v1.8.4"
+"v1.8.6"
 </pre>
 </td>
 		</tr>

integration/loki_micro_services_test.go

@@ -1061,6 +1061,7 @@ func TestCategorizedLabels(t *testing.T) {
 }
 
 func TestBloomFiltersEndToEnd(t *testing.T) {
+	t.Skip("skipping until blooms have settled")
 	commonFlags := []string{
 		"-bloom-compactor.compaction-interval=10s",
 		"-bloom-compactor.enable-compaction=true",
@@ -1174,7 +1175,7 @@ func TestBloomFiltersEndToEnd(t *testing.T) {
 	)
 	require.NoError(t, clu.Run())
 
-	now := time.Date(2024, time.January, 19, 12, 0, 0, 0, time.UTC)
+	now := time.Now()
 
 	cliDistributor := client.New(tenantID, "", tDistributor.HTTPURL())
 	cliDistributor.Now = now
@@ -1224,8 +1225,15 @@ func TestBloomFiltersEndToEnd(t *testing.T) {
 		// verify metrics that observe usage of block for filtering
 		metrics, err := cliBloomCompactor.Metrics()
 		require.NoError(t, err)
-		successfulRunCount := getMetricValue(t, "loki_bloomcompactor_runs_completed_total", metrics)
-		t.Log("successful bloom compactor runs", successfulRunCount)
+		successfulRunCount, labels, err := extractMetric(`loki_bloomcompactor_runs_completed_total`, metrics)
+		if err != nil {
+			return false
+		}
+		t.Log("bloom compactor runs", successfulRunCount, labels)
+		if labels["status"] != "success" {
+			return false
+		}
+
 		return successfulRunCount == 1
 	}, 30*time.Second, time.Second)
 

operator/CHANGELOG.md

@@ -1,5 +1,7 @@
 ## Main
 
+- [11824](https://github.com/grafana/loki/pull/11824) **xperimental**: Improve messages for errors in storage secret
+- [11524](https://github.com/grafana/loki/pull/11524) **JoaoBraveCoding**, **periklis**: Add OpenShift cloud credentials support for AWS STS
 - [11513](https://github.com/grafana/loki/pull/11513) **btaani**: Add a custom metric that collects Lokistacks requiring a schema upgrade
 - [11718](https://github.com/grafana/loki/pull/11718) **periklis**: Upgrade k8s.io, sigs.k8s.io and openshift deps
 - [11671](https://github.com/grafana/loki/pull/11671) **JoaoBraveCoding**: Update mixins to fix structured metadata dashboards

operator/apis/config/v1/projectconfig_types.go

@@ -51,6 +51,13 @@ type OpenShiftFeatureGates struct {
 
 	// Dashboards enables the loki-mixin dashboards into the OpenShift Console
 	Dashboards bool `json:"dashboards,omitempty"`
+
+	// ManagedAuthEnv enabled when the operator installation is on OpenShift STS clusters.
+	ManagedAuthEnv bool
+}
+
+func (o OpenShiftFeatureGates) ManagedAuthEnabled() bool {
+	return o.Enabled && o.ManagedAuthEnv
 }
 
 // FeatureGates is the supported set of all operator feature gates.

operator/apis/loki/v1/lokistack_types.go

@@ -1062,6 +1062,12 @@ const (
 	ReasonMissingObjectStorageSecret LokiStackConditionReason = "MissingObjectStorageSecret"
 	// ReasonInvalidObjectStorageSecret when the format of the secret is invalid.
 	ReasonInvalidObjectStorageSecret LokiStackConditionReason = "InvalidObjectStorageSecret"
+	// ReasonMissingCredentialsRequest when the required request for managed auth credentials to object
+	// storage is missing.
+	ReasonMissingCredentialsRequest LokiStackConditionReason = "MissingCredentialsRequest"
+	// ReasonMissingManagedAuthSecret when the required secret for managed auth credentials to object
+	// storage is missing.
+	ReasonMissingManagedAuthSecret LokiStackConditionReason = "MissingManagedAuthenticationSecret"
 	// ReasonInvalidObjectStorageSchema when the spec contains an invalid schema(s).
 	ReasonInvalidObjectStorageSchema LokiStackConditionReason = "InvalidObjectStorageSchema"
 	// ReasonMissingObjectStorageCAConfigMap when the required configmap to verify object storage

operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml

@@ -157,7 +157,7 @@ metadata:
     features.operators.openshift.io/fips-compliant: "false"
     features.operators.openshift.io/proxy-aware: "true"
     features.operators.openshift.io/tls-profiles: "true"
-    features.operators.openshift.io/token-auth-aws: "false"
+    features.operators.openshift.io/token-auth-aws: "true"
     features.operators.openshift.io/token-auth-azure: "false"
     features.operators.openshift.io/token-auth-gcp: "false"
     operators.operatorframework.io/builder: operator-sdk-unknown
@@ -1463,6 +1463,16 @@ spec:
           - patch
           - update
           - watch
+        - apiGroups:
+          - cloudcredential.openshift.io
+          resources:
+          - credentialsrequests
+          verbs:
+          - create
+          - delete
+          - get
+          - list
+          - watch
         - apiGroups:
           - config.openshift.io
           resources:

operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml

@@ -1443,6 +1443,16 @@ spec:
           - patch
           - update
           - watch
+        - apiGroups:
+          - cloudcredential.openshift.io
+          resources:
+          - credentialsrequests
+          verbs:
+          - create
+          - delete
+          - get
+          - list
+          - watch
         - apiGroups:
           - config.openshift.io
           resources:

operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml

@@ -164,7 +164,7 @@ metadata:
     features.operators.openshift.io/fips-compliant: "false"
     features.operators.openshift.io/proxy-aware: "true"
     features.operators.openshift.io/tls-profiles: "true"
-    features.operators.openshift.io/token-auth-aws: "false"
+    features.operators.openshift.io/token-auth-aws: "true"
     features.operators.openshift.io/token-auth-azure: "false"
     features.operators.openshift.io/token-auth-gcp: "false"
     olm.skipRange: '>=5.7.0-0 <5.9.0'
@@ -1448,6 +1448,16 @@ spec:
           - patch
           - update
           - watch
+        - apiGroups:
+          - cloudcredential.openshift.io
+          resources:
+          - credentialsrequests
+          verbs:
+          - create
+          - delete
+          - get
+          - list
+          - watch
         - apiGroups:
           - config.openshift.io
           resources:

operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml

@@ -14,7 +14,7 @@ metadata:
     features.operators.openshift.io/fips-compliant: "false"
     features.operators.openshift.io/proxy-aware: "true"
     features.operators.openshift.io/tls-profiles: "true"
-    features.operators.openshift.io/token-auth-aws: "false"
+    features.operators.openshift.io/token-auth-aws: "true"
     features.operators.openshift.io/token-auth-azure: "false"
     features.operators.openshift.io/token-auth-gcp: "false"
     repository: https://github.com/grafana/loki/tree/main/operator

operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml

@@ -20,7 +20,7 @@ metadata:
     features.operators.openshift.io/fips-compliant: "false"
     features.operators.openshift.io/proxy-aware: "true"
     features.operators.openshift.io/tls-profiles: "true"
-    features.operators.openshift.io/token-auth-aws: "false"
+    features.operators.openshift.io/token-auth-aws: "true"
     features.operators.openshift.io/token-auth-azure: "false"
     features.operators.openshift.io/token-auth-gcp: "false"
     olm.skipRange: '>=5.7.0-0 <5.9.0'

operator/config/rbac/role.yaml

@@ -47,6 +47,16 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - cloudcredential.openshift.io
+  resources:
+  - credentialsrequests
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
 - apiGroups:
   - config.openshift.io
   resources: