Promtail: allow for customisable stream lag labels (#4507)

* Allow for customisable stream lag labels Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Updating test Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Allowing for multiple labels Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Adding field to docs Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
grafana · Oct 21, 2021 · 703d68b · 703d68b
1 parent cc1d478
commit 703d68b
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 13 deletions.
diff --git a/clients/pkg/promtail/client/client.go b/clients/pkg/promtail/client/client.go
@@ -320,14 +320,21 @@ func (c *client) sendBatch(tenantID string, batch *batch) {
 				}
 				var lblSet model.LabelSet
 				for i := range lbls {
-					if lbls[i].Name == LatencyLabel {
-						lblSet = model.LabelSet{
-							model.LabelName(HostLabel):    model.LabelValue(c.cfg.URL.Host),
-							model.LabelName(LatencyLabel): model.LabelValue(lbls[i].Value),
+					for _, lbl := range c.cfg.StreamLagLabels {
+						if lbls[i].Name == lbl {
+							if lblSet == nil {
+								lblSet = model.LabelSet{}
+							}
+
+							lblSet = lblSet.Merge(model.LabelSet{
+								model.LabelName(lbl): model.LabelValue(lbls[i].Value),
+							})
 						}
 					}
 				}
 				if lblSet != nil {
+					// always set host
+					lblSet = lblSet.Merge(model.LabelSet{model.LabelName(HostLabel): model.LabelValue(c.cfg.URL.Host)})
 					c.metrics.streamLag.With(lblSet).Set(time.Since(s.Entries[len(s.Entries)-1].Timestamp).Seconds())
 				}
 			}

diff --git a/clients/pkg/promtail/client/config.go b/clients/pkg/promtail/client/config.go
@@ -37,6 +37,8 @@ type Config struct {
 	// The tenant ID to use when pushing logs to Loki (empty string means
 	// single tenant mode)
 	TenantID string `yaml:"tenant_id"`
+
+	StreamLagLabels flagext.StringSliceCSV `yaml:"stream_lag_labels"`
 }
 
 // RegisterFlags with prefix registers flags where every name is prefixed by
@@ -53,6 +55,9 @@ func (c *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.Var(&c.ExternalLabels, prefix+"client.external-labels", "list of external labels to add to each log (e.g: --client.external-labels=lb1=v1,lb2=v2)")
 
 	f.StringVar(&c.TenantID, prefix+"client.tenant-id", "", "Tenant ID to use when pushing logs to Loki.")
+
+	c.StreamLagLabels = []string{"filename"}
+	f.Var(&c.StreamLagLabels, prefix+"client.stream-lag-labels", "Comma-separated list of labels to use when calculating stream lag")
 }
 
 // RegisterFlags registers flags.
@@ -75,9 +80,10 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 				MaxRetries: MaxRetries,
 				MinBackoff: MinBackoff,
 			},
-			BatchSize: BatchSize,
-			BatchWait: BatchWait,
-			Timeout:   Timeout,
+			BatchSize:       BatchSize,
+			BatchWait:       BatchWait,
+			Timeout:         Timeout,
+			StreamLagLabels: []string{"filename"},
 		}
 	}
 

diff --git a/clients/pkg/promtail/client/config_test.go b/clients/pkg/promtail/client/config_test.go
@@ -48,9 +48,10 @@ func Test_Config(t *testing.T) {
 					MaxRetries: MaxRetries,
 					MinBackoff: MinBackoff,
 				},
-				BatchSize: BatchSize,
-				BatchWait: BatchWait,
-				Timeout:   Timeout,
+				BatchSize:       BatchSize,
+				BatchWait:       BatchWait,
+				Timeout:         Timeout,
+				StreamLagLabels: []string{"filename"},
 			},
 		},
 		{
@@ -64,9 +65,10 @@ func Test_Config(t *testing.T) {
 					MaxRetries: 20,
 					MinBackoff: 5 * time.Second,
 				},
-				BatchSize: 100 * 2048,
-				BatchWait: 5 * time.Second,
-				Timeout:   5 * time.Second,
+				BatchSize:       100 * 2048,
+				BatchWait:       5 * time.Second,
+				Timeout:         5 * time.Second,
+				StreamLagLabels: []string{"filename"},
 			},
 		},
 	}

diff --git a/docs/sources/clients/promtail/configuration.md b/docs/sources/clients/promtail/configuration.md
@@ -273,6 +273,12 @@ external_labels:
 
 # Maximum time to wait for a server to respond to a request
 [timeout: <duration> | default = 10s]
+
+# A comma-separated list of labels to include in the stream lag metric `promtail_stream_lag_seconds`.
+# The default value is "filename". A "host" label is always included.
+# The stream lag metric indicates which streams are falling behind on writes to Loki;
+# be mindful about not using too many labels here as it can explode cardinality.
+[stream_lag_labels: <string> | default = "filename"]
 ```
 
 ## positions