Fix: truncate long label values

We generate some label values, so they are not being subject to the
constraints the labels provided by the user are.

In order to prevent the metrics from being dropped by Prometheus because
the label values exceed the limit, truncate the values that exceed the
limit, and replace the last three bytes by '...' to indicate that
truncation took place.

For the specific case of certificates with a really long list of Subject
Alternate Names (SAN), they end up in a label. It's a list that looks
like `a.example.org,b.example.org,c.example.org`. We _could_ split that
list and push each individual value as a separate label value just to
avoid hitting the limit. The problem with that approach is that we have
seen certificates with over 700+ SANs in the wild, so this strategy would
push out 700+ metrics.

Signed-off-by: Marcelo E. Magallon <marcelo.magallon@grafana.com>

internal/scraper/scraper.go

@@ -30,6 +30,7 @@ const (
 	ProbeSuccessMetricName = "probe_success"
 	CheckInfoMetricName    = "sm_check_info"
 	CheckInfoSource        = "synthetic-monitoring-agent"
+	maxLabelValueLength    = 2048 // this is the default value in Prometheus
 )
 
 var (
@@ -597,7 +598,7 @@ func (s Scraper) extractTimeseries(t time.Time, metrics []*dto.MetricFamily, sha
 func extractTimeseries(t time.Time, metrics []*dto.MetricFamily, sharedLabels []labelPair, summaries map[uint64]prometheus.Summary, histograms map[uint64]prometheus.Histogram, logger zerolog.Logger) TimeSeries {
 	metricLabels := make([]prompb.Label, 0, len(sharedLabels))
 	for _, label := range sharedLabels {
-		metricLabels = append(metricLabels, prompb.Label{Name: label.name, Value: label.value})
+		metricLabels = append(metricLabels, prompb.Label{Name: label.name, Value: truncateLabelValue(label.value)})
 	}
 
 	var ts []prompb.TimeSeries
@@ -678,7 +679,7 @@ func appendDtoToTimeseries(ts []prompb.TimeSeries, t time.Time, mName string, sh
 	labels = append(labels, prompb.Label{Name: "__name__", Value: mName})
 	labels = append(labels, sharedLabels...)
 	for _, l := range ml {
-		labels = append(labels, prompb.Label{Name: *(l.Name), Value: *(l.Value)})
+		labels = append(labels, prompb.Label{Name: *(l.Name), Value: truncateLabelValue(*(l.Value))})
 	}
 
 	switch mType {
@@ -896,3 +897,15 @@ func withCheckID(ctx zerolog.Context, checkID int64) zerolog.Context {
 	}
 	return ctx.Int64("check_id", checkID)
 }
+
+func truncateLabelValue(str string) string {
+	if len(str) > maxLabelValueLength {
+		b := []byte(str[:maxLabelValueLength])
+		for i := maxLabelValueLength - 3; i < maxLabelValueLength; i++ {
+			b[i] = '.'
+		}
+		str = string(b)
+	}
+
+	return str
+}

internal/scraper/scraper_test.go

@@ -1171,3 +1171,52 @@ func mergeMaps(maps ...map[string]string) map[string]string {
 	}
 	return out
 }
+
+func TestTruncateLabelValue(t *testing.T) {
+	testcases := map[string]struct {
+		length         int
+		expectedLength int
+	}{
+		"zero": {
+			length:         0,
+			expectedLength: 0,
+		},
+		"one": {
+			length:         1,
+			expectedLength: 1,
+		},
+		"max/2": {
+			length:         maxLabelValueLength / 2,
+			expectedLength: maxLabelValueLength / 2,
+		},
+		"max-1": {
+			length:         maxLabelValueLength - 1,
+			expectedLength: maxLabelValueLength - 1,
+		},
+		"max": {
+			length:         maxLabelValueLength,
+			expectedLength: maxLabelValueLength,
+		},
+		"max+1": {
+			length:         maxLabelValueLength + 1,
+			expectedLength: maxLabelValueLength,
+		},
+		"2*max": {
+			length:         2 * maxLabelValueLength,
+			expectedLength: maxLabelValueLength,
+		},
+	}
+
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			input := strings.Repeat("a", tc.length)
+			expected := strings.Repeat("a", tc.expectedLength)
+			actual := truncateLabelValue(input)
+			require.Equal(t, len(expected), len(actual))
+			if tc.expectedLength < tc.length {
+				require.Equal(t, expected[:len(expected)-3], actual[:len(actual)-3])
+				require.Equal(t, "...", actual[len(actual)-3:])
+			}
+		})
+	}
+}