Add constrained labels and Constrained variant for all MetricVecs (#1151

)

* Introduce MetricVecOpts and add constraints to VariableLabels

MetricVecOpts exposes options specific to MetricVec initialisation. The
first option exposed by MetricVecOpts are constraints on VariableLabels,
allowing restrictions on the possible values a label can take, to
prevent cardinality explosion when the label value comes from a
non-trusted source (as a user input or HTTP header).

Signed-off-by: Quentin Devos <4972091+Okhoshi@users.noreply.github.com>

* Add tests

Signed-off-by: Quentin Devos <4972091+Okhoshi@users.noreply.github.com>

Signed-off-by: Quentin Devos <4972091+Okhoshi@users.noreply.github.com>

prometheus/counter.go

@@ -59,6 +59,18 @@ type ExemplarAdder interface {
 // CounterOpts is an alias for Opts. See there for doc comments.
 type CounterOpts Opts
 
+// CounterVecOpts bundles the options to create a CounterVec metric.
+// It is mandatory to set CounterOpts, see there for mandatory fields. VariableLabels
+// is optional and can safely be left to its default value.
+type CounterVecOpts struct {
+	CounterOpts
+
+	// VariableLabels are used to partition the metric vector by the given set
+	// of labels. Each label value will be constrained with the optional Contraint
+	// function, if provided.
+	VariableLabels ConstrainableLabels
+}
+
 // NewCounter creates a new Counter based on the provided CounterOpts.
 //
 // The returned implementation also implements ExemplarAdder. It is safe to
@@ -174,16 +186,24 @@ type CounterVec struct {
 // NewCounterVec creates a new CounterVec based on the provided CounterOpts and
 // partitioned by the given label names.
 func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec {
-	desc := NewDesc(
+	return V2.NewCounterVec(CounterVecOpts{
+		CounterOpts:    opts,
+		VariableLabels: UnconstrainedLabels(labelNames),
+	})
+}
+
+// NewCounterVec creates a new CounterVec based on the provided CounterVecOpts.
+func (v2) NewCounterVec(opts CounterVecOpts) *CounterVec {
+	desc := V2.NewDesc(
 		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
 		opts.Help,
-		labelNames,
+		opts.VariableLabels,
 		opts.ConstLabels,
 	)
 	return &CounterVec{
 		MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
 			if len(lvs) != len(desc.variableLabels) {
-				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs))
+				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.labelNames(), lvs))
 			}
 			result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: time.Now}
 			result.init(result) // Init self-collection.

prometheus/counter_test.go

@@ -102,7 +102,7 @@ func TestCounterVecGetMetricWithInvalidLabelValues(t *testing.T) {
 			Name: "test",
 		}, []string{"a"})
 
-		labelValues := make([]string, len(test.labels))
+		labelValues := make([]string, 0, len(test.labels))
 		for _, val := range test.labels {
 			labelValues = append(labelValues, val)
 		}

prometheus/desc.go

@@ -53,9 +53,9 @@ type Desc struct {
 	// constLabelPairs contains precalculated DTO label pairs based on
 	// the constant labels.
 	constLabelPairs []*dto.LabelPair
-	// variableLabels contains names of labels for which the metric
-	// maintains variable values.
-	variableLabels []string
+	// variableLabels contains names of labels and normalization function for
+	// which the metric maintains variable values.
+	variableLabels ConstrainedLabels
 	// id is a hash of the values of the ConstLabels and fqName. This
 	// must be unique among all registered descriptors and can therefore be
 	// used as an identifier of the descriptor.
@@ -79,10 +79,24 @@ type Desc struct {
 // For constLabels, the label values are constant. Therefore, they are fully
 // specified in the Desc. See the Collector example for a usage pattern.
 func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *Desc {
+	return V2.NewDesc(fqName, help, UnconstrainedLabels(variableLabels), constLabels)
+}
+
+// NewDesc allocates and initializes a new Desc. Errors are recorded in the Desc
+// and will be reported on registration time. variableLabels and constLabels can
+// be nil if no such labels should be set. fqName must not be empty.
+//
+// variableLabels only contain the label names and normalization functions. Their
+// label values are variable and therefore not part of the Desc. (They are managed
+// within the Metric.)
+//
+// For constLabels, the label values are constant. Therefore, they are fully
+// specified in the Desc. See the Collector example for a usage pattern.
+func (v2) NewDesc(fqName, help string, variableLabels ConstrainableLabels, constLabels Labels) *Desc {
 	d := &Desc{
 		fqName:         fqName,
 		help:           help,
-		variableLabels: variableLabels,
+		variableLabels: variableLabels.constrainedLabels(),
 	}
 	if !model.IsValidMetricName(model.LabelValue(fqName)) {
 		d.err = fmt.Errorf("%q is not a valid metric name", fqName)
@@ -92,7 +106,7 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *
 	// their sorted label names) plus the fqName (at position 0).
 	labelValues := make([]string, 1, len(constLabels)+1)
 	labelValues[0] = fqName
-	labelNames := make([]string, 0, len(constLabels)+len(variableLabels))
+	labelNames := make([]string, 0, len(constLabels)+len(d.variableLabels))
 	labelNameSet := map[string]struct{}{}
 	// First add only the const label names and sort them...
 	for labelName := range constLabels {
@@ -117,13 +131,13 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *
 	// Now add the variable label names, but prefix them with something that
 	// cannot be in a regular label name. That prevents matching the label
 	// dimension with a different mix between preset and variable labels.
-	for _, labelName := range variableLabels {
-		if !checkLabelName(labelName) {
-			d.err = fmt.Errorf("%q is not a valid label name for metric %q", labelName, fqName)
+	for _, label := range d.variableLabels {
+		if !checkLabelName(label.Name) {
+			d.err = fmt.Errorf("%q is not a valid label name for metric %q", label.Name, fqName)
 			return d
 		}
-		labelNames = append(labelNames, "$"+labelName)
-		labelNameSet[labelName] = struct{}{}
+		labelNames = append(labelNames, "$"+label.Name)
+		labelNameSet[label.Name] = struct{}{}
 	}
 	if len(labelNames) != len(labelNameSet) {
 		d.err = fmt.Errorf("duplicate label names in constant and variable labels for metric %q", fqName)

prometheus/examples_test.go

@@ -294,9 +294,9 @@ func ExampleRegister() {
 
 	// Output:
 	// taskCounter registered.
-	// taskCounterVec not registered: a previously registered descriptor with the same fully-qualified name as Desc{fqName: "worker_pool_completed_tasks_total", help: "Total number of tasks completed.", constLabels: {}, variableLabels: [worker_id]} has different label names or a different help string
+	// taskCounterVec not registered: a previously registered descriptor with the same fully-qualified name as Desc{fqName: "worker_pool_completed_tasks_total", help: "Total number of tasks completed.", constLabels: {}, variableLabels: [{worker_id <nil>}]} has different label names or a different help string
 	// taskCounter unregistered.
-	// taskCounterVec not registered: a previously registered descriptor with the same fully-qualified name as Desc{fqName: "worker_pool_completed_tasks_total", help: "Total number of tasks completed.", constLabels: {}, variableLabels: [worker_id]} has different label names or a different help string
+	// taskCounterVec not registered: a previously registered descriptor with the same fully-qualified name as Desc{fqName: "worker_pool_completed_tasks_total", help: "Total number of tasks completed.", constLabels: {}, variableLabels: [{worker_id <nil>}]} has different label names or a different help string
 	// taskCounterVec registered.
 	// Worker initialization failed: inconsistent label cardinality: expected 1 label values but got 2 in []string{"42", "spurious arg"}
 	// notMyCounter is nil.

prometheus/gauge.go

@@ -55,6 +55,18 @@ type Gauge interface {
 // GaugeOpts is an alias for Opts. See there for doc comments.
 type GaugeOpts Opts
 
+// GaugeVecOpts bundles the options to create a GaugeVec metric.
+// It is mandatory to set GaugeOpts, see there for mandatory fields. VariableLabels
+// is optional and can safely be left to its default value.
+type GaugeVecOpts struct {
+	GaugeOpts
+
+	// VariableLabels are used to partition the metric vector by the given set
+	// of labels. Each label value will be constrained with the optional Contraint
+	// function, if provided.
+	VariableLabels ConstrainableLabels
+}
+
 // NewGauge creates a new Gauge based on the provided GaugeOpts.
 //
 // The returned implementation is optimized for a fast Set method. If you have a
@@ -138,16 +150,24 @@ type GaugeVec struct {
 // NewGaugeVec creates a new GaugeVec based on the provided GaugeOpts and
 // partitioned by the given label names.
 func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec {
-	desc := NewDesc(
+	return V2.NewGaugeVec(GaugeVecOpts{
+		GaugeOpts:      opts,
+		VariableLabels: UnconstrainedLabels(labelNames),
+	})
+}
+
+// NewGaugeVec creates a new GaugeVec based on the provided GaugeVecOpts.
+func (v2) NewGaugeVec(opts GaugeVecOpts) *GaugeVec {
+	desc := V2.NewDesc(
 		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
 		opts.Help,
-		labelNames,
+		opts.VariableLabels,
 		opts.ConstLabels,
 	)
 	return &GaugeVec{
 		MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
 			if len(lvs) != len(desc.variableLabels) {
-				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs))
+				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.labelNames(), lvs))
 			}
 			result := &gauge{desc: desc, labelPairs: MakeLabelPairs(desc, lvs)}
 			result.init(result) // Init self-collection.

prometheus/histogram.go

@@ -469,6 +469,18 @@ type HistogramOpts struct {
 	NativeHistogramMaxZeroThreshold float64
 }
 
+// HistogramVecOpts bundles the options to create a HistogramVec metric.
+// It is mandatory to set HistogramOpts, see there for mandatory fields. VariableLabels
+// is optional and can safely be left to its default value.
+type HistogramVecOpts struct {
+	HistogramOpts
+
+	// VariableLabels are used to partition the metric vector by the given set
+	// of labels. Each label value will be constrained with the optional Contraint
+	// function, if provided.
+	VariableLabels ConstrainableLabels
+}
+
 // NewHistogram creates a new Histogram based on the provided HistogramOpts. It
 // panics if the buckets in HistogramOpts are not in strictly increasing order.
 //
@@ -489,11 +501,11 @@ func NewHistogram(opts HistogramOpts) Histogram {
 
 func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogram {
 	if len(desc.variableLabels) != len(labelValues) {
-		panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, labelValues))
+		panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.labelNames(), labelValues))
 	}
 
 	for _, n := range desc.variableLabels {
-		if n == bucketLabel {
+		if n.Name == bucketLabel {
 			panic(errBucketLabelNotAllowed)
 		}
 	}
@@ -1030,15 +1042,23 @@ type HistogramVec struct {
 // NewHistogramVec creates a new HistogramVec based on the provided HistogramOpts and
 // partitioned by the given label names.
 func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec {
-	desc := NewDesc(
+	return V2.NewHistogramVec(HistogramVecOpts{
+		HistogramOpts:  opts,
+		VariableLabels: UnconstrainedLabels(labelNames),
+	})
+}
+
+// NewHistogramVec creates a new HistogramVec based on the provided HistogramVecOpts.
+func (v2) NewHistogramVec(opts HistogramVecOpts) *HistogramVec {
+	desc := V2.NewDesc(
 		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
 		opts.Help,
-		labelNames,
+		opts.VariableLabels,
 		opts.ConstLabels,
 	)
 	return &HistogramVec{
 		MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
-			return newHistogram(desc, opts, lvs...)
+			return newHistogram(desc, opts.HistogramOpts, lvs...)
 		}),
 	}
 }

prometheus/labels.go

@@ -32,6 +32,78 @@ import (
 // create a Desc.
 type Labels map[string]string
 
+// ConstrainedLabels represents a label name and its constrain function
+// to normalize label values. This type is commonly used when constructing
+// metric vector Collectors.
+type ConstrainedLabel struct {
+	Name       string
+	Constraint func(string) string
+}
+
+func (cl ConstrainedLabel) Constrain(v string) string {
+	if cl.Constraint == nil {
+		return v
+	}
+	return cl.Constraint(v)
+}
+
+// ConstrainableLabels is an interface that allows creating of labels that can
+// be optionally constrained.
+//
+//	prometheus.V2().NewCounterVec(CounterVecOpts{
+//	  CounterOpts: {...}, // Usual CounterOpts fields
+//	  VariableLabels: []ConstrainedLabels{
+//	    {Name: "A"},
+//	    {Name: "B", Constraint: func(v string) string { ... }},
+//	  },
+//	})
+type ConstrainableLabels interface {
+	constrainedLabels() ConstrainedLabels
+	labelNames() []string
+}
+
+// ConstrainedLabels represents a collection of label name -> constrain function
+// to normalize label values. This type is commonly used when constructing
+// metric vector Collectors.
+type ConstrainedLabels []ConstrainedLabel
+
+func (cls ConstrainedLabels) constrainedLabels() ConstrainedLabels {
+	return cls
+}
+
+func (cls ConstrainedLabels) labelNames() []string {
+	names := make([]string, len(cls))
+	for i, label := range cls {
+		names[i] = label.Name
+	}
+	return names
+}
+
+// UnconstrainedLabels represents collection of label without any constraint on
+// their value. Thus, it is simply a collection of label names.
+//
+//	UnconstrainedLabels([]string{ "A", "B" })
+//
+// is equivalent to
+//
+//	ConstrainedLabels {
+//	  { Name: "A" },
+//	  { Name: "B" },
+//	}
+type UnconstrainedLabels []string
+
+func (uls UnconstrainedLabels) constrainedLabels() ConstrainedLabels {
+	constrainedLabels := make([]ConstrainedLabel, len(uls))
+	for i, l := range uls {
+		constrainedLabels[i] = ConstrainedLabel{Name: l}
+	}
+	return constrainedLabels
+}
+
+func (uls UnconstrainedLabels) labelNames() []string {
+	return uls
+}
+
 // reservedLabelPrefix is a prefix which is not legal in user-supplied
 // label names.
 const reservedLabelPrefix = "__"

prometheus/registry.go

@@ -962,7 +962,7 @@ func checkDescConsistency(
 	copy(lpsFromDesc, desc.constLabelPairs)
 	for _, l := range desc.variableLabels {
 		lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{
-			Name: proto.String(l),
+			Name: proto.String(l.Name),
 		})
 	}
 	if len(lpsFromDesc) != len(dtoMetric.Label) {

prometheus/summary.go

@@ -148,6 +148,18 @@ type SummaryOpts struct {
 	BufCap uint32
 }
 
+// SummaryVecOpts bundles the options to create a SummaryVec metric.
+// It is mandatory to set SummaryOpts, see there for mandatory fields. VariableLabels
+// is optional and can safely be left to its default value.
+type SummaryVecOpts struct {
+	SummaryOpts
+
+	// VariableLabels are used to partition the metric vector by the given set
+	// of labels. Each label value will be constrained with the optional Contraint
+	// function, if provided.
+	VariableLabels ConstrainableLabels
+}
+
 // Problem with the sliding-window decay algorithm... The Merge method of
 // perk/quantile is actually not working as advertised - and it might be
 // unfixable, as the underlying algorithm is apparently not capable of merging
@@ -178,11 +190,11 @@ func NewSummary(opts SummaryOpts) Summary {
 
 func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
 	if len(desc.variableLabels) != len(labelValues) {
-		panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, labelValues))
+		panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.labelNames(), labelValues))
 	}
 
 	for _, n := range desc.variableLabels {
-		if n == quantileLabel {
+		if n.Name == quantileLabel {
 			panic(errQuantileLabelNotAllowed)
 		}
 	}
@@ -530,20 +542,28 @@ type SummaryVec struct {
 // it is handled by the Prometheus server internally, “quantile” is an illegal
 // label name. NewSummaryVec will panic if this label name is used.
 func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec {
-	for _, ln := range labelNames {
+	return V2.NewSummaryVec(SummaryVecOpts{
+		SummaryOpts:    opts,
+		VariableLabels: UnconstrainedLabels(labelNames),
+	})
+}
+
+// NewSummaryVec creates a new SummaryVec based on the provided SummaryVecOpts.
+func (v2) NewSummaryVec(opts SummaryVecOpts) *SummaryVec {
+	for _, ln := range opts.VariableLabels.labelNames() {
 		if ln == quantileLabel {
 			panic(errQuantileLabelNotAllowed)
 		}
 	}
-	desc := NewDesc(
+	desc := V2.NewDesc(
 		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
 		opts.Help,
-		labelNames,
+		opts.VariableLabels,
 		opts.ConstLabels,
 	)
 	return &SummaryVec{
 		MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
-			return newSummary(desc, opts, lvs...)
+			return newSummary(desc, opts.SummaryOpts, lvs...)
 		}),
 	}
 }