From 48d2e18518e067eb79a277a7da0dfde5436bf83a Mon Sep 17 00:00:00 2001 From: Arthur Silva Sens Date: Wed, 28 Feb 2024 09:23:15 -0300 Subject: [PATCH] Write created lines when negotiating OpenMetrics (#504) * expfmt/openmetrics: Write created timestamps for counters, summaries and histograms * expfmt/encoder: Allow opt-in for OM created lines --------- Signed-off-by: Arthur Silva Sens --- expfmt/encode.go | 10 +++- expfmt/openmetrics_create.go | 91 ++++++++++++++++++++++++++++++- expfmt/openmetrics_create_test.go | 42 ++++++++++++-- 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/expfmt/encode.go b/expfmt/encode.go index 8fd80618..7f6cbe7d 100644 --- a/expfmt/encode.go +++ b/expfmt/encode.go @@ -139,7 +139,13 @@ func NegotiateIncludingOpenMetrics(h http.Header) Format { // interface is kept for backwards compatibility. // In cases where the Format does not allow for UTF-8 names, the global // NameEscapingScheme will be applied. -func NewEncoder(w io.Writer, format Format) Encoder { +// +// NewEncoder can be called with additional options to customize the OpenMetrics text output. +// For example: +// NewEncoder(w, FmtOpenMetrics_1_0_0, WithCreatedLines()) +// +// Extra options are ignored for all other formats. +func NewEncoder(w io.Writer, format Format, options ...EncoderOption) Encoder { escapingScheme := format.ToEscapingScheme() switch format.FormatType() { @@ -178,7 +184,7 @@ func NewEncoder(w io.Writer, format Format) Encoder { case TypeOpenMetrics: return encoderCloser{ encode: func(v *dto.MetricFamily) error { - _, err := MetricFamilyToOpenMetrics(w, model.EscapeMetricFamily(v, escapingScheme)) + _, err := MetricFamilyToOpenMetrics(w, model.EscapeMetricFamily(v, escapingScheme), options...) return err }, close: func() error { diff --git a/expfmt/openmetrics_create.go b/expfmt/openmetrics_create.go index ff12145a..63fc1f4d 100644 --- a/expfmt/openmetrics_create.go +++ b/expfmt/openmetrics_create.go @@ -22,11 +22,35 @@ import ( "strconv" "strings" + "google.golang.org/protobuf/types/known/timestamppb" + "github.com/prometheus/common/model" dto "github.com/prometheus/client_model/go" ) +type encoderOption struct { + withCreatedLines bool +} + +type EncoderOption func(*encoderOption) + +// WithCreatedLines is an EncoderOption that configures the OpenMetrics encoder +// to include _created lines (See +// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#counter-1). +// Created timestamps can improve the accuracy of series reset detection, but +// come with a bandwidth cost. +// +// At the time of writing, created timestamp ingestion is still experimental in +// Prometheus and need to be enabled with the feature-flag +// `--feature-flag=created-timestamp-zero-ingestion`, and breaking changes are +// still possible. Therefore, it is recommended to use this feature with caution. +func WithCreatedLines() EncoderOption { + return func(t *encoderOption) { + t.withCreatedLines = true + } +} + // MetricFamilyToOpenMetrics converts a MetricFamily proto message into the // OpenMetrics text format and writes the resulting lines to 'out'. It returns // the number of bytes written and any error encountered. The output will have @@ -64,15 +88,20 @@ import ( // its type will be set to `unknown` in that case to avoid invalid OpenMetrics // output. // -// - No support for the following (optional) features: `# UNIT` line, `_created` -// line, info type, stateset type, gaugehistogram type. +// - No support for the following (optional) features: `# UNIT` line, info type, +// stateset type, gaugehistogram type. // // - The size of exemplar labels is not checked (i.e. it's possible to create // exemplars that are larger than allowed by the OpenMetrics specification). // // - The value of Counters is not checked. (OpenMetrics doesn't allow counters // with a `NaN` value.) -func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int, err error) { +func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily, options ...EncoderOption) (written int, err error) { + toOM := encoderOption{} + for _, option := range options { + option(&toOM) + } + name := in.GetName() if name == "" { return 0, fmt.Errorf("MetricFamily has no name: %s", in) @@ -164,6 +193,7 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int return } + var createdTsBytesWritten int // Finally the samples, one line for each. for _, metric := range in.Metric { switch metricType { @@ -181,6 +211,10 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int metric.Counter.GetValue(), 0, false, metric.Counter.Exemplar, ) + if toOM.withCreatedLines && metric.Counter.CreatedTimestamp != nil { + createdTsBytesWritten, err = writeOpenMetricsCreated(w, name, "_total", metric, "", 0, metric.Counter.GetCreatedTimestamp()) + n += createdTsBytesWritten + } case dto.MetricType_GAUGE: if metric.Gauge == nil { return written, fmt.Errorf( @@ -235,6 +269,10 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int 0, metric.Summary.GetSampleCount(), true, nil, ) + if toOM.withCreatedLines && metric.Summary.CreatedTimestamp != nil { + createdTsBytesWritten, err = writeOpenMetricsCreated(w, name, "", metric, "", 0, metric.Summary.GetCreatedTimestamp()) + n += createdTsBytesWritten + } case dto.MetricType_HISTOGRAM: if metric.Histogram == nil { return written, fmt.Errorf( @@ -283,6 +321,10 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int 0, metric.Histogram.GetSampleCount(), true, nil, ) + if toOM.withCreatedLines && metric.Histogram.CreatedTimestamp != nil { + createdTsBytesWritten, err = writeOpenMetricsCreated(w, name, "", metric, "", 0, metric.Histogram.GetCreatedTimestamp()) + n += createdTsBytesWritten + } default: return written, fmt.Errorf( "unexpected type in metric %s %s", name, metric, @@ -473,6 +515,49 @@ func writeOpenMetricsNameAndLabelPairs( return written, nil } +// writeOpenMetricsCreated writes the created timestamp for a single time series +// following OpenMetrics text format to w, given the metric name, the metric proto +// message itself, optionally a suffix to be removed, e.g. '_total' for counters, +// an additional label name with a float64 value (use empty string as label name if +// not required) and the timestamp that represents the created timestamp. +// The function returns the number of bytes written and any error encountered. +func writeOpenMetricsCreated(w enhancedWriter, + name, suffixToTrim string, metric *dto.Metric, + additionalLabelName string, additionalLabelValue float64, + createdTimestamp *timestamppb.Timestamp, +) (int, error) { + written := 0 + n, err := writeOpenMetricsNameAndLabelPairs( + w, strings.TrimSuffix(name, suffixToTrim)+"_created", metric.Label, additionalLabelName, additionalLabelValue, + ) + written += n + if err != nil { + return written, err + } + + err = w.WriteByte(' ') + written++ + if err != nil { + return written, err + } + + // TODO(beorn7): Format this directly from components of ts to + // avoid overflow/underflow and precision issues of the float + // conversion. + n, err = writeOpenMetricsFloat(w, float64(createdTimestamp.AsTime().UnixNano())/1e9) + written += n + if err != nil { + return written, err + } + + err = w.WriteByte('\n') + written++ + if err != nil { + return written, err + } + return written, nil +} + // writeExemplar writes the provided exemplar in OpenMetrics format to w. The // function returns the number of bytes written and any error encountered. func writeExemplar(w enhancedWriter, e *dto.Exemplar) (int, error) { diff --git a/expfmt/openmetrics_create_test.go b/expfmt/openmetrics_create_test.go index db04de5e..c56ef22b 100644 --- a/expfmt/openmetrics_create_test.go +++ b/expfmt/openmetrics_create_test.go @@ -41,8 +41,9 @@ func TestCreateOpenMetrics(t *testing.T) { }() scenarios := []struct { - in *dto.MetricFamily - out string + in *dto.MetricFamily + options []EncoderOption + out string }{ // 0: Counter, timestamp given, no _total suffix. { @@ -306,6 +307,7 @@ unknown_name{name_1="value 1"} -1.23e-45 Value: proto.Float64(0), }, }, + CreatedTimestamp: openMetricsTimestamp, }, }, { @@ -336,10 +338,12 @@ unknown_name{name_1="value 1"} -1.23e-45 Value: proto.Float64(3), }, }, + CreatedTimestamp: openMetricsTimestamp, }, }, }, }, + options: []EncoderOption{WithCreatedLines()}, out: `# HELP summary_name summary docstring # TYPE summary_name summary summary_name{quantile="0.5"} -1.23 @@ -347,11 +351,13 @@ summary_name{quantile="0.9"} 0.2342354 summary_name{quantile="0.99"} 0.0 summary_name_sum -3.4567 summary_name_count 42 +summary_name_created 12345.6 summary_name{name_1="value 1",name_2="value 2",quantile="0.5"} 1.0 summary_name{name_1="value 1",name_2="value 2",quantile="0.9"} 2.0 summary_name{name_1="value 1",name_2="value 2",quantile="0.99"} 3.0 summary_name_sum{name_1="value 1",name_2="value 2"} 2010.1971 summary_name_count{name_1="value 1",name_2="value 2"} 4711 +summary_name_created{name_1="value 1",name_2="value 2"} 12345.6 `, }, // 7: Histogram @@ -387,10 +393,12 @@ summary_name_count{name_1="value 1",name_2="value 2"} 4711 CumulativeCount: proto.Uint64(2693), }, }, + CreatedTimestamp: openMetricsTimestamp, }, }, }, }, + options: []EncoderOption{WithCreatedLines()}, out: `# HELP request_duration_microseconds The response latency. # TYPE request_duration_microseconds histogram request_duration_microseconds_bucket{le="100.0"} 123 @@ -400,6 +408,7 @@ request_duration_microseconds_bucket{le="172.8"} 1524 request_duration_microseconds_bucket{le="+Inf"} 2693 request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 +request_duration_microseconds_created 12345.6 `, }, // 8: Histogram with missing +Inf bucket. @@ -522,7 +531,30 @@ request_duration_microseconds_count 2693 Metric: []*dto.Metric{ { Counter: &dto.Counter{ - Value: proto.Float64(42), + Value: proto.Float64(42), + CreatedTimestamp: openMetricsTimestamp, + }, + }, + }, + }, + options: []EncoderOption{WithCreatedLines()}, + out: `# HELP foos Number of foos. +# TYPE foos counter +foos_total 42.0 +foos_created 12345.6 +`, + }, + // 11: Simple Counter without created line. + { + in: &dto.MetricFamily{ + Name: proto.String("foos_total"), + Help: proto.String("Number of foos."), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(42), + CreatedTimestamp: openMetricsTimestamp, }, }, }, @@ -532,7 +564,7 @@ request_duration_microseconds_count 2693 foos_total 42.0 `, }, - // 11: No metric. + // 12: No metric. { in: &dto.MetricFamily{ Name: proto.String("name_total"), @@ -573,7 +605,7 @@ foos_total 42.0 for i, scenario := range scenarios { out := bytes.NewBuffer(make([]byte, 0, len(scenario.out))) - n, err := MetricFamilyToOpenMetrics(out, scenario.in) + n, err := MetricFamilyToOpenMetrics(out, scenario.in, scenario.options...) if err != nil { t.Errorf("%d. error: %s", i, err) continue