diff --git a/README.md b/README.md index b5d0fab8..d1a8de3d 100644 --- a/README.md +++ b/README.md @@ -49,9 +49,10 @@ can be used as a reference for setup: The matrix below lists the versions of Prometheus Server and other dependencies that have been qualified to work with releases of `stackdriver-prometheus-sidecar`. -| sidecar version | prometheus version | -| --- | --- | -| 0.2.x | 2.4.3 | +| sidecar version | **Prometheus 2.4.3** | **Prometheus 2.5.x** | +|------------|------------------|-------------------| +| **0.2.x** | ✓ | - | +| **master** | ✓ | ✓ | ## Source Code Headers diff --git a/cmd/stackdriver-prometheus-sidecar/main.go b/cmd/stackdriver-prometheus-sidecar/main.go index a9bdf705..691c4a69 100644 --- a/cmd/stackdriver-prometheus-sidecar/main.go +++ b/cmd/stackdriver-prometheus-sidecar/main.go @@ -613,8 +613,11 @@ func parseConfigFile(filename string) (map[string]string, []scrape.MetricMetadat var staticMetadata []scrape.MetricMetadata for _, sm := range fc.StaticMetadata { switch sm.Type { - case textparse.MetricTypeCounter, textparse.MetricTypeGauge, - textparse.MetricTypeHistogram, textparse.MetricTypeSummary, textparse.MetricTypeUntyped: + case metadata.MetricTypeUntyped: + // Convert "untyped" to the "unknown" type used internally as of Prometheus 2.5. + sm.Type = textparse.MetricTypeUnknown + case textparse.MetricTypeCounter, textparse.MetricTypeGauge, textparse.MetricTypeHistogram, + textparse.MetricTypeSummary, textparse.MetricTypeUnknown: default: return nil, nil, errors.Errorf("invalid metric type %q", sm.Type) } diff --git a/metadata/cache.go b/metadata/cache.go index 36b0ac1a..9015d082 100644 --- a/metadata/cache.go +++ b/metadata/cache.go @@ -46,6 +46,10 @@ type Cache struct { // the target metadata endpoint. const DefaultEndpointPath = "/api/v1/targets/metadata" +// The old metric type value for textparse.MetricTypeUnknown that is used in +// Prometheus 2.4 and earlier. +const MetricTypeUntyped = "untyped" + // NewCache returns a new cache that gets populated by the metadata endpoint // at the given URL. // It uses the default endpoint path if no specific path is provided. @@ -176,6 +180,10 @@ func (c *Cache) fetchMetric(ctx context.Context, job, instance, metric string) ( } d := apiResp.Data[0] + // Convert legacy "untyped" type used before Prometheus 2.5. + if d.Type == MetricTypeUntyped { + d.Type = textparse.MetricTypeUnknown + } return &metadataEntry{ MetricMetadata: scrape.MetricMetadata{ Metric: metric, @@ -212,6 +220,10 @@ func (c *Cache) fetchBatch(ctx context.Context, job, instance string) (map[strin result := make(map[string]*metadataEntry, len(apiResp.Data)+len(internalMetrics)) for _, md := range apiResp.Data { + // Convert legacy "untyped" type used before Prometheus 2.5. + if md.Type == MetricTypeUntyped { + md.Type = textparse.MetricTypeUnknown + } result[md.Metric] = &metadataEntry{ MetricMetadata: scrape.MetricMetadata{ Metric: md.Metric, diff --git a/metadata/cache_test.go b/metadata/cache_test.go index 34cec44d..89a45ed7 100644 --- a/metadata/cache_test.go +++ b/metadata/cache_test.go @@ -35,7 +35,8 @@ func TestCache_Get(t *testing.T) { {Metric: "metric2", Type: textparse.MetricTypeGauge, Help: "help_metric2"}, {Metric: "metric3", Type: textparse.MetricTypeHistogram, Help: "help_metric3"}, {Metric: "metric4", Type: textparse.MetricTypeSummary, Help: "help_metric4"}, - {Metric: "metric5", Type: textparse.MetricTypeUntyped, Help: "help_metric5"}, + {Metric: "metric5", Type: textparse.MetricTypeUnknown, Help: "help_metric5"}, + {Metric: "metric6", Type: MetricTypeUntyped, Help: "help_metric6"}, } var handler func(qMetric, qMatch string) *apiResponse @@ -129,6 +130,22 @@ func TestCache_Get(t *testing.T) { } expect(metrics[4], md) + // Test "untyped" metric type from Prometheus 2.4. + handler = func(qMetric, qMatch string) *apiResponse { + if qMetric != "metric6" { + t.Fatalf("unexpected metric %q in request", qMetric) + } + if qMatch != `{job="prometheus",instance="localhost:9090"}` { + t.Fatalf("unexpected matcher %q in request", qMatch) + } + return &apiResponse{Status: "success", Data: metrics[5:6]} + } + md, err = c.Get(ctx, "prometheus", "localhost:9090", "metric6") + if err != nil { + t.Fatal(err) + } + expect(apiMetadata{Metric: "metric6", Type: textparse.MetricTypeUnknown, Help: "help_metric6"}, md) + // The scrape layer's metrics should not fire off requests. md, err = c.Get(ctx, "prometheus", "localhost:9090", "up") if err != nil { diff --git a/retrieval/series_cache.go b/retrieval/series_cache.go index a4f528eb..4dfe33fd 100644 --- a/retrieval/series_cache.go +++ b/retrieval/series_cache.go @@ -407,7 +407,7 @@ func (c *seriesCache) refresh(ctx context.Context, ref uint64) error { case textparse.MetricTypeCounter: ts.MetricKind = metric_pb.MetricDescriptor_CUMULATIVE ts.ValueType = metric_pb.MetricDescriptor_DOUBLE - case textparse.MetricTypeGauge, textparse.MetricTypeUntyped: + case textparse.MetricTypeGauge, textparse.MetricTypeUnknown: ts.MetricKind = metric_pb.MetricDescriptor_GAUGE ts.ValueType = metric_pb.MetricDescriptor_DOUBLE case textparse.MetricTypeSummary: diff --git a/retrieval/transform.go b/retrieval/transform.go index ce47a932..58eec3c8 100644 --- a/retrieval/transform.go +++ b/retrieval/transform.go @@ -70,7 +70,7 @@ func (b *sampleBuilder) next(ctx context.Context, samples []tsdb.RefSample) (*mo point.Interval.StartTime = getTimestamp(resetTimestamp) point.Value = &monitoring_pb.TypedValue{&monitoring_pb.TypedValue_DoubleValue{v}} - case textparse.MetricTypeGauge, textparse.MetricTypeUntyped: + case textparse.MetricTypeGauge, textparse.MetricTypeUnknown: point.Value = &monitoring_pb.TypedValue{&monitoring_pb.TypedValue_DoubleValue{sample.V}} case textparse.MetricTypeSummary: diff --git a/retrieval/transform_test.go b/retrieval/transform_test.go index dbdd98d3..60739acf 100644 --- a/retrieval/transform_test.go +++ b/retrieval/transform_test.go @@ -94,7 +94,7 @@ func TestSampleBuilder(t *testing.T) { metadata: metadataMap{ "job1/instance1/metric1": &scrape.MetricMetadata{Type: textparse.MetricTypeGauge, Metric: "metric1"}, "job1/instance1/metric2": &scrape.MetricMetadata{Type: textparse.MetricTypeCounter, Metric: "metric2"}, - "job1/instance1/labelnum_ok": &scrape.MetricMetadata{Type: textparse.MetricTypeUntyped, Metric: "labelnum_ok"}, + "job1/instance1/labelnum_ok": &scrape.MetricMetadata{Type: textparse.MetricTypeUnknown, Metric: "labelnum_ok"}, "job1/instance1/labelnum_bad": &scrape.MetricMetadata{Type: textparse.MetricTypeGauge, Metric: "labelnum_bad"}, }, input: []tsdb.RefSample{ diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/cpu.oput b/vendor/github.com/prometheus/prometheus/pkg/textparse/cpu.oput deleted file mode 100644 index b1ceb883..00000000 Binary files a/vendor/github.com/prometheus/prometheus/pkg/textparse/cpu.oput and /dev/null differ diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/interface.go b/vendor/github.com/prometheus/prometheus/pkg/textparse/interface.go new file mode 100644 index 00000000..330dffa8 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/interface.go @@ -0,0 +1,91 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package textparse + +import ( + "mime" + + "github.com/prometheus/prometheus/pkg/labels" +) + +// Parser parses samples from a byte slice of samples in the official +// Prometheus and OpenMetrics text exposition formats. +type Parser interface { + // Series returns the bytes of the series, the timestamp if set, and the value + // of the current sample. + Series() ([]byte, *int64, float64) + + // Help returns the metric name and help text in the current entry. + // Must only be called after Next returned a help entry. + // The returned byte slices become invalid after the next call to Next. + Help() ([]byte, []byte) + + // Type returns the metric name and type in the current entry. + // Must only be called after Next returned a type entry. + // The returned byte slices become invalid after the next call to Next. + Type() ([]byte, MetricType) + + // Unit returns the metric name and unit in the current entry. + // Must only be called after Next returned a unit entry. + // The returned byte slices become invalid after the next call to Next. + Unit() ([]byte, []byte) + + // Comment returns the text of the current comment. + // Must only be called after Next returned a comment entry. + // The returned byte slice becomes invalid after the next call to Next. + Comment() []byte + + // Metric writes the labels of the current sample into the passed labels. + // It returns the string from which the metric was parsed. + Metric(l *labels.Labels) string + + // Next advances the parser to the next sample. It returns false if no + // more samples were read or an error occurred. + Next() (Entry, error) +} + +// New returns a new parser of the byte slice. +func New(b []byte, contentType string) Parser { + mediaType, _, err := mime.ParseMediaType(contentType) + if err == nil && mediaType == "application/openmetrics-text" { + return NewOpenMetricsParser(b) + } + return NewPromParser(b) +} + +// Entry represents the type of a parsed entry. +type Entry int + +const ( + EntryInvalid Entry = -1 + EntryType Entry = 0 + EntryHelp Entry = 1 + EntrySeries Entry = 2 + EntryComment Entry = 3 + EntryUnit Entry = 4 +) + +// MetricType represents metric type values. +type MetricType string + +const ( + MetricTypeCounter = "counter" + MetricTypeGauge = "gauge" + MetricTypeHistogram = "histogram" + MetricTypeGaugeHistogram = "gaugehistogram" + MetricTypeSummary = "summary" + MetricTypeInfo = "info" + MetricTypeStateset = "stateset" + MetricTypeUnknown = "unknown" +) diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/openmetricslex.l b/vendor/github.com/prometheus/prometheus/pkg/textparse/openmetricslex.l new file mode 100644 index 00000000..a259885f --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/openmetricslex.l @@ -0,0 +1,71 @@ +%{ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package textparse + +import ( + "fmt" +) + +// Lex is called by the parser generated by "go tool yacc" to obtain each +// token. The method is opened before the matching rules block and closed at +// the end of the file. +func (l *openMetricsLexer) Lex() token { + if l.i >= len(l.b) { + return tEOF + } + c := l.b[l.i] + l.start = l.i + +%} + +D [0-9] +L [a-zA-Z_] +M [a-zA-Z_:] +C [^\n] +S [ ] + +%x sComment sMeta1 sMeta2 sLabels sLValue sValue sTimestamp + +%yyc c +%yyn c = l.next() +%yyt l.state + + +%% + +#{S} l.state = sComment +HELP{S} l.state = sMeta1; return tHelp +TYPE{S} l.state = sMeta1; return tType +UNIT{S} l.state = sMeta1; return tUnit +"EOF"\n? l.state = sInit; return tEofWord +{M}({M}|{D})* l.state = sMeta2; return tMName +{S}{C}*\n l.state = sInit; return tText + +{M}({M}|{D})* l.state = sValue; return tMName +\{ l.state = sLabels; return tBraceOpen +{L}({L}|{D})* return tLName +\} l.state = sValue; return tBraceClose += l.state = sLValue; return tEqual +, return tComma +\"(\\.|[^\\"\n])*\" l.state = sLabels; return tLValue +{S}[^ \n]+ l.state = sTimestamp; return tValue +{S}[^ \n]+ return tTimestamp +{S}#{S}{C}*\n l.state = sInit; return tLinebreak +\n l.state = sInit; return tLinebreak + +%% + + return tInvalid +} diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/openmetricslex.l.go b/vendor/github.com/prometheus/prometheus/pkg/textparse/openmetricslex.l.go new file mode 100644 index 00000000..4dd7ddd7 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/openmetricslex.l.go @@ -0,0 +1,586 @@ +// CAUTION: Generated file - DO NOT EDIT. + +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package textparse + +import ( + "fmt" +) + +// Lex is called by the parser generated by "go tool yacc" to obtain each +// token. The method is opened before the matching rules block and closed at +// the end of the file. +func (l *openMetricsLexer) Lex() token { + if l.i >= len(l.b) { + return tEOF + } + c := l.b[l.i] + l.start = l.i + +yystate0: + + switch yyt := l.state; yyt { + default: + panic(fmt.Errorf(`invalid start condition %d`, yyt)) + case 0: // start condition: INITIAL + goto yystart1 + case 1: // start condition: sComment + goto yystart5 + case 2: // start condition: sMeta1 + goto yystart25 + case 3: // start condition: sMeta2 + goto yystart27 + case 4: // start condition: sLabels + goto yystart30 + case 5: // start condition: sLValue + goto yystart35 + case 6: // start condition: sValue + goto yystart39 + case 7: // start condition: sTimestamp + goto yystart43 + } + + goto yystate0 // silence unused label error + goto yystate1 // silence unused label error +yystate1: + c = l.next() +yystart1: + switch { + default: + goto yyabort + case c == '#': + goto yystate2 + case c == ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate4 + } + +yystate2: + c = l.next() + switch { + default: + goto yyabort + case c == ' ': + goto yystate3 + } + +yystate3: + c = l.next() + goto yyrule1 + +yystate4: + c = l.next() + switch { + default: + goto yyrule8 + case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate4 + } + + goto yystate5 // silence unused label error +yystate5: + c = l.next() +yystart5: + switch { + default: + goto yyabort + case c == 'E': + goto yystate6 + case c == 'H': + goto yystate10 + case c == 'T': + goto yystate15 + case c == 'U': + goto yystate20 + } + +yystate6: + c = l.next() + switch { + default: + goto yyabort + case c == 'O': + goto yystate7 + } + +yystate7: + c = l.next() + switch { + default: + goto yyabort + case c == 'F': + goto yystate8 + } + +yystate8: + c = l.next() + switch { + default: + goto yyrule5 + case c == '\n': + goto yystate9 + } + +yystate9: + c = l.next() + goto yyrule5 + +yystate10: + c = l.next() + switch { + default: + goto yyabort + case c == 'E': + goto yystate11 + } + +yystate11: + c = l.next() + switch { + default: + goto yyabort + case c == 'L': + goto yystate12 + } + +yystate12: + c = l.next() + switch { + default: + goto yyabort + case c == 'P': + goto yystate13 + } + +yystate13: + c = l.next() + switch { + default: + goto yyabort + case c == ' ': + goto yystate14 + } + +yystate14: + c = l.next() + goto yyrule2 + +yystate15: + c = l.next() + switch { + default: + goto yyabort + case c == 'Y': + goto yystate16 + } + +yystate16: + c = l.next() + switch { + default: + goto yyabort + case c == 'P': + goto yystate17 + } + +yystate17: + c = l.next() + switch { + default: + goto yyabort + case c == 'E': + goto yystate18 + } + +yystate18: + c = l.next() + switch { + default: + goto yyabort + case c == ' ': + goto yystate19 + } + +yystate19: + c = l.next() + goto yyrule3 + +yystate20: + c = l.next() + switch { + default: + goto yyabort + case c == 'N': + goto yystate21 + } + +yystate21: + c = l.next() + switch { + default: + goto yyabort + case c == 'I': + goto yystate22 + } + +yystate22: + c = l.next() + switch { + default: + goto yyabort + case c == 'T': + goto yystate23 + } + +yystate23: + c = l.next() + switch { + default: + goto yyabort + case c == ' ': + goto yystate24 + } + +yystate24: + c = l.next() + goto yyrule4 + + goto yystate25 // silence unused label error +yystate25: + c = l.next() +yystart25: + switch { + default: + goto yyabort + case c == ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate26 + } + +yystate26: + c = l.next() + switch { + default: + goto yyrule6 + case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate26 + } + + goto yystate27 // silence unused label error +yystate27: + c = l.next() +yystart27: + switch { + default: + goto yyabort + case c == ' ': + goto yystate28 + } + +yystate28: + c = l.next() + switch { + default: + goto yyabort + case c == '\n': + goto yystate29 + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ': + goto yystate28 + } + +yystate29: + c = l.next() + goto yyrule7 + + goto yystate30 // silence unused label error +yystate30: + c = l.next() +yystart30: + switch { + default: + goto yyabort + case c == ',': + goto yystate31 + case c == '=': + goto yystate32 + case c == '}': + goto yystate34 + case c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate33 + } + +yystate31: + c = l.next() + goto yyrule13 + +yystate32: + c = l.next() + goto yyrule12 + +yystate33: + c = l.next() + switch { + default: + goto yyrule10 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate33 + } + +yystate34: + c = l.next() + goto yyrule11 + + goto yystate35 // silence unused label error +yystate35: + c = l.next() +yystart35: + switch { + default: + goto yyabort + case c == '"': + goto yystate36 + } + +yystate36: + c = l.next() + switch { + default: + goto yyabort + case c == '"': + goto yystate37 + case c == '\\': + goto yystate38 + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ': + goto yystate36 + } + +yystate37: + c = l.next() + goto yyrule14 + +yystate38: + c = l.next() + switch { + default: + goto yyabort + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ': + goto yystate36 + } + + goto yystate39 // silence unused label error +yystate39: + c = l.next() +yystart39: + switch { + default: + goto yyabort + case c == ' ': + goto yystate40 + case c == '{': + goto yystate42 + } + +yystate40: + c = l.next() + switch { + default: + goto yyabort + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ': + goto yystate41 + } + +yystate41: + c = l.next() + switch { + default: + goto yyrule15 + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ': + goto yystate41 + } + +yystate42: + c = l.next() + goto yyrule9 + + goto yystate43 // silence unused label error +yystate43: + c = l.next() +yystart43: + switch { + default: + goto yyabort + case c == ' ': + goto yystate45 + case c == '\n': + goto yystate44 + } + +yystate44: + c = l.next() + goto yyrule18 + +yystate45: + c = l.next() + switch { + default: + goto yyabort + case c == '#': + goto yystate47 + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= '\x1f' || c == '!' || c == '"' || c >= '$' && c <= 'ÿ': + goto yystate46 + } + +yystate46: + c = l.next() + switch { + default: + goto yyrule16 + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ': + goto yystate46 + } + +yystate47: + c = l.next() + switch { + default: + goto yyrule16 + case c == ' ': + goto yystate48 + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ': + goto yystate46 + } + +yystate48: + c = l.next() + switch { + default: + goto yyabort + case c == '\n': + goto yystate49 + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ': + goto yystate48 + } + +yystate49: + c = l.next() + goto yyrule17 + +yyrule1: // #{S} + { + l.state = sComment + goto yystate0 + } +yyrule2: // HELP{S} + { + l.state = sMeta1 + return tHelp + goto yystate0 + } +yyrule3: // TYPE{S} + { + l.state = sMeta1 + return tType + goto yystate0 + } +yyrule4: // UNIT{S} + { + l.state = sMeta1 + return tUnit + goto yystate0 + } +yyrule5: // "EOF"\n? + { + l.state = sInit + return tEofWord + goto yystate0 + } +yyrule6: // {M}({M}|{D})* + { + l.state = sMeta2 + return tMName + goto yystate0 + } +yyrule7: // {S}{C}*\n + { + l.state = sInit + return tText + goto yystate0 + } +yyrule8: // {M}({M}|{D})* + { + l.state = sValue + return tMName + goto yystate0 + } +yyrule9: // \{ + { + l.state = sLabels + return tBraceOpen + goto yystate0 + } +yyrule10: // {L}({L}|{D})* + { + return tLName + } +yyrule11: // \} + { + l.state = sValue + return tBraceClose + goto yystate0 + } +yyrule12: // = + { + l.state = sLValue + return tEqual + goto yystate0 + } +yyrule13: // , + { + return tComma + } +yyrule14: // \"(\\.|[^\\"\n])*\" + { + l.state = sLabels + return tLValue + goto yystate0 + } +yyrule15: // {S}[^ \n]+ + { + l.state = sTimestamp + return tValue + goto yystate0 + } +yyrule16: // {S}[^ \n]+ + { + return tTimestamp + } +yyrule17: // {S}#{S}{C}*\n + { + l.state = sInit + return tLinebreak + goto yystate0 + } +yyrule18: // \n + { + l.state = sInit + return tLinebreak + goto yystate0 + } + panic("unreachable") + + goto yyabort // silence unused label error + +yyabort: // no lexem recognized + + return tInvalid +} diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/openmetricsparse.go b/vendor/github.com/prometheus/prometheus/pkg/textparse/openmetricsparse.go new file mode 100644 index 00000000..2d7b9215 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/openmetricsparse.go @@ -0,0 +1,347 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate go get github.com/cznic/golex +//go:generate golex -o=openmetricslex.l.go openmetricslex.l + +package textparse + +import ( + "errors" + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + "unicode/utf8" + + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/value" +) + +type openMetricsLexer struct { + b []byte + i int + start int + err error + state int +} + +// buf returns the buffer of the current token. +func (l *openMetricsLexer) buf() []byte { + return l.b[l.start:l.i] +} + +func (l *openMetricsLexer) cur() byte { + return l.b[l.i] +} + +// next advances the openMetricsLexer to the next character. +func (l *openMetricsLexer) next() byte { + l.i++ + if l.i >= len(l.b) { + l.err = io.EOF + return byte(tEOF) + } + // Lex struggles with null bytes. If we are in a label value or help string, where + // they are allowed, consume them here immediately. + for l.b[l.i] == 0 && (l.state == sLValue || l.state == sMeta2 || l.state == sComment) { + l.i++ + if l.i >= len(l.b) { + l.err = io.EOF + return byte(tEOF) + } + } + return l.b[l.i] +} + +func (l *openMetricsLexer) Error(es string) { + l.err = errors.New(es) +} + +// OpenMetricsParser parses samples from a byte slice of samples in the official +// OpenMetrics text exposition format. +// This is based on the working draft https://docs.google.com/document/u/1/d/1KwV0mAXwwbvvifBvDKH_LU1YjyXE_wxCkHNoCGq1GX0/edit +type OpenMetricsParser struct { + l *openMetricsLexer + series []byte + text []byte + mtype MetricType + val float64 + ts int64 + hasTS bool + start int + offsets []int +} + +// New returns a new parser of the byte slice. +func NewOpenMetricsParser(b []byte) Parser { + return &OpenMetricsParser{l: &openMetricsLexer{b: b}} +} + +// Series returns the bytes of the series, the timestamp if set, and the value +// of the current sample. +func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) { + if p.hasTS { + return p.series, &p.ts, p.val + } + return p.series, nil, p.val +} + +// Help returns the metric name and help text in the current entry. +// Must only be called after Next returned a help entry. +// The returned byte slices become invalid after the next call to Next. +func (p *OpenMetricsParser) Help() ([]byte, []byte) { + m := p.l.b[p.offsets[0]:p.offsets[1]] + + // Replacer causes allocations. Replace only when necessary. + if strings.IndexByte(yoloString(p.text), byte('\\')) >= 0 { + // OpenMetrics always uses the Prometheus format label value escaping. + return m, []byte(lvalReplacer.Replace(string(p.text))) + } + return m, p.text +} + +// Type returns the metric name and type in the current entry. +// Must only be called after Next returned a type entry. +// The returned byte slices become invalid after the next call to Next. +func (p *OpenMetricsParser) Type() ([]byte, MetricType) { + return p.l.b[p.offsets[0]:p.offsets[1]], p.mtype +} + +// Unit returns the metric name and unit in the current entry. +// Must only be called after Next returned a unit entry. +// The returned byte slices become invalid after the next call to Next. +func (p *OpenMetricsParser) Unit() ([]byte, []byte) { + // The Prometheus format does not have units. + return p.l.b[p.offsets[0]:p.offsets[1]], p.text +} + +// Comment returns the text of the current comment. +// Must only be called after Next returned a comment entry. +// The returned byte slice becomes invalid after the next call to Next. +func (p *OpenMetricsParser) Comment() []byte { + return p.text +} + +// Metric writes the labels of the current sample into the passed labels. +// It returns the string from which the metric was parsed. +func (p *OpenMetricsParser) Metric(l *labels.Labels) string { + // Allocate the full immutable string immediately, so we just + // have to create references on it below. + s := string(p.series) + + *l = append(*l, labels.Label{ + Name: labels.MetricName, + Value: s[:p.offsets[0]-p.start], + }) + + for i := 1; i < len(p.offsets); i += 4 { + a := p.offsets[i] - p.start + b := p.offsets[i+1] - p.start + c := p.offsets[i+2] - p.start + d := p.offsets[i+3] - p.start + + // Replacer causes allocations. Replace only when necessary. + if strings.IndexByte(s[c:d], byte('\\')) >= 0 { + *l = append(*l, labels.Label{Name: s[a:b], Value: lvalReplacer.Replace(s[c:d])}) + continue + } + *l = append(*l, labels.Label{Name: s[a:b], Value: s[c:d]}) + } + + // Sort labels. We can skip the first entry since the metric name is + // already at the right place. + sort.Sort((*l)[1:]) + + return s +} + +// nextToken returns the next token from the openMetricsLexer. +func (p *OpenMetricsParser) nextToken() token { + tok := p.l.Lex() + return tok +} + +// Next advances the parser to the next sample. It returns false if no +// more samples were read or an error occurred. +func (p *OpenMetricsParser) Next() (Entry, error) { + var err error + + p.start = p.l.i + p.offsets = p.offsets[:0] + + switch t := p.nextToken(); t { + case tEofWord: + if t := p.nextToken(); t != tEOF { + return EntryInvalid, fmt.Errorf("unexpected data after # EOF") + } + return EntryInvalid, io.EOF + case tEOF: + return EntryInvalid, parseError("unexpected end of data", t) + case tHelp, tType, tUnit: + switch t := p.nextToken(); t { + case tMName: + p.offsets = append(p.offsets, p.l.start, p.l.i) + default: + return EntryInvalid, parseError("expected metric name after HELP", t) + } + switch t := p.nextToken(); t { + case tText: + if len(p.l.buf()) > 1 { + p.text = p.l.buf()[1 : len(p.l.buf())-1] + } else { + p.text = []byte{} + } + default: + return EntryInvalid, parseError("expected text in HELP", t) + } + switch t { + case tType: + switch s := yoloString(p.text); s { + case "counter": + p.mtype = MetricTypeCounter + case "gauge": + p.mtype = MetricTypeGauge + case "histogram": + p.mtype = MetricTypeHistogram + case "gaugehistogram": + p.mtype = MetricTypeGaugeHistogram + case "summary": + p.mtype = MetricTypeSummary + case "info": + p.mtype = MetricTypeInfo + case "stateset": + p.mtype = MetricTypeStateset + case "unknown": + p.mtype = MetricTypeUnknown + default: + return EntryInvalid, fmt.Errorf("invalid metric type %q", s) + } + case tHelp: + if !utf8.Valid(p.text) { + return EntryInvalid, fmt.Errorf("help text is not a valid utf8 string") + } + } + switch t { + case tHelp: + return EntryHelp, nil + case tType: + return EntryType, nil + case tUnit: + m := yoloString(p.l.b[p.offsets[0]:p.offsets[1]]) + u := yoloString(p.text) + if len(u) > 0 { + if !strings.HasSuffix(m, u) || len(m) < len(u)+1 || p.l.b[p.offsets[1]-len(u)-1] != '_' { + return EntryInvalid, fmt.Errorf("unit not a suffix of metric %q", m) + } + } + return EntryUnit, nil + } + + case tMName: + p.offsets = append(p.offsets, p.l.i) + p.series = p.l.b[p.start:p.l.i] + + t2 := p.nextToken() + if t2 == tBraceOpen { + if err := p.parseLVals(); err != nil { + return EntryInvalid, err + } + p.series = p.l.b[p.start:p.l.i] + t2 = p.nextToken() + } + if t2 != tValue { + return EntryInvalid, parseError("expected value after metric", t) + } + if p.val, err = strconv.ParseFloat(yoloString(p.l.buf()[1:]), 64); err != nil { + return EntryInvalid, err + } + // Ensure canonical NaN value. + if math.IsNaN(p.val) { + p.val = math.Float64frombits(value.NormalNaN) + } + p.hasTS = false + switch p.nextToken() { + case tLinebreak: + break + case tTimestamp: + p.hasTS = true + var ts float64 + // A float is enough to hold what we need for millisecond resolution. + if ts, err = strconv.ParseFloat(yoloString(p.l.buf()[1:]), 64); err != nil { + return EntryInvalid, err + } + p.ts = int64(ts * 1000) + if t2 := p.nextToken(); t2 != tLinebreak { + return EntryInvalid, parseError("expected next entry after timestamp", t) + } + default: + return EntryInvalid, parseError("expected timestamp or new record", t) + } + return EntrySeries, nil + + default: + err = fmt.Errorf("%q %q is not a valid start token", t, string(p.l.cur())) + } + return EntryInvalid, err +} + +func (p *OpenMetricsParser) parseLVals() error { + first := true + for { + t := p.nextToken() + switch t { + case tBraceClose: + return nil + case tComma: + if first { + return parseError("expected label name or left brace", t) + } + t = p.nextToken() + if t != tLName { + return parseError("expected label name", t) + } + case tLName: + if !first { + return parseError("expected comma", t) + } + default: + if first { + return parseError("expected label name or left brace", t) + } + return parseError("expected comma or left brace", t) + + } + first = false + // t is now a label name. + + p.offsets = append(p.offsets, p.l.start, p.l.i) + + if t := p.nextToken(); t != tEqual { + return parseError("expected equal", t) + } + if t := p.nextToken(); t != tLValue { + return parseError("expected label value", t) + } + if !utf8.Valid(p.l.buf()) { + return fmt.Errorf("invalid UTF-8 label value") + } + + // The openMetricsLexer ensures the value string is quoted. Strip first + // and last character. + p.offsets = append(p.offsets, p.l.start+1, p.l.i-1) + + } +} diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l b/vendor/github.com/prometheus/prometheus/pkg/textparse/promlex.l similarity index 95% rename from vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l rename to vendor/github.com/prometheus/prometheus/pkg/textparse/promlex.l index a6b728c7..c3c5c3bb 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/promlex.l @@ -32,7 +32,7 @@ const ( // Lex is called by the parser generated by "go tool yacc" to obtain each // token. The method is opened before the matching rules block and closed at // the end of the file. -func (l *lexer) Lex() token { +func (l *promlexer) Lex() token { if l.i >= len(l.b) { return tEOF } @@ -64,7 +64,7 @@ C [^\n] HELP[\t ]+ l.state = sMeta1; return tHelp TYPE[\t ]+ l.state = sMeta1; return tType {M}({M}|{D})* l.state = sMeta2; return tMName -{C}+ l.state = sInit; return tText +{C}* l.state = sInit; return tText {M}({M}|{D})* l.state = sValue; return tMName \{ l.state = sLabels; return tBraceOpen @@ -87,7 +87,7 @@ C [^\n] return tInvalid } -func (l *lexer) consumeComment() token { +func (l *promlexer) consumeComment() token { for c := l.cur(); ; c = l.next() { switch c { case 0: diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l.go b/vendor/github.com/prometheus/prometheus/pkg/textparse/promlex.l.go similarity index 98% rename from vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l.go rename to vendor/github.com/prometheus/prometheus/pkg/textparse/promlex.l.go index 33a6a9fc..843feefc 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l.go +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/promlex.l.go @@ -1,4 +1,4 @@ -// Code generated by golex. DO NOT EDIT. +// CAUTION: Generated file - DO NOT EDIT. // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,7 +33,7 @@ const ( // Lex is called by the parser generated by "go tool yacc" to obtain each // token. The method is opened before the matching rules block and closed at // the end of the file. -func (l *lexer) Lex() token { +func (l *promlexer) Lex() token { if l.i >= len(l.b) { return tEOF } @@ -260,7 +260,7 @@ yystate21: yystart21: switch { default: - goto yyabort + goto yyrule9 case c == '\t' || c == ' ': goto yystate23 case c >= '\x01' && c <= '\b' || c >= '\v' && c <= '\x1f' || c >= '!' && c <= 'ÿ': @@ -463,7 +463,7 @@ yyrule8: // {M}({M}|{D})* return tMName goto yystate0 } -yyrule9: // {C}+ +yyrule9: // {C}* { l.state = sInit return tText @@ -537,7 +537,7 @@ yyabort: // no lexem recognized return tInvalid } -func (l *lexer) consumeComment() token { +func (l *promlexer) consumeComment() token { for c := l.cur(); ; c = l.next() { switch c { case 0: diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/parse.go b/vendor/github.com/prometheus/prometheus/pkg/textparse/promparse.go similarity index 81% rename from vendor/github.com/prometheus/prometheus/pkg/textparse/parse.go rename to vendor/github.com/prometheus/prometheus/pkg/textparse/promparse.go index 4bc3e137..2b2cbdab 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/textparse/parse.go +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/promparse.go @@ -12,9 +12,8 @@ // limitations under the License. //go:generate go get github.com/cznic/golex -//go:generate golex -o=lex.l.go lex.l +//go:generate golex -o=promlex.l.go promlex.l -// Package textparse contains an efficient parser for the Prometheus text format. package textparse import ( @@ -28,12 +27,11 @@ import ( "unicode/utf8" "unsafe" - "github.com/prometheus/prometheus/pkg/value" - "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/value" ) -type lexer struct { +type promlexer struct { b []byte i int start int @@ -50,6 +48,8 @@ const ( tWhitespace tHelp tType + tUnit + tEofWord tText tComment tBlank @@ -78,6 +78,10 @@ func (t token) String() string { return "HELP" case tType: return "TYPE" + case tUnit: + return "UNIT" + case tEofWord: + return "EOFWORD" case tText: return "TEXT" case tComment: @@ -107,37 +111,37 @@ func (t token) String() string { } // buf returns the buffer of the current token. -func (l *lexer) buf() []byte { +func (l *promlexer) buf() []byte { return l.b[l.start:l.i] } -func (l *lexer) cur() byte { +func (l *promlexer) cur() byte { return l.b[l.i] } -// next advances the lexer to the next character. -func (l *lexer) next() byte { +// next advances the promlexer to the next character. +func (l *promlexer) next() byte { l.i++ if l.i >= len(l.b) { l.err = io.EOF return byte(tEOF) } - // Lex struggles with null bytes. If we are in a label value, where + // Lex struggles with null bytes. If we are in a label value or help string, where // they are allowed, consume them here immediately. - for l.b[l.i] == 0 && l.state == sLValue { + for l.b[l.i] == 0 && (l.state == sLValue || l.state == sMeta2 || l.state == sComment) { l.i++ } return l.b[l.i] } -func (l *lexer) Error(es string) { +func (l *promlexer) Error(es string) { l.err = errors.New(es) } -// Parser parses samples from a byte slice of samples in the official +// PromParser parses samples from a byte slice of samples in the official // Prometheus text exposition format. -type Parser struct { - l *lexer +type PromParser struct { + l *promlexer series []byte text []byte mtype MetricType @@ -149,13 +153,13 @@ type Parser struct { } // New returns a new parser of the byte slice. -func New(b []byte) *Parser { - return &Parser{l: &lexer{b: append(b, '\n')}} +func NewPromParser(b []byte) Parser { + return &PromParser{l: &promlexer{b: append(b, '\n')}} } // Series returns the bytes of the series, the timestamp if set, and the value // of the current sample. -func (p *Parser) Series() ([]byte, *int64, float64) { +func (p *PromParser) Series() ([]byte, *int64, float64) { if p.hasTS { return p.series, &p.ts, p.val } @@ -165,7 +169,7 @@ func (p *Parser) Series() ([]byte, *int64, float64) { // Help returns the metric name and help text in the current entry. // Must only be called after Next returned a help entry. // The returned byte slices become invalid after the next call to Next. -func (p *Parser) Help() ([]byte, []byte) { +func (p *PromParser) Help() ([]byte, []byte) { m := p.l.b[p.offsets[0]:p.offsets[1]] // Replacer causes allocations. Replace only when necessary. @@ -178,20 +182,28 @@ func (p *Parser) Help() ([]byte, []byte) { // Type returns the metric name and type in the current entry. // Must only be called after Next returned a type entry. // The returned byte slices become invalid after the next call to Next. -func (p *Parser) Type() ([]byte, MetricType) { +func (p *PromParser) Type() ([]byte, MetricType) { return p.l.b[p.offsets[0]:p.offsets[1]], p.mtype } +// Unit returns the metric name and unit in the current entry. +// Must only be called after Next returned a unit entry. +// The returned byte slices become invalid after the next call to Next. +func (p *PromParser) Unit() ([]byte, []byte) { + // The Prometheus format does not have units. + return nil, nil +} + // Comment returns the text of the current comment. // Must only be called after Next returned a comment entry. // The returned byte slice becomes invalid after the next call to Next. -func (p *Parser) Comment() []byte { +func (p *PromParser) Comment() []byte { return p.text } // Metric writes the labels of the current sample into the passed labels. // It returns the string from which the metric was parsed. -func (p *Parser) Metric(l *labels.Labels) string { +func (p *PromParser) Metric(l *labels.Labels) string { // Allocate the full immutable string immediately, so we just // have to create references on it below. s := string(p.series) @@ -222,9 +234,9 @@ func (p *Parser) Metric(l *labels.Labels) string { return s } -// nextToken returns the next token from the lexer. It skips over tabs +// nextToken returns the next token from the promlexer. It skips over tabs // and spaces. -func (p *Parser) nextToken() token { +func (p *PromParser) nextToken() token { for { if tok := p.l.Lex(); tok != tWhitespace { return tok @@ -232,35 +244,13 @@ func (p *Parser) nextToken() token { } } -// Entry represents the type of a parsed entry. -type Entry int - -const ( - EntryInvalid Entry = -1 - EntryType Entry = 0 - EntryHelp Entry = 1 - EntrySeries Entry = 2 - EntryComment Entry = 3 -) - -// MetricType represents metric type values. -type MetricType string - -const ( - MetricTypeCounter = "counter" - MetricTypeGauge = "gauge" - MetricTypeHistogram = "histogram" - MetricTypeSummary = "summary" - MetricTypeUntyped = "untyped" -) - func parseError(exp string, got token) error { return fmt.Errorf("%s, got %q", exp, got) } // Next advances the parser to the next sample. It returns false if no // more samples were read or an error occurred. -func (p *Parser) Next() (Entry, error) { +func (p *PromParser) Next() (Entry, error) { var err error p.start = p.l.i @@ -282,11 +272,16 @@ func (p *Parser) Next() (Entry, error) { } switch t := p.nextToken(); t { case tText: - p.text = p.l.buf()[1:] + if len(p.l.buf()) > 1 { + p.text = p.l.buf()[1:] + } else { + p.text = []byte{} + } default: return EntryInvalid, parseError("expected text in HELP", t) } - if t == tType { + switch t { + case tType: switch s := yoloString(p.text); s { case "counter": p.mtype = MetricTypeCounter @@ -297,10 +292,14 @@ func (p *Parser) Next() (Entry, error) { case "summary": p.mtype = MetricTypeSummary case "untyped": - p.mtype = MetricTypeUntyped + p.mtype = MetricTypeUnknown default: return EntryInvalid, fmt.Errorf("invalid metric type %q", s) } + case tHelp: + if !utf8.Valid(p.text) { + return EntryInvalid, fmt.Errorf("help text is not a valid utf8 string") + } } if t := p.nextToken(); t != tLinebreak { return EntryInvalid, parseError("linebreak expected after metadata", t) @@ -363,7 +362,7 @@ func (p *Parser) Next() (Entry, error) { return EntryInvalid, err } -func (p *Parser) parseLVals() error { +func (p *PromParser) parseLVals() error { t := p.nextToken() for { switch t { @@ -385,7 +384,7 @@ func (p *Parser) parseLVals() error { return fmt.Errorf("invalid UTF-8 label value") } - // The lexer ensures the value string is quoted. Strip first + // The promlexer ensures the value string is quoted. Strip first // and last character. p.offsets = append(p.offsets, p.l.start+1, p.l.i-1) diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/testdata.nometa.txt b/vendor/github.com/prometheus/prometheus/pkg/textparse/promtestdata.nometa.txt similarity index 99% rename from vendor/github.com/prometheus/prometheus/pkg/textparse/testdata.nometa.txt rename to vendor/github.com/prometheus/prometheus/pkg/textparse/promtestdata.nometa.txt index e760ad26..235f0aa4 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/textparse/testdata.nometa.txt +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/promtestdata.nometa.txt @@ -408,3 +408,4 @@ prometheus_target_sync_length_seconds_sum{scrape_job="prometheus"} 0.00020043300 prometheus_target_sync_length_seconds_count{scrape_job="prometheus"} 1 prometheus_treecache_watcher_goroutines 0 prometheus_treecache_zookeeper_failures_total 0 +# EOF diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/testdata.txt b/vendor/github.com/prometheus/prometheus/pkg/textparse/promtestdata.txt similarity index 99% rename from vendor/github.com/prometheus/prometheus/pkg/textparse/testdata.txt rename to vendor/github.com/prometheus/prometheus/pkg/textparse/promtestdata.txt index c7f2a7af..174f383e 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/textparse/testdata.txt +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/promtestdata.txt @@ -525,4 +525,5 @@ prometheus_target_sync_length_seconds_count{scrape_job="prometheus"} 1 prometheus_treecache_watcher_goroutines 0 # HELP prometheus_treecache_zookeeper_failures_total The total number of ZooKeeper failures. # TYPE prometheus_treecache_zookeeper_failures_total counter -prometheus_treecache_zookeeper_failures_total 0 \ No newline at end of file +prometheus_treecache_zookeeper_failures_total 0 +# EOF diff --git a/vendor/github.com/prometheus/prometheus/scrape/manager.go b/vendor/github.com/prometheus/prometheus/scrape/manager.go index 4c576952..8cf75d9e 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/manager.go +++ b/vendor/github.com/prometheus/prometheus/scrape/manager.go @@ -14,9 +14,9 @@ package scrape import ( - "fmt" "reflect" "sync" + "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -33,13 +33,16 @@ type Appendable interface { // NewManager is the Manager constructor func NewManager(logger log.Logger, app Appendable) *Manager { + if logger == nil { + logger = log.NewNopLogger() + } return &Manager{ append: app, logger: logger, scrapeConfigs: make(map[string]*config.ScrapeConfig), scrapePools: make(map[string]*scrapePool), graceShut: make(chan struct{}), - targetsAll: make(map[string][]*Target), + triggerReload: make(chan struct{}, 1), } } @@ -50,28 +53,83 @@ type Manager struct { append Appendable graceShut chan struct{} - mtxTargets sync.Mutex // Guards the fields below. - targetsActive []*Target - targetsDropped []*Target - targetsAll map[string][]*Target - mtxScrape sync.Mutex // Guards the fields below. scrapeConfigs map[string]*config.ScrapeConfig scrapePools map[string]*scrapePool + targetSets map[string][]*targetgroup.Group + + triggerReload chan struct{} } -// Run starts background processing to handle target updates and reload the scraping loops. +// Run receives and saves target set updates and triggers the scraping loops reloading. +// Reloading happens in the background so that it doesn't block receiving targets updates. func (m *Manager) Run(tsets <-chan map[string][]*targetgroup.Group) error { + go m.reloader() for { select { case ts := <-tsets: - m.reload(ts) + m.updateTsets(ts) + + select { + case m.triggerReload <- struct{}{}: + default: + } + case <-m.graceShut: return nil } } } +func (m *Manager) reloader() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + case <-m.graceShut: + return + case <-ticker.C: + select { + case <-m.triggerReload: + m.reload() + case <-m.graceShut: + return + } + } + } +} + +func (m *Manager) reload() { + m.mtxScrape.Lock() + var wg sync.WaitGroup + for setName, groups := range m.targetSets { + var sp *scrapePool + existing, ok := m.scrapePools[setName] + if !ok { + scrapeConfig, ok := m.scrapeConfigs[setName] + if !ok { + level.Error(m.logger).Log("msg", "error reloading target set", "err", "invalid config id:"+setName) + return + } + sp = newScrapePool(scrapeConfig, m.append, log.With(m.logger, "scrape_pool", setName)) + m.scrapePools[setName] = sp + } else { + sp = existing + } + + wg.Add(1) + // Run the sync in parallel as these take a while and at high load can't catch up. + go func(sp *scrapePool, groups []*targetgroup.Group) { + sp.Sync(groups) + wg.Done() + }(sp, groups) + + } + m.mtxScrape.Unlock() + wg.Wait() +} + // Stop cancels all running scrape pools and blocks until all have exited. func (m *Manager) Stop() { m.mtxScrape.Lock() @@ -83,6 +141,12 @@ func (m *Manager) Stop() { close(m.graceShut) } +func (m *Manager) updateTsets(tsets map[string][]*targetgroup.Group) { + m.mtxScrape.Lock() + m.targetSets = tsets + m.mtxScrape.Unlock() +} + // ApplyConfig resets the manager's target providers and job configurations as defined by the new cfg. func (m *Manager) ApplyConfig(cfg *config.Config) error { m.mtxScrape.Lock() @@ -109,64 +173,37 @@ func (m *Manager) ApplyConfig(cfg *config.Config) error { // TargetsAll returns active and dropped targets grouped by job_name. func (m *Manager) TargetsAll() map[string][]*Target { - m.mtxTargets.Lock() - defer m.mtxTargets.Unlock() - return m.targetsAll -} + m.mtxScrape.Lock() + defer m.mtxScrape.Unlock() -// TargetsActive returns the active targets currently being scraped. -func (m *Manager) TargetsActive() []*Target { - m.mtxTargets.Lock() - defer m.mtxTargets.Unlock() - return m.targetsActive -} + targets := make(map[string][]*Target, len(m.scrapePools)) + for tset, sp := range m.scrapePools { + targets[tset] = append(sp.ActiveTargets(), sp.DroppedTargets()...) -// TargetsDropped returns the dropped targets during relabelling. -func (m *Manager) TargetsDropped() []*Target { - m.mtxTargets.Lock() - defer m.mtxTargets.Unlock() - return m.targetsDropped + } + return targets } -func (m *Manager) targetsUpdate(active, dropped map[string][]*Target) { - m.mtxTargets.Lock() - defer m.mtxTargets.Unlock() - - m.targetsAll = make(map[string][]*Target) - m.targetsActive = nil - m.targetsDropped = nil - for jobName, targets := range active { - m.targetsAll[jobName] = append(m.targetsAll[jobName], targets...) - m.targetsActive = append(m.targetsActive, targets...) +// TargetsActive returns the active targets currently being scraped. +func (m *Manager) TargetsActive() map[string][]*Target { + m.mtxScrape.Lock() + defer m.mtxScrape.Unlock() + targets := make(map[string][]*Target, len(m.scrapePools)) + for tset, sp := range m.scrapePools { + targets[tset] = sp.ActiveTargets() } - for jobName, targets := range dropped { - m.targetsAll[jobName] = append(m.targetsAll[jobName], targets...) - m.targetsDropped = append(m.targetsDropped, targets...) - } + return targets } -func (m *Manager) reload(t map[string][]*targetgroup.Group) { +// TargetsDropped returns the dropped targets during relabelling. +func (m *Manager) TargetsDropped() map[string][]*Target { m.mtxScrape.Lock() defer m.mtxScrape.Unlock() - tDropped := make(map[string][]*Target) - tActive := make(map[string][]*Target) - - for tsetName, tgroup := range t { - var sp *scrapePool - if existing, ok := m.scrapePools[tsetName]; !ok { - scrapeConfig, ok := m.scrapeConfigs[tsetName] - if !ok { - level.Error(m.logger).Log("msg", "error reloading target set", "err", fmt.Sprintf("invalid config id:%v", tsetName)) - continue - } - sp = newScrapePool(scrapeConfig, m.append, log.With(m.logger, "scrape_pool", tsetName)) - m.scrapePools[tsetName] = sp - } else { - sp = existing - } - tActive[tsetName], tDropped[tsetName] = sp.Sync(tgroup) + targets := make(map[string][]*Target, len(m.scrapePools)) + for tset, sp := range m.scrapePools { + targets[tset] = sp.DroppedTargets() } - m.targetsUpdate(tActive, tDropped) + return targets } diff --git a/vendor/github.com/prometheus/prometheus/scrape/scrape.go b/vendor/github.com/prometheus/prometheus/scrape/scrape.go index 1b493696..3060c34a 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/scrape.go +++ b/vendor/github.com/prometheus/prometheus/scrape/scrape.go @@ -124,7 +124,7 @@ type scrapePool struct { client *http.Client // Targets and loops must always be synchronized to have the same // set of hashes. - targets map[uint64]*Target + activeTargets map[uint64]*Target droppedTargets []*Target loops map[uint64]loop cancel context.CancelFunc @@ -152,13 +152,13 @@ func newScrapePool(cfg *config.ScrapeConfig, app Appendable, logger log.Logger) ctx, cancel := context.WithCancel(context.Background()) sp := &scrapePool{ - cancel: cancel, - appendable: app, - config: cfg, - client: client, - targets: map[uint64]*Target{}, - loops: map[uint64]loop{}, - logger: logger, + cancel: cancel, + appendable: app, + config: cfg, + client: client, + activeTargets: map[uint64]*Target{}, + loops: map[uint64]loop{}, + logger: logger, } sp.newLoop = func(t *Target, s scraper, limit int, honor bool, mrc []*config.RelabelConfig) loop { // Update the targets retrieval function for metadata to a new scrape cache. @@ -186,6 +186,23 @@ func newScrapePool(cfg *config.ScrapeConfig, app Appendable, logger log.Logger) return sp } +func (sp *scrapePool) ActiveTargets() []*Target { + sp.mtx.Lock() + defer sp.mtx.Unlock() + + var tActive []*Target + for _, t := range sp.activeTargets { + tActive = append(tActive, t) + } + return tActive +} + +func (sp *scrapePool) DroppedTargets() []*Target { + sp.mtx.Lock() + defer sp.mtx.Unlock() + return sp.droppedTargets +} + // stop terminates all scrape loops and returns after they all terminated. func (sp *scrapePool) stop() { sp.cancel() @@ -203,7 +220,7 @@ func (sp *scrapePool) stop() { }(l) delete(sp.loops, fp) - delete(sp.targets, fp) + delete(sp.activeTargets, fp) } wg.Wait() } @@ -236,7 +253,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) { for fp, oldLoop := range sp.loops { var ( - t = sp.targets[fp] + t = sp.activeTargets[fp] s = &targetScraper{Target: t, client: sp.client, timeout: timeout} newLoop = sp.newLoop(t, s, limit, honor, mrc) ) @@ -260,7 +277,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) { // Sync converts target groups into actual scrape targets and synchronizes // the currently running scraper with the resulting set and returns all scraped and dropped targets. -func (sp *scrapePool) Sync(tgs []*targetgroup.Group) (tActive []*Target, tDropped []*Target) { +func (sp *scrapePool) Sync(tgs []*targetgroup.Group) { start := time.Now() var all []*Target @@ -287,15 +304,6 @@ func (sp *scrapePool) Sync(tgs []*targetgroup.Group) (tActive []*Target, tDroppe time.Since(start).Seconds(), ) targetScrapePoolSyncsCounter.WithLabelValues(sp.config.JobName).Inc() - - sp.mtx.RLock() - for _, t := range sp.targets { - tActive = append(tActive, t) - } - tDropped = sp.droppedTargets - sp.mtx.RUnlock() - - return tActive, tDropped } // sync takes a list of potentially duplicated targets, deduplicates them, starts @@ -319,34 +327,36 @@ func (sp *scrapePool) sync(targets []*Target) { hash := t.hash() uniqueTargets[hash] = struct{}{} - if _, ok := sp.targets[hash]; !ok { + if _, ok := sp.activeTargets[hash]; !ok { s := &targetScraper{Target: t, client: sp.client, timeout: timeout} l := sp.newLoop(t, s, limit, honor, mrc) - sp.targets[hash] = t + sp.activeTargets[hash] = t sp.loops[hash] = l go l.run(interval, timeout, nil) } else { // Need to keep the most updated labels information // for displaying it in the Service Discovery web page. - sp.targets[hash].SetDiscoveredLabels(t.DiscoveredLabels()) + sp.activeTargets[hash].SetDiscoveredLabels(t.DiscoveredLabels()) } } var wg sync.WaitGroup // Stop and remove old targets and scraper loops. - for hash := range sp.targets { + for hash := range sp.activeTargets { if _, ok := uniqueTargets[hash]; !ok { wg.Add(1) go func(l loop) { + l.stop() + wg.Done() }(sp.loops[hash]) delete(sp.loops, hash) - delete(sp.targets, hash) + delete(sp.activeTargets, hash) } } @@ -424,7 +434,7 @@ func appender(app storage.Appender, limit int) storage.Appender { // A scraper retrieves samples and accepts a status report at the end. type scraper interface { - scrape(ctx context.Context, w io.Writer) error + scrape(ctx context.Context, w io.Writer) (string, error) report(start time.Time, dur time.Duration, err error) offset(interval time.Duration) time.Duration } @@ -441,15 +451,15 @@ type targetScraper struct { buf *bufio.Reader } -const acceptHeader = `text/plain;version=0.0.4;q=1,*/*;q=0.1` +const acceptHeader = `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1` var userAgentHeader = fmt.Sprintf("Prometheus/%s", version.Version) -func (s *targetScraper) scrape(ctx context.Context, w io.Writer) error { +func (s *targetScraper) scrape(ctx context.Context, w io.Writer) (string, error) { if s.req == nil { req, err := http.NewRequest("GET", s.URL().String(), nil) if err != nil { - return err + return "", err } req.Header.Add("Accept", acceptHeader) req.Header.Add("Accept-Encoding", "gzip") @@ -461,33 +471,38 @@ func (s *targetScraper) scrape(ctx context.Context, w io.Writer) error { resp, err := ctxhttp.Do(ctx, s.client, s.req) if err != nil { - return err + return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("server returned HTTP status %s", resp.Status) + return "", fmt.Errorf("server returned HTTP status %s", resp.Status) } if resp.Header.Get("Content-Encoding") != "gzip" { _, err = io.Copy(w, resp.Body) - return err + return "", err } if s.gzipr == nil { s.buf = bufio.NewReader(resp.Body) s.gzipr, err = gzip.NewReader(s.buf) if err != nil { - return err + return "", err } } else { s.buf.Reset(resp.Body) - s.gzipr.Reset(s.buf) + if err = s.gzipr.Reset(s.buf); err != nil { + return "", err + } } _, err = io.Copy(w, s.gzipr) s.gzipr.Close() - return err + if err != nil { + return "", err + } + return resp.Header.Get("Content-Type"), nil } // A loop can run and be stopped again. It must not be reused after it was stopped. @@ -547,9 +562,10 @@ type scrapeCache struct { // metaEntry holds meta information about a metric. type metaEntry struct { - lastIter uint64 // last scrape iteration the entry was observed + lastIter uint64 // Last scrape iteration the entry was observed at. typ textparse.MetricType help string + unit string } func newScrapeCache() *scrapeCache { @@ -644,7 +660,7 @@ func (c *scrapeCache) setType(metric []byte, t textparse.MetricType) { e, ok := c.metadata[yoloString(metric)] if !ok { - e = &metaEntry{typ: textparse.MetricTypeUntyped} + e = &metaEntry{typ: textparse.MetricTypeUnknown} c.metadata[string(metric)] = e } e.typ = t @@ -658,7 +674,7 @@ func (c *scrapeCache) setHelp(metric, help []byte) { e, ok := c.metadata[yoloString(metric)] if !ok { - e = &metaEntry{typ: textparse.MetricTypeUntyped} + e = &metaEntry{typ: textparse.MetricTypeUnknown} c.metadata[string(metric)] = e } if e.help != yoloString(help) { @@ -669,6 +685,22 @@ func (c *scrapeCache) setHelp(metric, help []byte) { c.metaMtx.Unlock() } +func (c *scrapeCache) setUnit(metric, unit []byte) { + c.metaMtx.Lock() + + e, ok := c.metadata[yoloString(metric)] + if !ok { + e = &metaEntry{typ: textparse.MetricTypeUnknown} + c.metadata[string(metric)] = e + } + if e.unit != yoloString(unit) { + e.unit = string(unit) + } + e.lastIter = c.iter + + c.metaMtx.Unlock() +} + func (c *scrapeCache) getMetadata(metric string) (MetricMetadata, bool) { c.metaMtx.Lock() defer c.metaMtx.Unlock() @@ -681,6 +713,7 @@ func (c *scrapeCache) getMetadata(metric string) (MetricMetadata, bool) { Metric: metric, Type: m.typ, Help: m.help, + Unit: m.unit, }, true } @@ -695,6 +728,7 @@ func (c *scrapeCache) listMetadata() []MetricMetadata { Metric: m, Type: e.typ, Help: e.help, + Unit: e.unit, }) } return res @@ -777,7 +811,7 @@ mainLoop: b := sl.buffers.Get(sl.lastScrapeSize).([]byte) buf := bytes.NewBuffer(b) - scrapeErr := sl.scraper.scrape(scrapeCtx, buf) + contentType, scrapeErr := sl.scraper.scrape(scrapeCtx, buf) cancel() if scrapeErr == nil { @@ -797,12 +831,12 @@ mainLoop: // A failed scrape is the same as an empty scrape, // we still call sl.append to trigger stale markers. - total, added, appErr := sl.append(b, start) + total, added, appErr := sl.append(b, contentType, start) if appErr != nil { level.Warn(sl.l).Log("msg", "append failed", "err", appErr) // The append failed, probably due to a parse error or sample limit. // Call sl.append again with an empty scrape to trigger stale markers. - if _, _, err := sl.append([]byte{}, start); err != nil { + if _, _, err := sl.append([]byte{}, "", start); err != nil { level.Warn(sl.l).Log("msg", "append failed", "err", err) } } @@ -813,7 +847,9 @@ mainLoop: scrapeErr = appErr } - sl.report(start, time.Since(start), total, added, scrapeErr) + if err := sl.report(start, time.Since(start), total, added, scrapeErr); err != nil { + level.Warn(sl.l).Log("msg", "appending scrape report failed", "err", err) + } last = start select { @@ -871,7 +907,7 @@ func (sl *scrapeLoop) endOfRunStaleness(last time.Time, ticker *time.Ticker, int // Call sl.append again with an empty scrape to trigger stale markers. // If the target has since been recreated and scraped, the // stale markers will be out of order and ignored. - if _, _, err := sl.append([]byte{}, staleTime); err != nil { + if _, _, err := sl.append([]byte{}, "", staleTime); err != nil { level.Error(sl.l).Log("msg", "stale append failed", "err", err) } if err := sl.reportStale(staleTime); err != nil { @@ -907,10 +943,10 @@ func (s samples) Less(i, j int) bool { return s[i].t < s[j].t } -func (sl *scrapeLoop) append(b []byte, ts time.Time) (total, added int, err error) { +func (sl *scrapeLoop) append(b []byte, contentType string, ts time.Time) (total, added int, err error) { var ( app = sl.appender() - p = textparse.New(b) + p = textparse.New(b, contentType) defTime = timestamp.FromTime(ts) numOutOfOrder = 0 numDuplicates = 0 @@ -934,6 +970,9 @@ loop: case textparse.EntryHelp: sl.cache.setHelp(p.Help()) continue + case textparse.EntryUnit: + sl.cache.setUnit(p.Unit()) + continue case textparse.EntryComment: continue default: diff --git a/vendor/github.com/prometheus/prometheus/scrape/target.go b/vendor/github.com/prometheus/prometheus/scrape/target.go index 0528e384..3502dcf9 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/target.go +++ b/vendor/github.com/prometheus/prometheus/scrape/target.go @@ -53,11 +53,12 @@ type Target struct { // Additional URL parmeters that are part of the target URL. params url.Values - mtx sync.RWMutex - lastError error - lastScrape time.Time - health TargetHealth - metadata metricMetadataStore + mtx sync.RWMutex + lastError error + lastScrape time.Time + lastScrapeDuration time.Duration + health TargetHealth + metadata metricMetadataStore } // NewTarget creates a reasonably configured target for querying. @@ -84,6 +85,7 @@ type MetricMetadata struct { Metric string Type textparse.MetricType Help string + Unit string } func (t *Target) MetadataList() []MetricMetadata { @@ -206,6 +208,7 @@ func (t *Target) report(start time.Time, dur time.Duration, err error) { t.lastError = err t.lastScrape = start + t.lastScrapeDuration = dur } // LastError returns the error encountered during the last scrape. @@ -224,6 +227,14 @@ func (t *Target) LastScrape() time.Time { return t.lastScrape } +// LastScrapeDuration returns how long the last scrape of the target took. +func (t *Target) LastScrapeDuration() time.Duration { + t.mtx.RLock() + defer t.mtx.RUnlock() + + return t.lastScrapeDuration +} + // Health returns the last known health state of the target. func (t *Target) Health() TargetHealth { t.mtx.RLock() diff --git a/vendor/vendor.json b/vendor/vendor.json index 787c9f9b..a8605f86 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -780,10 +780,10 @@ "revisionTime": "2018-06-05T10:30:19Z" }, { - "checksumSHA1": "zkMYxyNMJLOL05bHwKpzzFNrWgk=", + "checksumSHA1": "Gryza0KJX4i6dS6UYr/DBtvEWa0=", "path": "github.com/prometheus/prometheus/pkg/textparse", - "revision": "ae96914199b4e1e47b27a3789c3044ee8a14e796", - "revisionTime": "2018-06-05T10:30:19Z" + "revision": "67dc912ac8b24f94a1fc478f352d25179c94ab9b", + "revisionTime": "2018-11-06T11:38:56Z" }, { "checksumSHA1": "aBxiHq8WBoLD+8U0/PxdWufs64o=", @@ -798,10 +798,10 @@ "revisionTime": "2018-06-05T10:30:19Z" }, { - "checksumSHA1": "AJaBokB3sohb8Cv+QJAaJq9ysfU=", + "checksumSHA1": "+MySwSXOb6nwbb3tQi3GaVxDHbM=", "path": "github.com/prometheus/prometheus/scrape", - "revision": "ae96914199b4e1e47b27a3789c3044ee8a14e796", - "revisionTime": "2018-06-05T10:30:19Z" + "revision": "67dc912ac8b24f94a1fc478f352d25179c94ab9b", + "revisionTime": "2018-11-06T11:38:56Z" }, { "checksumSHA1": "xYVbgwVChu2859wdb9cwqEokcR4=",