From f1f0b52de85d0aa9ebda9f8a77c87bbd856f7259 Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Mon, 22 Mar 2021 14:35:22 -0700 Subject: [PATCH 1/3] prom: Save __name__ before relabel for look up metric type #190 --- .../prometheus_scraper/metrics_receiver.go | 20 +++++++++------- .../metrics_type_handler.go | 13 ++++++++--- plugins/inputs/prometheus_scraper/start.go | 23 +++++++++++++++++++ 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/plugins/inputs/prometheus_scraper/metrics_receiver.go b/plugins/inputs/prometheus_scraper/metrics_receiver.go index 01d6d56f0b..f1a035ece3 100644 --- a/plugins/inputs/prometheus_scraper/metrics_receiver.go +++ b/plugins/inputs/prometheus_scraper/metrics_receiver.go @@ -17,11 +17,14 @@ import ( type PrometheusMetricBatch []*PrometheusMetric type PrometheusMetric struct { - tags map[string]string - metricName string - metricValue float64 - metricType string - timeInMS int64 // Unix time in milli-seconds + tags map[string]string + metricName string + // We use this name to look up metric type because user can relabel __name___. + // See https://github.com/aws/amazon-cloudwatch-agent/issues/190 + metricNameBeforeRelabel string + metricValue float64 + metricType string + timeInMS int64 // Unix time in milli-seconds } func (pm *PrometheusMetric) isValueValid() bool { @@ -71,9 +74,10 @@ func (ma *metricAppender) Add(ls labels.Labels, t int64, v float64) (uint64, err } pm := &PrometheusMetric{ - metricName: metricName, - metricValue: v, - timeInMS: t, + metricName: metricName, + metricNameBeforeRelabel: ls.Get(savedScrapeNameLabel), + metricValue: v, + timeInMS: t, } pm.tags = labelMap diff --git a/plugins/inputs/prometheus_scraper/metrics_type_handler.go b/plugins/inputs/prometheus_scraper/metrics_type_handler.go index 3adf63e213..f4b7daff9f 100644 --- a/plugins/inputs/prometheus_scraper/metrics_type_handler.go +++ b/plugins/inputs/prometheus_scraper/metrics_type_handler.go @@ -162,6 +162,7 @@ func (mth *metricsTypeHandler) Handle(pmb PrometheusMetricBatch) (result Prometh return result } + // TODO(pingleig): we can save job as we did with __name__. mc, err := mth.ms.Get(jobName, instanceId) if err != nil { log.Printf("E! metricsTypeHandler.mc.Get(jobName, instanceId) error. jobName: %v; instanceId: %v \n", jobName, instanceId) @@ -170,18 +171,22 @@ func (mth *metricsTypeHandler) Handle(pmb PrometheusMetricBatch) (result Prometh return result } for _, pm := range pmb { + // log for https://github.com/aws/amazon-cloudwatch-agent/issues/190 + if pm.metricNameBeforeRelabel != pm.metricName { + log.Printf("D! metric name changed from %q to %q during relabel", pm.metricNameBeforeRelabel, pm.metricName) + } // normalize the summary metric first, then if metric name == standardMetricName, it means it is not been normalized by summary // , then normalize the counter suffix if it failed to find metadata. - standardMetricName := normalizeMetricName(pm.metricName, histogramSummarySuffixes) + standardMetricName := normalizeMetricName(pm.metricNameBeforeRelabel, histogramSummarySuffixes) mm, ok := mc.Metadata(standardMetricName) if !ok { if pm.metricName != standardMetricName { // perform a 2nd lookup with the original metric name // It could happen if non histogram/summary ends with one of those _count/_sum suffixes - mm, ok = mc.Metadata(pm.metricName) + mm, ok = mc.Metadata(pm.metricNameBeforeRelabel) } else { // normalize the counter type suffixes, like "_total" suffix - standardMetricName = normalizeMetricName(pm.metricName, counterSuffixes) + standardMetricName = normalizeMetricName(pm.metricNameBeforeRelabel, counterSuffixes) mm, ok = mc.Metadata(standardMetricName) } } @@ -200,6 +205,8 @@ func (mth *metricsTypeHandler) Handle(pmb PrometheusMetricBatch) (result Prometh // skip the non-internal metrics with empty metric type due to cache not ready continue } + // Remove magic labels + delete(pm.tags, savedScrapeNameLabel) result = append(result, pm) } diff --git a/plugins/inputs/prometheus_scraper/start.go b/plugins/inputs/prometheus_scraper/start.go index a19c4852f6..7aaeb85e06 100644 --- a/plugins/inputs/prometheus_scraper/start.go +++ b/plugins/inputs/prometheus_scraper/start.go @@ -32,11 +32,13 @@ import ( "github.com/oklog/run" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" "github.com/prometheus/common/promlog" "github.com/prometheus/common/version" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" sdConfig "github.com/prometheus/prometheus/discovery/config" + "github.com/prometheus/prometheus/pkg/relabel" promRuntime "github.com/prometheus/prometheus/pkg/runtime" "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/storage" @@ -260,6 +262,10 @@ func Start(configFilePath string, receiver storage.Appendable, shutDownChan chan wg.Done() } +const ( + savedScrapeNameLabel = "cwagent_saved_scrape_name" // just arbitrary name that end user won't override in relabel config +) + func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config) error) (err error) { level.Info(logger).Log("msg", "Loading configuration file", "filename", filename) @@ -277,6 +283,23 @@ func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config return errors.Wrapf(err, "couldn't load configuration (--config.file=%q)", filename) } + // For saving name before relabel https://github.com/aws/amazon-cloudwatch-agent/issues/190 + for _, scrapeConfig := range conf.ScrapeConfigs { + // We only got __name__ after scrape, so it's in metric_relabel_configs instead of relabel_configs. + metricRelabelConfigs := []*relabel.Config{ + { + Action: relabel.Replace, + Regex: relabel.MustNewRegexp("(.*)"), + Replacement: "$1", + TargetLabel: savedScrapeNameLabel, + SourceLabels: model.LabelNames{"__name__"}, + }, + } + level.Info(logger).Log("msg", "Add extra metric_relabel_configs", "configs", metricRelabelConfigs) + // prepend so our relabel rule comes first + scrapeConfig.MetricRelabelConfigs = append(metricRelabelConfigs, scrapeConfig.MetricRelabelConfigs...) + } + failed := false for _, rl := range rls { if err := rl(conf); err != nil { From d07cd2cec987fb8a917609cebeb788ccd2c2752c Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Mon, 22 Mar 2021 15:44:16 -0700 Subject: [PATCH 2/3] prom: Fix broken unit test for metricNameBeforeRelabel --- .../metric_type_handler_test.go | 109 ++++++++++-------- 1 file changed, 64 insertions(+), 45 deletions(-) diff --git a/plugins/inputs/prometheus_scraper/metric_type_handler_test.go b/plugins/inputs/prometheus_scraper/metric_type_handler_test.go index 3ae0c1f98b..004a6bc37f 100644 --- a/plugins/inputs/prometheus_scraper/metric_type_handler_test.go +++ b/plugins/inputs/prometheus_scraper/metric_type_handler_test.go @@ -186,12 +186,14 @@ func TestNewMetricsTypeHandler_HandleWithUnknownTarget(t *testing.T) { pmb := make(PrometheusMetricBatch, 0) pmb = append(pmb, &PrometheusMetric{ - metricName: "m1", - tags: map[string]string{"job": "job_unknown", "instance": "instance_unknown"}, + metricName: "m1", + metricNameBeforeRelabel: "m1", + tags: map[string]string{"job": "job_unknown", "instance": "instance_unknown"}, }, &PrometheusMetric{ - metricName: "m2", - tags: map[string]string{"job": "job_unknown", "instance": "instance_unknown"}, + metricName: "m2", + metricNameBeforeRelabel: "m2", + tags: map[string]string{"job": "job_unknown", "instance": "instance_unknown"}, }) result := metricsTypeHandler.Handle(pmb) @@ -204,29 +206,34 @@ func TestNewMetricsTypeHandler_HandleWithNormalTarget(t *testing.T) { pmb := make(PrometheusMetricBatch, 0) pmb = append(pmb, &PrometheusMetric{ - metricName: "m3", - tags: map[string]string{"job": "job1", "instance": "instance1"}, + metricName: "m3", + metricNameBeforeRelabel: "m3", + tags: map[string]string{"job": "job1", "instance": "instance1"}, }, &PrometheusMetric{ - metricName: "m1", - tags: map[string]string{"job": "job1", "instance": "instance1"}, + metricName: "m1", + metricNameBeforeRelabel: "m1", + tags: map[string]string{"job": "job1", "instance": "instance1"}, }, &PrometheusMetric{ - metricName: "m2", - tags: map[string]string{"job": "job1", "instance": "instance1"}, + metricName: "m2", + metricNameBeforeRelabel: "m2", + tags: map[string]string{"job": "job1", "instance": "instance1"}, }) result := metricsTypeHandler.Handle(pmb) assert.Equal(t, 2, len(result)) expectedMetric1 := PrometheusMetric{ - metricName: "m1", - metricType: textparse.MetricTypeCounter, - tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, + metricName: "m1", + metricNameBeforeRelabel: "m1", + metricType: textparse.MetricTypeCounter, + tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, } expectedMetric2 := PrometheusMetric{ - metricName: "m2", - metricType: textparse.MetricTypeCounter, - tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, + metricName: "m2", + metricNameBeforeRelabel: "m2", + metricType: textparse.MetricTypeCounter, + tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, } assert.Equal(t, *result[0], expectedMetric1) assert.Equal(t, *result[1], expectedMetric2) @@ -238,29 +245,34 @@ func TestNewMetricsTypeHandler_HandleWithReplacedJobname(t *testing.T) { pmb := make(PrometheusMetricBatch, 0) pmb = append(pmb, &PrometheusMetric{ - metricName: "m1", - tags: map[string]string{"job": "job2_replaced", "instance": "instance2"}, + metricName: "m1", + metricNameBeforeRelabel: "m1", + tags: map[string]string{"job": "job2_replaced", "instance": "instance2"}, }, &PrometheusMetric{ - metricName: "m3", - tags: map[string]string{"job": "job2_replaced", "instance": "instance2"}, + metricName: "m3", + metricNameBeforeRelabel: "m3", + tags: map[string]string{"job": "job2_replaced", "instance": "instance2"}, }, &PrometheusMetric{ - metricName: "m2", - tags: map[string]string{"job": "job2_replaced", "instance": "instance2"}, + metricName: "m2", + metricNameBeforeRelabel: "m2", + tags: map[string]string{"job": "job2_replaced", "instance": "instance2"}, }) result := metricsTypeHandler.Handle(pmb) assert.Equal(t, 2, len(result)) expectedMetric1 := PrometheusMetric{ - metricName: "m1", - metricType: textparse.MetricTypeGauge, - tags: map[string]string{"job": "job2_replaced", "instance": "instance2", "prom_metric_type": textparse.MetricTypeGauge}, + metricName: "m1", + metricNameBeforeRelabel: "m1", + metricType: textparse.MetricTypeGauge, + tags: map[string]string{"job": "job2_replaced", "instance": "instance2", "prom_metric_type": textparse.MetricTypeGauge}, } expectedMetric2 := PrometheusMetric{ - metricName: "m2", - metricType: textparse.MetricTypeGauge, - tags: map[string]string{"job": "job2_replaced", "instance": "instance2", "prom_metric_type": textparse.MetricTypeGauge}, + metricName: "m2", + metricNameBeforeRelabel: "m2", + metricType: textparse.MetricTypeGauge, + tags: map[string]string{"job": "job2_replaced", "instance": "instance2", "prom_metric_type": textparse.MetricTypeGauge}, } assert.Equal(t, *result[0], expectedMetric1) assert.Equal(t, *result[1], expectedMetric2) @@ -272,38 +284,45 @@ func TestNewMetricsTypeHandler_HandleWithMetricSuffix(t *testing.T) { pmb := make(PrometheusMetricBatch, 0) pmb = append(pmb, &PrometheusMetric{ - metricName: "m3_sum", - tags: map[string]string{"job": "job1", "instance": "instance1"}, + metricName: "m3_sum", + metricNameBeforeRelabel: "m3_sum", + tags: map[string]string{"job": "job1", "instance": "instance1"}, }, &PrometheusMetric{ - metricName: "m1_sum", - tags: map[string]string{"job": "job1", "instance": "instance1"}, + metricName: "m1_sum", + metricNameBeforeRelabel: "m1_sum", + tags: map[string]string{"job": "job1", "instance": "instance1"}, }, &PrometheusMetric{ - metricName: "m2_count", - tags: map[string]string{"job": "job1", "instance": "instance1"}, + metricName: "m2_count", + metricNameBeforeRelabel: "m2_count", + tags: map[string]string{"job": "job1", "instance": "instance1"}, }, &PrometheusMetric{ - metricName: "m4_total", - tags: map[string]string{"job": "job1", "instance": "instance1"}, + metricName: "m4_total", + metricNameBeforeRelabel: "m4_total", + tags: map[string]string{"job": "job1", "instance": "instance1"}, }) result := metricsTypeHandler.Handle(pmb) assert.Equal(t, 3, len(result)) expectedMetric1 := PrometheusMetric{ - metricName: "m1_sum", - metricType: textparse.MetricTypeCounter, - tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, + metricName: "m1_sum", + metricNameBeforeRelabel: "m1_sum", + metricType: textparse.MetricTypeCounter, + tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, } expectedMetric2 := PrometheusMetric{ - metricName: "m2_count", - metricType: textparse.MetricTypeCounter, - tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, + metricName: "m2_count", + metricNameBeforeRelabel: "m2_count", + metricType: textparse.MetricTypeCounter, + tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, } expectedMetric4 := PrometheusMetric{ - metricName: "m4_total", - metricType: textparse.MetricTypeCounter, - tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, + metricName: "m4_total", + metricNameBeforeRelabel: "m4_total", + metricType: textparse.MetricTypeCounter, + tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, } assert.Equal(t, *result[0], expectedMetric1) assert.Equal(t, *result[1], expectedMetric2) From 2cb9c5785f754f6d8be1951c4a104c7b0928995b Mon Sep 17 00:00:00 2001 From: Pinglei Guo Date: Mon, 22 Mar 2021 15:54:05 -0700 Subject: [PATCH 3/3] prom: Add test for metric name relabel --- .../metric_type_handler_test.go | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/plugins/inputs/prometheus_scraper/metric_type_handler_test.go b/plugins/inputs/prometheus_scraper/metric_type_handler_test.go index 004a6bc37f..8d030e23fb 100644 --- a/plugins/inputs/prometheus_scraper/metric_type_handler_test.go +++ b/plugins/inputs/prometheus_scraper/metric_type_handler_test.go @@ -328,3 +328,44 @@ func TestNewMetricsTypeHandler_HandleWithMetricSuffix(t *testing.T) { assert.Equal(t, *result[1], expectedMetric2) assert.Equal(t, *result[2], expectedMetric4) } + +// https://github.com/aws/amazon-cloudwatch-agent/issues/190 +func TestNewMetricsTypeHandler_HandleRelabelName(t *testing.T) { + metricsTypeHandler := NewMetricsTypeHandler() + metricsTypeHandler.SetScrapeManager(&mockScrapeManager{}) + pmb := make(PrometheusMetricBatch, 0) + pmb = append(pmb, + &PrometheusMetric{ + metricName: "m3_changed", + metricNameBeforeRelabel: "m3", + tags: map[string]string{"job": "job1", "instance": "instance1", savedScrapeNameLabel: "m3"}, + }, + &PrometheusMetric{ + metricName: "m1", + metricNameBeforeRelabel: "m1", + tags: map[string]string{"job": "job1", "instance": "instance1", savedScrapeNameLabel: "m1"}, + }, + &PrometheusMetric{ + metricName: "m2_changed", + metricNameBeforeRelabel: "m2", + tags: map[string]string{"job": "job1", "instance": "instance1", savedScrapeNameLabel: "m2"}, + }) + + result := metricsTypeHandler.Handle(pmb) + assert.Equal(t, 2, len(result)) + expectedMetric1 := PrometheusMetric{ + metricName: "m1", + metricNameBeforeRelabel: "m1", + metricType: textparse.MetricTypeCounter, + // The saved label should be gone + tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, + } + expectedMetric2 := PrometheusMetric{ + metricName: "m2_changed", + metricNameBeforeRelabel: "m2", + metricType: textparse.MetricTypeCounter, + tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter}, + } + assert.Equal(t, *result[0], expectedMetric1) + assert.Equal(t, *result[1], expectedMetric2) +}