Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

prom: Save __name__ before relabel for look up metric type #191

Merged
merged 3 commits into from
Mar 23, 2021
Merged

prom: Save __name__ before relabel for look up metric type #191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f1f0b52
prom: Save __name__ before relabel for look up metric type #190
pingleig Mar 22, 2021
d07cd2c
prom: Fix broken unit test for metricNameBeforeRelabel
pingleig Mar 22, 2021
2cb9c57
prom: Add test for metric name relabel
pingleig Mar 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 105 additions & 45 deletions plugins/inputs/prometheus_scraper/metric_type_handler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,12 +186,14 @@ func TestNewMetricsTypeHandler_HandleWithUnknownTarget(t *testing.T) {
pmb := make(PrometheusMetricBatch, 0)
pmb = append(pmb,
&PrometheusMetric{
metricName: "m1",
tags: map[string]string{"job": "job_unknown", "instance": "instance_unknown"},
metricName: "m1",
metricNameBeforeRelabel: "m1",
tags: map[string]string{"job": "job_unknown", "instance": "instance_unknown"},
},
&PrometheusMetric{
metricName: "m2",
tags: map[string]string{"job": "job_unknown", "instance": "instance_unknown"},
metricName: "m2",
metricNameBeforeRelabel: "m2",
tags: map[string]string{"job": "job_unknown", "instance": "instance_unknown"},
})

result := metricsTypeHandler.Handle(pmb)
Expand All @@ -204,29 +206,34 @@ func TestNewMetricsTypeHandler_HandleWithNormalTarget(t *testing.T) {
pmb := make(PrometheusMetricBatch, 0)
pmb = append(pmb,
&PrometheusMetric{
metricName: "m3",
tags: map[string]string{"job": "job1", "instance": "instance1"},
metricName: "m3",
metricNameBeforeRelabel: "m3",
tags: map[string]string{"job": "job1", "instance": "instance1"},
},
&PrometheusMetric{
metricName: "m1",
tags: map[string]string{"job": "job1", "instance": "instance1"},
metricName: "m1",
metricNameBeforeRelabel: "m1",
tags: map[string]string{"job": "job1", "instance": "instance1"},
},
&PrometheusMetric{
metricName: "m2",
tags: map[string]string{"job": "job1", "instance": "instance1"},
metricName: "m2",
metricNameBeforeRelabel: "m2",
tags: map[string]string{"job": "job1", "instance": "instance1"},
})

result := metricsTypeHandler.Handle(pmb)
assert.Equal(t, 2, len(result))
expectedMetric1 := PrometheusMetric{
metricName: "m1",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
metricName: "m1",
metricNameBeforeRelabel: "m1",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
}
expectedMetric2 := PrometheusMetric{
metricName: "m2",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
metricName: "m2",
metricNameBeforeRelabel: "m2",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
}
assert.Equal(t, *result[0], expectedMetric1)
assert.Equal(t, *result[1], expectedMetric2)
Expand All @@ -238,29 +245,34 @@ func TestNewMetricsTypeHandler_HandleWithReplacedJobname(t *testing.T) {
pmb := make(PrometheusMetricBatch, 0)
pmb = append(pmb,
&PrometheusMetric{
metricName: "m1",
tags: map[string]string{"job": "job2_replaced", "instance": "instance2"},
metricName: "m1",
metricNameBeforeRelabel: "m1",
tags: map[string]string{"job": "job2_replaced", "instance": "instance2"},
},
&PrometheusMetric{
metricName: "m3",
tags: map[string]string{"job": "job2_replaced", "instance": "instance2"},
metricName: "m3",
metricNameBeforeRelabel: "m3",
tags: map[string]string{"job": "job2_replaced", "instance": "instance2"},
},
&PrometheusMetric{
metricName: "m2",
tags: map[string]string{"job": "job2_replaced", "instance": "instance2"},
metricName: "m2",
metricNameBeforeRelabel: "m2",
tags: map[string]string{"job": "job2_replaced", "instance": "instance2"},
})

result := metricsTypeHandler.Handle(pmb)
assert.Equal(t, 2, len(result))
expectedMetric1 := PrometheusMetric{
metricName: "m1",
metricType: textparse.MetricTypeGauge,
tags: map[string]string{"job": "job2_replaced", "instance": "instance2", "prom_metric_type": textparse.MetricTypeGauge},
metricName: "m1",
metricNameBeforeRelabel: "m1",
metricType: textparse.MetricTypeGauge,
tags: map[string]string{"job": "job2_replaced", "instance": "instance2", "prom_metric_type": textparse.MetricTypeGauge},
}
expectedMetric2 := PrometheusMetric{
metricName: "m2",
metricType: textparse.MetricTypeGauge,
tags: map[string]string{"job": "job2_replaced", "instance": "instance2", "prom_metric_type": textparse.MetricTypeGauge},
metricName: "m2",
metricNameBeforeRelabel: "m2",
metricType: textparse.MetricTypeGauge,
tags: map[string]string{"job": "job2_replaced", "instance": "instance2", "prom_metric_type": textparse.MetricTypeGauge},
}
assert.Equal(t, *result[0], expectedMetric1)
assert.Equal(t, *result[1], expectedMetric2)
Expand All @@ -272,40 +284,88 @@ func TestNewMetricsTypeHandler_HandleWithMetricSuffix(t *testing.T) {
pmb := make(PrometheusMetricBatch, 0)
pmb = append(pmb,
&PrometheusMetric{
metricName: "m3_sum",
tags: map[string]string{"job": "job1", "instance": "instance1"},
metricName: "m3_sum",
metricNameBeforeRelabel: "m3_sum",
tags: map[string]string{"job": "job1", "instance": "instance1"},
},
&PrometheusMetric{
metricName: "m1_sum",
tags: map[string]string{"job": "job1", "instance": "instance1"},
metricName: "m1_sum",
metricNameBeforeRelabel: "m1_sum",
tags: map[string]string{"job": "job1", "instance": "instance1"},
},
&PrometheusMetric{
metricName: "m2_count",
tags: map[string]string{"job": "job1", "instance": "instance1"},
metricName: "m2_count",
metricNameBeforeRelabel: "m2_count",
tags: map[string]string{"job": "job1", "instance": "instance1"},
},
&PrometheusMetric{
metricName: "m4_total",
tags: map[string]string{"job": "job1", "instance": "instance1"},
metricName: "m4_total",
metricNameBeforeRelabel: "m4_total",
tags: map[string]string{"job": "job1", "instance": "instance1"},
})

result := metricsTypeHandler.Handle(pmb)
assert.Equal(t, 3, len(result))
expectedMetric1 := PrometheusMetric{
metricName: "m1_sum",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
metricName: "m1_sum",
metricNameBeforeRelabel: "m1_sum",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
}
expectedMetric2 := PrometheusMetric{
metricName: "m2_count",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
metricName: "m2_count",
metricNameBeforeRelabel: "m2_count",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
}
expectedMetric4 := PrometheusMetric{
metricName: "m4_total",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
metricName: "m4_total",
metricNameBeforeRelabel: "m4_total",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
}
assert.Equal(t, *result[0], expectedMetric1)
assert.Equal(t, *result[1], expectedMetric2)
assert.Equal(t, *result[2], expectedMetric4)
}

// https://github.com/aws/amazon-cloudwatch-agent/issues/190
func TestNewMetricsTypeHandler_HandleRelabelName(t *testing.T) {
metricsTypeHandler := NewMetricsTypeHandler()
metricsTypeHandler.SetScrapeManager(&mockScrapeManager{})
pmb := make(PrometheusMetricBatch, 0)
pmb = append(pmb,
&PrometheusMetric{
metricName: "m3_changed",
metricNameBeforeRelabel: "m3",
tags: map[string]string{"job": "job1", "instance": "instance1", savedScrapeNameLabel: "m3"},
},
&PrometheusMetric{
metricName: "m1",
metricNameBeforeRelabel: "m1",
tags: map[string]string{"job": "job1", "instance": "instance1", savedScrapeNameLabel: "m1"},
},
&PrometheusMetric{
metricName: "m2_changed",
metricNameBeforeRelabel: "m2",
tags: map[string]string{"job": "job1", "instance": "instance1", savedScrapeNameLabel: "m2"},
})

result := metricsTypeHandler.Handle(pmb)
assert.Equal(t, 2, len(result))
expectedMetric1 := PrometheusMetric{
metricName: "m1",
metricNameBeforeRelabel: "m1",
metricType: textparse.MetricTypeCounter,
// The saved label should be gone
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
}
expectedMetric2 := PrometheusMetric{
metricName: "m2_changed",
metricNameBeforeRelabel: "m2",
metricType: textparse.MetricTypeCounter,
tags: map[string]string{"job": "job1", "instance": "instance1", "prom_metric_type": textparse.MetricTypeCounter},
}
assert.Equal(t, *result[0], expectedMetric1)
assert.Equal(t, *result[1], expectedMetric2)
}
20 changes: 12 additions & 8 deletions plugins/inputs/prometheus_scraper/metrics_receiver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,14 @@ import (
type PrometheusMetricBatch []*PrometheusMetric

type PrometheusMetric struct {
tags map[string]string
metricName string
metricValue float64
metricType string
timeInMS int64 // Unix time in milli-seconds
tags map[string]string
metricName string
// We use this name to look up metric type because user can relabel __name___.
// See https://github.com/aws/amazon-cloudwatch-agent/issues/190
metricNameBeforeRelabel string
metricValue float64
metricType string
timeInMS int64 // Unix time in milli-seconds
}

func (pm *PrometheusMetric) isValueValid() bool {
Expand Down Expand Up @@ -71,9 +74,10 @@ func (ma *metricAppender) Add(ls labels.Labels, t int64, v float64) (uint64, err
}

pm := &PrometheusMetric{
metricName: metricName,
metricValue: v,
timeInMS: t,
metricName: metricName,
metricNameBeforeRelabel: ls.Get(savedScrapeNameLabel),
metricValue: v,
timeInMS: t,
}

pm.tags = labelMap
Expand Down
13 changes: 10 additions & 3 deletions plugins/inputs/prometheus_scraper/metrics_type_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ func (mth *metricsTypeHandler) Handle(pmb PrometheusMetricBatch) (result Prometh
return result
}

// TODO(pingleig): we can save job as we did with __name__.
mc, err := mth.ms.Get(jobName, instanceId)
if err != nil {
log.Printf("E! metricsTypeHandler.mc.Get(jobName, instanceId) error. jobName: %v; instanceId: %v \n", jobName, instanceId)
Expand All @@ -170,18 +171,22 @@ func (mth *metricsTypeHandler) Handle(pmb PrometheusMetricBatch) (result Prometh
return result
}
for _, pm := range pmb {
// log for https://github.com/aws/amazon-cloudwatch-agent/issues/190
if pm.metricNameBeforeRelabel != pm.metricName {
log.Printf("D! metric name changed from %q to %q during relabel", pm.metricNameBeforeRelabel, pm.metricName)
}
// normalize the summary metric first, then if metric name == standardMetricName, it means it is not been normalized by summary
// , then normalize the counter suffix if it failed to find metadata.
standardMetricName := normalizeMetricName(pm.metricName, histogramSummarySuffixes)
standardMetricName := normalizeMetricName(pm.metricNameBeforeRelabel, histogramSummarySuffixes)
mm, ok := mc.Metadata(standardMetricName)
if !ok {
if pm.metricName != standardMetricName {
// perform a 2nd lookup with the original metric name
// It could happen if non histogram/summary ends with one of those _count/_sum suffixes
mm, ok = mc.Metadata(pm.metricName)
mm, ok = mc.Metadata(pm.metricNameBeforeRelabel)
} else {
// normalize the counter type suffixes, like "_total" suffix
standardMetricName = normalizeMetricName(pm.metricName, counterSuffixes)
standardMetricName = normalizeMetricName(pm.metricNameBeforeRelabel, counterSuffixes)
mm, ok = mc.Metadata(standardMetricName)
}
}
Expand All @@ -200,6 +205,8 @@ func (mth *metricsTypeHandler) Handle(pmb PrometheusMetricBatch) (result Prometh
// skip the non-internal metrics with empty metric type due to cache not ready
continue
}
// Remove magic labels
delete(pm.tags, savedScrapeNameLabel)
result = append(result, pm)
}

Expand Down
23 changes: 23 additions & 0 deletions plugins/inputs/prometheus_scraper/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,13 @@ import (
"github.com/oklog/run"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"github.com/prometheus/common/promlog"
"github.com/prometheus/common/version"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery"
sdConfig "github.com/prometheus/prometheus/discovery/config"
"github.com/prometheus/prometheus/pkg/relabel"
promRuntime "github.com/prometheus/prometheus/pkg/runtime"
"github.com/prometheus/prometheus/scrape"
"github.com/prometheus/prometheus/storage"
Expand Down Expand Up @@ -260,6 +262,10 @@ func Start(configFilePath string, receiver storage.Appendable, shutDownChan chan
wg.Done()
}

const (
savedScrapeNameLabel = "cwagent_saved_scrape_name" // just arbitrary name that end user won't override in relabel config
)

func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config) error) (err error) {
level.Info(logger).Log("msg", "Loading configuration file", "filename", filename)

Expand All @@ -277,6 +283,23 @@ func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config
return errors.Wrapf(err, "couldn't load configuration (--config.file=%q)", filename)
}

// For saving name before relabel https://github.com/aws/amazon-cloudwatch-agent/issues/190
for _, scrapeConfig := range conf.ScrapeConfigs {
// We only got __name__ after scrape, so it's in metric_relabel_configs instead of relabel_configs.
metricRelabelConfigs := []*relabel.Config{
{
Action: relabel.Replace,
Regex: relabel.MustNewRegexp("(.*)"),
Replacement: "$1",
TargetLabel: savedScrapeNameLabel,
SourceLabels: model.LabelNames{"__name__"},
},
}
level.Info(logger).Log("msg", "Add extra metric_relabel_configs", "configs", metricRelabelConfigs)
// prepend so our relabel rule comes first
scrapeConfig.MetricRelabelConfigs = append(metricRelabelConfigs, scrapeConfig.MetricRelabelConfigs...)
}

failed := false
for _, rl := range rls {
if err := rl(conf); err != nil {
Expand Down