Skip to content

Commit

Permalink
use the prometheus.type metric metadata to support double-writing unk…
Browse files Browse the repository at this point in the history 
…nown-typed metrics
  • Loading branch information
@dashpole
dashpole committed Apr 23, 2024
1 parent 6aa1db0 commit 04bb441
Show file tree
Hide file tree
Showing 8 changed files with 66 additions and 175 deletions.
36 changes: 12 additions & 24 deletions exporter/collector/googlemanagedprometheus/extra_metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,13 @@ package googlemanagedprometheus
import (
"time"

"go.opentelemetry.io/collector/featuregate"
"github.com/prometheus/common/model"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/pmetric"
semconv "go.opentelemetry.io/collector/semconv/v1.18.0"
)

const (
// Special attribute key used by Ops Agent prometheus receiver to denote untyped
// prometheus metric. Internal use only.
GCPOpsAgentUntypedMetricKey = "prometheus.googleapis.com/internal/untyped_metric"

gcpUntypedDoubleExportGateKey = "gcp.untypedDoubleExport"
)

var untypedDoubleExportFeatureGate = featuregate.GlobalRegistry().MustRegister(
gcpUntypedDoubleExportGateKey,
featuregate.StageAlpha,
featuregate.WithRegisterFromVersion("v0.77.0"),
featuregate.WithRegisterDescription("Enable automatically exporting untyped Prometheus metrics as both gauge and cumulative to GCP."),
featuregate.WithRegisterReferenceURL("https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/pull/668"))
const prometheusMetricMetadataTypeKey = "prometheus.type"

func (c Config) ExtraMetrics(m pmetric.Metrics) {
addUntypedMetrics(m)
Expand All @@ -47,9 +34,6 @@ func (c Config) ExtraMetrics(m pmetric.Metrics) {
// addUntypedMetrics looks for any Gauge data point with the special Ops Agent untyped metric
// attribute and duplicates that data point to a matching Sum.
func addUntypedMetrics(m pmetric.Metrics) {
if !untypedDoubleExportFeatureGate.IsEnabled() {
return
}
rms := m.ResourceMetrics()
for i := 0; i < rms.Len(); i++ {
rm := rms.At(i)
Expand All @@ -69,17 +53,16 @@ func addUntypedMetrics(m pmetric.Metrics) {
continue
}

if !isUnknown(metric) {
continue
}

// attribute is set on the data point
gauge := metric.Gauge()
points := gauge.DataPoints()
for l := 0; l < points.Len(); l++ {
point := points.At(l)
val, ok := point.Attributes().Get(GCPOpsAgentUntypedMetricKey)
if !(ok && val.AsString() == "true") {
continue
}

// Add the new Sum metric and copy values over from this Gauge
point := points.At(l)
newMetric := mes.AppendEmpty()
newMetric.SetName(metric.Name())
newMetric.SetDescription(metric.Description())
Expand All @@ -105,6 +88,11 @@ func addUntypedMetrics(m pmetric.Metrics) {
}
}

func isUnknown(metric pmetric.Metric) bool {
originalType, ok := metric.Metadata().Get(prometheusMetricMetadataTypeKey)
return ok && originalType.Str() == string(model.MetricTypeUnknown)
}

// resourceID identifies a resource. We only send one target info for each
// resourceID.
type resourceID struct {
Expand Down
104 changes: 32 additions & 72 deletions exporter/collector/googlemanagedprometheus/extra_metrics_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"time"

"github.com/stretchr/testify/assert"
"go.opentelemetry.io/collector/featuregate"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/pmetric"
semconv "go.opentelemetry.io/collector/semconv/v1.18.0"
Expand Down Expand Up @@ -53,11 +52,10 @@ func appendMetric(metrics pmetric.Metrics, timestamp time.Time) pmetric.Metrics
func TestAddExtraMetrics(t *testing.T) {
timestamp := time.Now()
for _, tc := range []struct {
input pmetric.Metrics
expected pmetric.ResourceMetricsSlice
name string
config Config
enableUntypedFeatureGate bool
input pmetric.Metrics
expected pmetric.ResourceMetricsSlice
name string
config Config
}{
{
name: "add target info from resource metric",
Expand Down Expand Up @@ -322,49 +320,43 @@ func TestAddExtraMetrics(t *testing.T) {
}(),
},
{
name: "add untyped Sum metric from Gauge",
enableUntypedFeatureGate: true,
config: Config{},
name: "add untyped Sum metric from Gauge",
config: Config{},
input: func() pmetric.Metrics {
metrics := testMetric(timestamp)
metrics.ResourceMetrics().At(0).
ScopeMetrics().At(0).
Metrics().At(0).
Gauge().DataPoints().At(0).
Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "true")
Metadata().PutStr("prometheus.type", "unknown")
return metrics
}(),
expected: func() pmetric.ResourceMetricsSlice {
metrics := testMetric(timestamp).ResourceMetrics()

dataPoint := metrics.At(0).
metrics.At(0).
ScopeMetrics().At(0).
Metrics().At(0).
Gauge().DataPoints().At(0)
dataPoint.Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "true")
Metadata().PutStr("prometheus.type", "unknown")

metric := metrics.At(0).ScopeMetrics().At(0).Metrics().AppendEmpty()
metric.SetName("baz-metric")
metric.SetEmptySum().DataPoints().AppendEmpty().SetIntValue(2112)
metric.Sum().SetIsMonotonic(true)
metric.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
metric.Sum().DataPoints().At(0).SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
metric.Sum().DataPoints().At(0).Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "true")

return metrics
}(),
},
{
name: "add untyped Sum metric from Gauge (double value)",
enableUntypedFeatureGate: true,
config: Config{},
name: "add untyped Sum metric from Gauge (double value)",
config: Config{},
input: func() pmetric.Metrics {
metrics := testMetric(timestamp)
metrics.ResourceMetrics().At(0).
ScopeMetrics().At(0).
Metrics().At(0).
Gauge().DataPoints().At(0).
Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "true")
Metadata().PutStr("prometheus.type", "unknown")
metrics.ResourceMetrics().At(0).
ScopeMetrics().At(0).
Metrics().At(0).
Expand All @@ -375,100 +367,68 @@ func TestAddExtraMetrics(t *testing.T) {
expected: func() pmetric.ResourceMetricsSlice {
metrics := testMetric(timestamp).ResourceMetrics()

dataPoint := metrics.At(0).
metrics.At(0).ScopeMetrics().At(0).Metrics().At(0).Metadata().PutStr("prometheus.type", "unknown")
metrics.At(0).
ScopeMetrics().At(0).
Metrics().At(0).
Gauge().DataPoints().At(0)
dataPoint.Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "true")
dataPoint.SetDoubleValue(123.5)
Gauge().DataPoints().At(0).SetDoubleValue(123.5)

metric := metrics.At(0).ScopeMetrics().At(0).Metrics().AppendEmpty()
metric.SetName("baz-metric")
metric.SetEmptySum().DataPoints().AppendEmpty().SetDoubleValue(123.5)
metric.Sum().SetIsMonotonic(true)
metric.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
metric.Sum().DataPoints().At(0).SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
metric.Sum().DataPoints().At(0).Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "true")

return metrics
}(),
},
{
name: "untyped Gauge does nothing if feature gate is disabled",
enableUntypedFeatureGate: false,
config: Config{},
input: func() pmetric.Metrics {
metrics := testMetric(timestamp)
metrics.ResourceMetrics().At(0).
ScopeMetrics().At(0).
Metrics().At(0).
Gauge().DataPoints().At(0).
Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "true")
return metrics
}(),
expected: func() pmetric.ResourceMetricsSlice {
metrics := testMetric(timestamp).ResourceMetrics()
dataPoint := metrics.At(0).
ScopeMetrics().At(0).
Metrics().At(0).
Gauge().DataPoints().At(0)
dataPoint.Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "true")
return metrics
}(),
},
{
name: "untyped Gauge does nothing if feature gate is enabled and key!=true",
enableUntypedFeatureGate: true,
config: Config{},
name: "untyped Gauge does nothing if feature gate is enabled and key!=unknown",
config: Config{},
input: func() pmetric.Metrics {
metrics := testMetric(timestamp)
metrics.ResourceMetrics().At(0).
ScopeMetrics().At(0).
Metrics().At(0).
Gauge().DataPoints().At(0).
Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "foo")
Metadata().PutStr("prometheus.type", "foo")
return metrics
}(),
expected: func() pmetric.ResourceMetricsSlice {
metrics := testMetric(timestamp).ResourceMetrics()
dataPoint := metrics.At(0).
metrics.At(0).
ScopeMetrics().At(0).
Metrics().At(0).
Gauge().DataPoints().At(0)
dataPoint.Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "foo")
Metadata().PutStr("prometheus.type", "foo")
return metrics
}(),
},
{
name: "untyped non-Gauge does nothing if feature gate is enabled",
enableUntypedFeatureGate: true,
config: Config{},
name: "untyped non-Gauge does nothing if feature gate is enabled",
config: Config{},
input: func() pmetric.Metrics {
metrics := testMetric(timestamp)
metrics.ResourceMetrics().At(0).
metric := metrics.ResourceMetrics().At(0).
ScopeMetrics().At(0).
Metrics().AppendEmpty().SetEmptyHistogram().DataPoints().AppendEmpty().
Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "true")
Metrics().AppendEmpty()

metric.SetEmptyHistogram().DataPoints().AppendEmpty()
metric.Metadata().PutStr("prometheus.type", "unknown")
return metrics
}(),
expected: func() pmetric.ResourceMetricsSlice {
metrics := testMetric(timestamp).ResourceMetrics()
metrics.At(0).
metric := metrics.At(0).
ScopeMetrics().At(0).
Metrics().AppendEmpty().SetEmptyHistogram().DataPoints().AppendEmpty().
Attributes().PutStr(GCPOpsAgentUntypedMetricKey, "true")
Metrics().AppendEmpty()

metric.SetEmptyHistogram().DataPoints().AppendEmpty()
metric.Metadata().PutStr("prometheus.type", "unknown")
return metrics
}(),
},
} {
t.Run(tc.name, func(t *testing.T) {
if tc.enableUntypedFeatureGate {
assert.NoError(t, featuregate.GlobalRegistry().Set(gcpUntypedDoubleExportGateKey, true))
// disable it after the test is over
defer func() {
assert.NoError(t, featuregate.GlobalRegistry().Set(gcpUntypedDoubleExportGateKey, false))
}()
}
m := tc.input
tc.config.ExtraMetrics(m)
assert.EqualValues(t, tc.expected, m.ResourceMetrics())
Expand Down
2 changes: 2 additions & 0 deletions exporter/collector/googlemanagedprometheus/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ toolchain go1.22.0

require (
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.99.0
github.com/prometheus/common v0.53.0
github.com/stretchr/testify v1.9.0
go.opentelemetry.io/collector/featuregate v1.6.0
go.opentelemetry.io/collector/pdata v1.6.0
Expand All @@ -21,6 +22,7 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/sys v0.18.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions exporter/collector/googlemanagedprometheus/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometh
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.99.0/go.mod h1:sQID67FDKapC9OJNgVxcG7MscZRpWIfgWrSodEZGedo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos=
github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8=
github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE=
github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand Down
30 changes: 9 additions & 21 deletions exporter/collector/googlemanagedprometheus/naming.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

// GetMetricName returns the metric name with GMP-specific suffixes. The.
func (c Config) GetMetricName(baseName string, metric pmetric.Metric) (string, error) {
unknown := isUnknown(metric)
// First, build a name that is compliant with prometheus conventions
compliantName := prometheus.BuildCompliantName(metric, "", c.AddMetricSuffixes)
// Second, ad the GMP-specific suffix
Expand All @@ -34,9 +35,9 @@ func (c Config) GetMetricName(baseName string, metric pmetric.Metric) (string, e
// Non-monotonic sums are converted to GCM gauges
return compliantName + "/gauge", nil
}
return getUnknownMetricName(metric.Sum().DataPoints(), "/counter", "counter", metric.Name(), compliantName), nil
return getUnknownMetricName(metric.Sum().DataPoints(), unknown, "/counter", "counter", metric.Name(), compliantName), nil
case pmetric.MetricTypeGauge:
return getUnknownMetricName(metric.Gauge().DataPoints(), "/gauge", "", metric.Name(), compliantName), nil
return getUnknownMetricName(metric.Gauge().DataPoints(), unknown, "/gauge", "", metric.Name(), compliantName), nil
case pmetric.MetricTypeSummary:
// summaries are sent as the following series:
// * Sum: prometheus.googleapis.com/<baseName>_sum/summary:counter
Expand All @@ -60,35 +61,22 @@ func (c Config) GetMetricName(baseName string, metric pmetric.Metric) (string, e
// "/unknown" (eg, for Gauge) or "/unknown:{secondarySuffix}" (eg, "/unknown:counter" for Sum).
// It also removes the "_total" suffix on an unknown counter, if this suffix was not present in
// the original metric name before calling prometheus.BuildCompliantName(), which is hacky.
// It is based on the untyped_prometheus_metric data point attribute, and behind a feature gate.
func getUnknownMetricName(points pmetric.NumberDataPointSlice, suffix, secondarySuffix, originalName, compliantName string) string {
if !untypedDoubleExportFeatureGate.IsEnabled() {
return compliantName + suffix
}

func getUnknownMetricName(points pmetric.NumberDataPointSlice, unknown bool, suffix, secondarySuffix, originalName, compliantName string) string {
nameTokens := strings.FieldsFunc(
originalName,
func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) },
)

untyped := false
newSuffix := suffix
for i := 0; i < points.Len(); i++ {
point := points.At(i)
if val, ok := point.Attributes().Get(GCPOpsAgentUntypedMetricKey); ok && val.AsString() == "true" {
untyped = true
// delete the special Ops Agent untyped attribute
point.Attributes().Remove(GCPOpsAgentUntypedMetricKey)
newSuffix = "/unknown"
if len(secondarySuffix) > 0 {
newSuffix = newSuffix + ":" + secondarySuffix
}
// even though we have the suffix, keep looping to remove the attribute from other points, if any
if unknown {
newSuffix = "/unknown"
if len(secondarySuffix) > 0 {
newSuffix = newSuffix + ":" + secondarySuffix
}
}

// de-normalize "_total" suffix for counters where not present on original metric name
if untyped && nameTokens[len(nameTokens)-1] != "total" && strings.HasSuffix(compliantName, "_total") {
if unknown && nameTokens[len(nameTokens)-1] != "total" && strings.HasSuffix(compliantName, "_total") {
compliantName = strings.TrimSuffix(compliantName, "_total")
}
return compliantName + newSuffix
Expand Down
Loading

0 comments on commit 04bb441

Please sign in to comment.