sfxexporter: Add include metrics option

exporter/signalfxexporter/README.md

@@ -45,6 +45,20 @@ The following configuration options can also be configured:
   excluded from sending to Signalfx backend. If `send_compatible_metrics`
   or `translation_rules` options are enabled, the exclusion will be applied
   on translated metrics. See [here](./testdata/config.yaml) for examples.
+- `include_metrics`: List of filters to override exclusion of any metrics.
+  This option can be used to included metrics that are otherwise dropped by
+  default. See [here](./translation/default_metrics.go) for a list of metrics
+  that are dropped by default. For example, the following configuration can be
+  used to send through some of that are dropped by default.
+  ```yaml
+  include_metrics:
+    # When sending in translated metrics.
+    - metric_names: [cpu.interrupt, cpu.user, cpu.system]
+    # When sending in metrics in OTel convention.
+    - metric_name: system.cpu.time
+      dimensions:
+        state: [interrupt, user, system]
+  ```
 - `headers` (no default): Headers to pass in the payload.
 - `log_dimension_updates` (default = `false`): Whether or not to log dimension
   updates.

exporter/signalfxexporter/config.go

@@ -92,6 +92,11 @@ type Config struct {
 	// on translated metrics.
 	ExcludeMetrics []dpfilters.MetricFilter `mapstructure:"exclude_metrics"`
 
+	// IncludeMetrics defines dpfilter.MetricFilters to override exclusion any of metric.
+	// This option can be used to included metrics that are otherwise dropped by default.
+	// See ./translation/default_metrics.go for a list of metrics that are dropped by default.
+	IncludeMetrics []dpfilters.MetricFilter `mapstructure:"include_metrics"`
+
 	// Correlation configuration for syncing traces service and environment to metrics.
 	Correlation *correlation.Config `mapstructure:"correlation"`
 }

exporter/signalfxexporter/config_test.go

@@ -127,6 +127,14 @@ func TestLoadConfig(t *testing.T) {
 				},
 			},
 		},
+		IncludeMetrics: []dpfilters.MetricFilter{
+			{
+				MetricName: "metric1",
+			},
+			{
+				MetricNames: []string{"metric2", "metric3"},
+			},
+		},
 		DeltaTranslationTTL: 3600,
 		Correlation: &correlation.Config{
 			HTTPClientSettings: confighttp.HTTPClientSettings{

exporter/signalfxexporter/exporter.go

@@ -90,7 +90,7 @@ func newSignalFxExporter(
 
 	headers := buildHeaders(config)
 
-	converter, err := translation.NewMetricsConverter(logger, options.metricTranslator, config.ExcludeMetrics)
+	converter, err := translation.NewMetricsConverter(logger, options.metricTranslator, config.ExcludeMetrics, config.IncludeMetrics)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create metric converter: %v", err)
 	}

exporter/signalfxexporter/exporter_test.go

@@ -194,7 +194,7 @@ func TestConsumeMetrics(t *testing.T) {
 			serverURL, err := url.Parse(server.URL)
 			assert.NoError(t, err)
 
-			c, err := translation.NewMetricsConverter(zap.NewNop(), nil, nil)
+			c, err := translation.NewMetricsConverter(zap.NewNop(), nil, nil, nil)
 			require.NoError(t, err)
 			require.NotNil(t, c)
 			dpClient := &sfxDPClient{
@@ -974,7 +974,7 @@ func BenchmarkExporterConsumeData(b *testing.B) {
 	serverURL, err := url.Parse(server.URL)
 	assert.NoError(b, err)
 
-	c, err := translation.NewMetricsConverter(zap.NewNop(), nil, nil)
+	c, err := translation.NewMetricsConverter(zap.NewNop(), nil, nil, nil)
 	require.NoError(b, err)
 	require.NotNil(b, c)
 	dpClient := &sfxDPClient{

exporter/signalfxexporter/factory_test.go

@@ -245,7 +245,7 @@ func TestDefaultTranslationRules(t *testing.T) {
 	require.NoError(t, err)
 	data := testMetricsData()
 
-	c, err := translation.NewMetricsConverter(zap.NewNop(), tr, nil)
+	c, err := translation.NewMetricsConverter(zap.NewNop(), tr, nil, nil)
 	require.NoError(t, err)
 	translated := c.MetricDataToSignalFxV2(data)
 	require.NotNil(t, translated)

exporter/signalfxexporter/testdata/config.yaml

@@ -44,6 +44,10 @@ exporters:
       - metric_name: cpu.utilization
         dimensions:
           container_name: /^[A-Z][A-Z]$/
+    include_metrics:
+      - metric_name: metric1
+      - metric_names: [metric2, metric3]
+
 
 
 

exporter/signalfxexporter/translation/converter.go

@@ -58,8 +58,12 @@ type MetricsConverter struct {
 // NewMetricsConverter creates a MetricsConverter from the passed in logger and
 // MetricTranslator. Pass in a nil MetricTranslator to not use translation
 // rules.
-func NewMetricsConverter(logger *zap.Logger, t *MetricTranslator, excludes []dpfilters.MetricFilter) (*MetricsConverter, error) {
-	fs, err := dpfilters.NewFilterSet(excludes)
+func NewMetricsConverter(
+	logger *zap.Logger,
+	t *MetricTranslator,
+	excludes []dpfilters.MetricFilter,
+	includes []dpfilters.MetricFilter) (*MetricsConverter, error) {
+	fs, err := dpfilters.NewFilterSet(excludes, includes)
 	if err != nil {
 		return nil, err
 	}

exporter/signalfxexporter/translation/converter_test.go

@@ -101,6 +101,7 @@ func Test_MetricDataToSignalFxV2(t *testing.T) {
 		name              string
 		metricsDataFn     func() pdata.ResourceMetrics
 		excludeMetrics    []dpfilters.MetricFilter
+		includeMetrics    []dpfilters.MetricFilter
 		wantSfxDataPoints []*sfxpb.DataPoint
 	}{
 		{
@@ -559,10 +560,80 @@ func Test_MetricDataToSignalFxV2(t *testing.T) {
 				doubleSFxDataPoint("cumulative_double_with_dims", tsMSecs, &sfxMetricTypeCumulativeCounter, differentLabelMap, doubleVal),
 			},
 		},
+		{
+			// To validate that filters in include serves as override to the ones in exclude list.
+			name: "with_include_and_exclude_metrics_filter",
+			metricsDataFn: func() pdata.ResourceMetrics {
+				out := pdata.NewResourceMetrics()
+				out.InstrumentationLibraryMetrics().Resize(1)
+				ilm := out.InstrumentationLibraryMetrics().At(0)
+				ilm.Metrics().Resize(4)
+
+				{
+					m := ilm.Metrics().At(0)
+					m.SetName("gauge_double_with_dims")
+					m.SetDataType(pdata.MetricDataTypeDoubleGauge)
+					m.DoubleGauge().DataPoints().Append(doublePtWithLabels)
+				}
+				{
+					m := ilm.Metrics().At(1)
+					m.SetName("gauge_int_with_dims")
+					m.SetDataType(pdata.MetricDataTypeIntGauge)
+					m.IntGauge().DataPoints().Append(int64PtWithLabels)
+				}
+				{
+					m := ilm.Metrics().At(2)
+					m.SetName("cumulative_double_with_dims")
+					m.SetDataType(pdata.MetricDataTypeDoubleSum)
+					m.DoubleSum().SetIsMonotonic(true)
+					m.DoubleSum().DataPoints().Append(doublePtWithLabels)
+					m.DoubleSum().DataPoints().Append(doublePtWithDifferentLabels)
+				}
+				{
+					m := ilm.Metrics().At(3)
+					m.SetName("cumulative_int_with_dims")
+					m.SetDataType(pdata.MetricDataTypeIntSum)
+					m.IntSum().SetIsMonotonic(true)
+					m.IntSum().DataPoints().Append(int64PtWithLabels)
+				}
+
+				return out
+			},
+			excludeMetrics: []dpfilters.MetricFilter{
+				{
+					MetricNames: []string{"gauge_double_with_dims"},
+				},
+				{
+					MetricName: "cumulative_int_with_dims",
+				},
+				{
+					MetricName: "gauge_int_with_dims",
+					Dimensions: map[string]interface{}{
+						"k0": []interface{}{"v1"},
+					},
+				},
+				{
+					MetricName: "cumulative_double_with_dims",
+					Dimensions: map[string]interface{}{
+						"k0": []interface{}{"v0"},
+					},
+				},
+			},
+			includeMetrics: []dpfilters.MetricFilter{
+				{
+					MetricName: "cumulative_int_with_dims",
+				},
+			},
+			wantSfxDataPoints: []*sfxpb.DataPoint{
+				int64SFxDataPoint("gauge_int_with_dims", tsMSecs, &sfxMetricTypeGauge, labelMap, int64Val),
+				doubleSFxDataPoint("cumulative_double_with_dims", tsMSecs, &sfxMetricTypeCumulativeCounter, differentLabelMap, doubleVal),
+				int64SFxDataPoint("cumulative_int_with_dims", tsMSecs, &sfxMetricTypeCumulativeCounter, labelMap, int64Val),
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c, err := NewMetricsConverter(logger, nil, tt.excludeMetrics)
+			c, err := NewMetricsConverter(logger, nil, tt.excludeMetrics, tt.includeMetrics)
 			require.NoError(t, err)
 			gotSfxDataPoints := c.MetricDataToSignalFxV2(tt.metricsDataFn())
 			// Sort SFx dimensions since they are built from maps and the order
@@ -611,7 +682,7 @@ func TestMetricDataToSignalFxV2WithTranslation(t *testing.T) {
 			},
 		},
 	}
-	c, err := NewMetricsConverter(zap.NewNop(), translator, nil)
+	c, err := NewMetricsConverter(zap.NewNop(), translator, nil, nil)
 	require.NoError(t, err)
 	assert.EqualValues(t, expected, c.MetricDataToSignalFxV2(wrapMetric(md)))
 }
@@ -780,7 +851,7 @@ func TestNewMetricsConverter(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := NewMetricsConverter(zap.NewNop(), nil, tt.excludes)
+			got, err := NewMetricsConverter(zap.NewNop(), nil, tt.excludes, nil)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("NewMetricsConverter() error = %v, wantErr %v", err, tt.wantErr)
 				return

exporter/signalfxexporter/translation/dpfilters/filterset.go

@@ -19,23 +19,48 @@ import sfxpb "github.com/signalfx/com_signalfx_metrics_protobuf/model"
 // FilterSet is a collection of datapont filters, any one of which must match
 // for a datapoint to be matched.
 type FilterSet struct {
-	ExcludeFilters []*dataPointFilter
+	excludeFilters []*dataPointFilter
+	includeFilters []*dataPointFilter
 }
 
 // Matches sends a datapoint through each of the filters in the set and returns
 // true if at least one of them matches the datapoint.
 func (fs *FilterSet) Matches(dp *sfxpb.DataPoint) bool {
-	for _, ex := range fs.ExcludeFilters {
+	for _, ex := range fs.excludeFilters {
 		if ex.Matches(dp) {
+			// If we match an exclusionary filter, run through each inclusion
+			// filter and see if anything includes the metrics.
+			for _, in := range fs.includeFilters {
+				if in.Matches(dp) {
+					return false
+				}
+			}
 			return true
 		}
 	}
 	return false
 }
 
-func NewFilterSet(excludes []MetricFilter) (*FilterSet, error) {
-	var excludeSet []*dataPointFilter
-	for _, f := range excludes {
+func NewFilterSet(excludes []MetricFilter, includes []MetricFilter) (*FilterSet, error) {
+	excludeSet, err := getDataPointFilters(excludes)
+	if err != nil {
+		return nil, err
+	}
+
+	includeSet, err := getDataPointFilters(includes)
+	if err != nil {
+		return nil, err
+	}
+
+	return &FilterSet{
+		excludeFilters: excludeSet,
+		includeFilters: includeSet,
+	}, nil
+}
+
+func getDataPointFilters(metricFilters []MetricFilter) ([]*dataPointFilter, error) {
+	var out []*dataPointFilter
+	for _, f := range metricFilters {
 		dimSet, err := f.normalize()
 		if err != nil {
 			return nil, err
@@ -46,9 +71,7 @@ func NewFilterSet(excludes []MetricFilter) (*FilterSet, error) {
 			return nil, err
 		}
 
-		excludeSet = append(excludeSet, dpf)
+		out = append(out, dpf)
 	}
-	return &FilterSet{
-		ExcludeFilters: excludeSet,
-	}, nil
+	return out, nil
 }