Add Prometheus Receiver tests to run OpenMetrics test cases (open-tel…

…emetry#6409)

* Add Prometheus receiver test to run OpenMetrics test cases

* fix: resolved gosec lint issue

* fix error handling in getTestCase

* remove http requests in tests by copying openmetrics data into testdata

* only skip failing openmetrics tests using a deny-list

receiver/prometheusreceiver/metrics_receiver_helper_test.go

@@ -44,14 +44,15 @@ type mockPrometheusResponse struct {
 }
 
 type mockPrometheus struct {
-	mu          sync.Mutex // mu protects the fields below.
-	endpoints   map[string][]mockPrometheusResponse
-	accessIndex map[string]*int32
-	wg          *sync.WaitGroup
-	srv         *httptest.Server
+	mu             sync.Mutex // mu protects the fields below.
+	endpoints      map[string][]mockPrometheusResponse
+	accessIndex    map[string]*int32
+	wg             *sync.WaitGroup
+	srv            *httptest.Server
+	useOpenMetrics bool
 }
 
-func newMockPrometheus(endpoints map[string][]mockPrometheusResponse) *mockPrometheus {
+func newMockPrometheus(endpoints map[string][]mockPrometheusResponse, openMetricsContentType bool) *mockPrometheus {
 	accessIndex := make(map[string]*int32)
 	wg := &sync.WaitGroup{}
 	wg.Add(len(endpoints))
@@ -60,9 +61,10 @@ func newMockPrometheus(endpoints map[string][]mockPrometheusResponse) *mockProme
 		accessIndex[k] = &v
 	}
 	mp := &mockPrometheus{
-		wg:          wg,
-		accessIndex: accessIndex,
-		endpoints:   endpoints,
+		wg:             wg,
+		accessIndex:    accessIndex,
+		endpoints:      endpoints,
+		useOpenMetrics: openMetricsContentType,
 	}
 	srv := httptest.NewServer(mp)
 	mp.srv = srv
@@ -73,6 +75,9 @@ func (mp *mockPrometheus) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	mp.mu.Lock()
 	defer mp.mu.Unlock()
 
+	if mp.useOpenMetrics {
+		rw.Header().Set("Content-Type", "application/openmetrics-text")
+	}
 	iptr, ok := mp.accessIndex[req.URL.Path]
 	if !ok {
 		rw.WriteHeader(404)
@@ -113,7 +118,7 @@ type testData struct {
 
 // setupMockPrometheus to create a mocked prometheus based on targets, returning the server and a prometheus exporting
 // config
-func setupMockPrometheus(tds ...*testData) (*mockPrometheus, *promcfg.Config, error) {
+func setupMockPrometheus(openMetricsContentType bool, tds ...*testData) (*mockPrometheus, *promcfg.Config, error) {
 	jobs := make([]map[string]interface{}, 0, len(tds))
 	endpoints := make(map[string][]mockPrometheusResponse)
 	metricPaths := make([]string, 0)
@@ -122,7 +127,7 @@ func setupMockPrometheus(tds ...*testData) (*mockPrometheus, *promcfg.Config, er
 		endpoints[metricPath] = t.pages
 		metricPaths = append(metricPaths, metricPath)
 	}
-	mp := newMockPrometheus(endpoints)
+	mp := newMockPrometheus(endpoints, openMetricsContentType)
 	u, _ := url.Parse(mp.srv.URL)
 	host, port, _ := net.SplitHostPort(u.Host)
 	for i := 0; i < len(tds); i++ {
@@ -457,9 +462,9 @@ func compareSummary(count uint64, sum float64, quantiles [][]float64) summaryPoi
 	}
 }
 
-func testComponent(t *testing.T, targets []*testData, useStartTimeMetric bool, startTimeMetricRegex string) {
+func testComponent(t *testing.T, targets []*testData, useStartTimeMetric bool, startTimeMetricRegex string, useOpenMetrics bool) {
 	// 1. setup mock server
-	mp, cfg, err := setupMockPrometheus(targets...)
+	mp, cfg, err := setupMockPrometheus(useOpenMetrics, targets...)
 	require.Nilf(t, err, "Failed to create Prometheus config: %v", err)
 	defer mp.Close()
 
@@ -490,14 +495,17 @@ func testComponent(t *testing.T, targets []*testData, useStartTimeMetric bool, s
 	// loop to validate outputs for each targets
 	for _, target := range targets {
 		t.Run(target.name, func(t *testing.T) {
-			validScrapes := getValidScrapes(t, pResults[target.name])
+			validScrapes := pResults[target.name]
+			if !useOpenMetrics {
+				validScrapes = getValidScrapes(t, pResults[target.name])
+			}
 			target.validateFunc(t, target, validScrapes)
 		})
 	}
 }
 
 // starts prometheus receiver with custom config, retrieves metrics from MetricsSink
-func testComponentCustomConfig(t *testing.T, targets []*testData, mp *mockPrometheus, cfg *promcfg.Config) {
+func testComponentCustomConfig(t *testing.T, targets []*testData, mp *mockPrometheus, cfg *promcfg.Config, useOpenMetrics bool) {
 	ctx := context.Background()
 	defer mp.Close()
 
@@ -523,7 +531,10 @@ func testComponentCustomConfig(t *testing.T, targets []*testData, mp *mockPromet
 	// loop to validate outputs for each targets
 	for _, target := range targets {
 		t.Run(target.name, func(t *testing.T) {
-			validScrapes := getValidScrapes(t, pResults[target.name])
+			validScrapes := pResults[target.name]
+			if !useOpenMetrics {
+				validScrapes = getValidScrapes(t, pResults[target.name])
+			}
 			target.validateFunc(t, target, validScrapes)
 		})
 	}

receiver/prometheusreceiver/metrics_receiver_labels_test.go

@@ -38,11 +38,11 @@ func TestExternalLabels(t *testing.T) {
 		},
 	}
 
-	mp, cfg, err := setupMockPrometheus(targets...)
+	mp, cfg, err := setupMockPrometheus(false, targets...)
 	cfg.GlobalConfig.ExternalLabels = labels.FromStrings("key", "value")
 	require.Nilf(t, err, "Failed to create Prometheus config: %v", err)
 
-	testComponentCustomConfig(t, targets, mp, cfg)
+	testComponentCustomConfig(t, targets, mp, cfg, false)
 }
 
 func verifyExternalLabels(t *testing.T, td *testData, rms []*pdata.ResourceMetrics) {
@@ -130,15 +130,15 @@ func TestLabelLimitConfig(t *testing.T) {
 		},
 	}
 
-	mp, cfg, err := setupMockPrometheus(targets...)
+	mp, cfg, err := setupMockPrometheus(false, targets...)
 	require.Nilf(t, err, "Failed to create Prometheus config: %v", err)
 
 	// set label limit in scrape_config
 	for _, scrapeCfg := range cfg.ScrapeConfigs {
 		scrapeCfg.LabelLimit = 5
 	}
 
-	testComponentCustomConfig(t, targets, mp, cfg)
+	testComponentCustomConfig(t, targets, mp, cfg, false)
 }
 
 const targetLabelLimits1 = `
@@ -256,15 +256,15 @@ func TestLabelNameLimitConfig(t *testing.T) {
 		},
 	}
 
-	mp, cfg, err := setupMockPrometheus(targets...)
+	mp, cfg, err := setupMockPrometheus(false, targets...)
 	require.Nilf(t, err, "Failed to create Prometheus config: %v", err)
 
 	// set label name limit in scrape_config
 	for _, scrapeCfg := range cfg.ScrapeConfigs {
 		scrapeCfg.LabelNameLengthLimit = 20
 	}
 
-	testComponentCustomConfig(t, targets, mp, cfg)
+	testComponentCustomConfig(t, targets, mp, cfg, false)
 }
 
 const targetLabelValueLimit = `

receiver/prometheusreceiver/metrics_receiver_open_metrics_test.go

@@ -0,0 +1,147 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheusreceiver
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/model/pdata"
+)
+
+const testDir = "./testdata/openmetrics/"
+
+var skippedTests = map[string]struct{}{
+	"bad_clashing_names_0": {}, "bad_clashing_names_1": {}, "bad_clashing_names_2": {},
+	"bad_counter_values_0": {}, "bad_counter_values_1": {}, "bad_counter_values_2": {},
+	"bad_counter_values_3": {}, "bad_counter_values_5": {}, "bad_counter_values_6": {},
+	"bad_counter_values_10": {}, "bad_counter_values_11": {}, "bad_counter_values_12": {},
+	"bad_counter_values_13": {}, "bad_counter_values_14": {}, "bad_counter_values_15": {},
+	"bad_counter_values_16": {}, "bad_counter_values_17": {}, "bad_counter_values_18": {},
+	"bad_counter_values_19": {}, "bad_exemplars_on_unallowed_samples_2": {}, "bad_exemplar_timestamp_0": {},
+	"bad_exemplar_timestamp_1": {}, "bad_exemplar_timestamp_2": {}, "bad_grouping_or_ordering_0": {},
+	"bad_grouping_or_ordering_2": {}, "bad_grouping_or_ordering_3": {}, "bad_grouping_or_ordering_4": {},
+	"bad_grouping_or_ordering_5": {}, "bad_grouping_or_ordering_6": {}, "bad_grouping_or_ordering_7": {},
+	"bad_grouping_or_ordering_8": {}, "bad_grouping_or_ordering_9": {}, "bad_grouping_or_ordering_10": {},
+	"bad_histograms_0": {}, "bad_histograms_1": {}, "bad_histograms_2": {}, "bad_histograms_3": {},
+	"bad_histograms_6": {}, "bad_histograms_7": {}, "bad_histograms_8": {},
+	"bad_info_and_stateset_values_0": {}, "bad_info_and_stateset_values_1": {}, "bad_metadata_in_wrong_place_0": {},
+	"bad_metadata_in_wrong_place_1": {}, "bad_metadata_in_wrong_place_2": {},
+	"bad_missing_or_invalid_labels_for_a_type_1": {}, "bad_missing_or_invalid_labels_for_a_type_3": {},
+	"bad_missing_or_invalid_labels_for_a_type_4": {}, "bad_missing_or_invalid_labels_for_a_type_6": {},
+	"bad_missing_or_invalid_labels_for_a_type_7": {}, "bad_repeated_metadata_0": {},
+	"bad_repeated_metadata_1": {}, "bad_repeated_metadata_3": {}, "bad_stateset_info_values_0": {},
+	"bad_stateset_info_values_1": {}, "bad_stateset_info_values_2": {}, "bad_stateset_info_values_3": {},
+	"bad_timestamp_4": {}, "bad_timestamp_5": {}, "bad_timestamp_7": {}, "bad_unit_6": {}, "bad_unit_7": {},
+}
+
+func verifyPositiveTarget(t *testing.T, _ *testData, mds []*pdata.ResourceMetrics) {
+	require.Greater(t, len(mds), 0, "At least one resource metric should be present")
+	metrics := getMetrics(mds[0])
+	assertUp(t, 1, metrics)
+}
+
+// Test open metrics positive test cases
+func TestOpenMetricsPositive(t *testing.T) {
+	targetsMap := getOpenMetricsTestData(false)
+	targets := make([]*testData, 0)
+	for k, v := range targetsMap {
+		testData := &testData{
+			name: k,
+			pages: []mockPrometheusResponse{
+				{code: 200, data: v},
+			},
+			validateFunc: verifyPositiveTarget,
+		}
+		targets = append(targets, testData)
+	}
+
+	testComponent(t, targets, false, "", true)
+}
+
+func verifyNegativeTarget(t *testing.T, td *testData, mds []*pdata.ResourceMetrics) {
+	// failing negative tests are skipped since prometheus scrape package is currently not fully
+	// compatible with OpenMetrics tests and successfully scrapes some invalid metrics
+	// see: https://github.com/prometheus/prometheus/issues/9699
+	if _, ok := skippedTests[td.name]; ok {
+		t.Skip("skipping failing negative OpenMetrics parser tests")
+	}
+
+	require.Greater(t, len(mds), 0, "At least one resource metric should be present")
+	metrics := getMetrics(mds[0])
+	assertUp(t, 0, metrics)
+}
+
+// Test open metrics negative test cases
+func TestOpenMetricsNegative(t *testing.T) {
+	targetsMap := getOpenMetricsTestData(true)
+	targets := make([]*testData, 0)
+	for k, v := range targetsMap {
+		testData := &testData{
+			name: k,
+			pages: []mockPrometheusResponse{
+				{code: 200, data: v},
+			},
+			validateFunc: verifyNegativeTarget,
+		}
+		targets = append(targets, testData)
+	}
+
+	testComponent(t, targets, false, "", true)
+}
+
+//reads test data from testdata/openmetrics directory
+func getOpenMetricsTestData(negativeTestsOnly bool) map[string]string {
+	testDir, err := os.Open(testDir)
+	if err != nil {
+		log.Fatalf("failed opening openmetrics test directory")
+	}
+	defer testDir.Close()
+
+	//read all test file names in testdata/openmetrics
+	testList, _ := testDir.Readdirnames(0)
+
+	targetsData := make(map[string]string)
+	for _, testName := range testList {
+		//ignore hidden files
+		if strings.HasPrefix(testName, ".") {
+			continue
+		}
+		if negativeTestsOnly && !strings.Contains(testName, "bad") {
+			continue
+		} else if !negativeTestsOnly && strings.Contains(testName, "bad") {
+			continue
+		}
+		if testData, err := readTestCase(testName); err == nil {
+			targetsData[testName] = testData
+		}
+	}
+	return targetsData
+}
+
+func readTestCase(testName string) (string, error) {
+	filePath := fmt.Sprintf("%s/%s/metrics", testDir, testName)
+	content, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		log.Printf("failed opening file: %s", filePath)
+		return "", err
+	}
+	return string(content), nil
+}

receiver/prometheusreceiver/metrics_receiver_test.go

@@ -734,7 +734,7 @@ func TestCoreMetricsEndToEnd(t *testing.T) {
 			validateFunc: verifyTarget3,
 		},
 	}
-	testComponent(t, targets, false, "")
+	testComponent(t, targets, false, "", false)
 }
 
 var startTimeMetricPage = `
@@ -820,7 +820,7 @@ func TestStartTimeMetric(t *testing.T) {
 			validateFunc: verifyStartTimeMetricPage,
 		},
 	}
-	testComponent(t, targets, true, "")
+	testComponent(t, targets, true, "", false)
 }
 
 var startTimeMetricRegexPage = `
@@ -869,5 +869,5 @@ func TestStartTimeMetricRegex(t *testing.T) {
 			validateFunc: verifyStartTimeMetricPage,
 		},
 	}
-	testComponent(t, targets, true, "^(.+_)*process_start_time_seconds$")
+	testComponent(t, targets, true, "^(.+_)*process_start_time_seconds$", false)
 }

receiver/prometheusreceiver/testdata/openmetrics/bad_blank_line/metrics

@@ -0,0 +1,3 @@
+a 1
+
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_clashing_names_0/metrics

@@ -0,0 +1,3 @@
+# TYPE a counter
+# TYPE a counter
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_clashing_names_1/metrics

@@ -0,0 +1,3 @@
+# TYPE a info
+# TYPE a counter
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_clashing_names_2/metrics

@@ -0,0 +1,3 @@
+# TYPE a_created gauge
+# TYPE a counter
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_0/metrics

@@ -0,0 +1,3 @@
+# TYPE a counter
+a_total NaN
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_1/metrics

@@ -0,0 +1,3 @@
+# TYPE a counter
+a_total -1
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_10/metrics

@@ -0,0 +1,3 @@
+# TYPE a gaugehistogram
+a_bucket{le=" Inf"} NaN
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_11/metrics

@@ -0,0 +1,4 @@
+# TYPE a gaugehistogram
+a_bucket{le=" Inf"} -1
+a_gcount -1
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_12/metrics

@@ -0,0 +1,3 @@
+# TYPE a gaugehistogram
+a_bucket{le=" Inf"} -1
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_13/metrics

@@ -0,0 +1,4 @@
+# TYPE a gaugehistogram
+a_bucket{le=" Inf"} 1
+a_gsum -1
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_14/metrics

@@ -0,0 +1,4 @@
+# TYPE a gaugehistogram
+a_bucket{le=" Inf"} 1
+a_gsum NaN
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_15/metrics

@@ -0,0 +1,3 @@
+# TYPE a summary
+a_sum NaN
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_16/metrics

@@ -0,0 +1,3 @@
+# TYPE a summary
+a_count NaN
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_17/metrics

@@ -0,0 +1,3 @@
+# TYPE a summary
+a_sum -1
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_18/metrics

@@ -0,0 +1,3 @@
+# TYPE a summary
+a_count -1
+# EOF

receiver/prometheusreceiver/testdata/openmetrics/bad_counter_values_19/metrics

@@ -0,0 +1,3 @@
+# TYPE a summary
+a{quantile="0.5"} -1
+# EOF