From f3301a376ca252760d060bdbd93e34b89235d9a0 Mon Sep 17 00:00:00 2001 From: Rajeshwari Muthupandian Date: Tue, 9 May 2023 13:23:36 -0700 Subject: [PATCH 1/6] issue-21715 fetching metric vlaues with composite(all) dimension keys --- ...monitor_receiver-composite-dimensions.yaml | 16 +++++ receiver/azuremonitorreceiver/scraper.go | 59 +++++++++++++------ 2 files changed, 57 insertions(+), 18 deletions(-) create mode 100755 .chloggen/azure_monitor_receiver-composite-dimensions.yaml diff --git a/.chloggen/azure_monitor_receiver-composite-dimensions.yaml b/.chloggen/azure_monitor_receiver-composite-dimensions.yaml new file mode 100755 index 0000000000000..04812d9714eb4 --- /dev/null +++ b/.chloggen/azure_monitor_receiver-composite-dimensions.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: receiver/azuremonitorreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Retrieve metric values with all dimension keys in filter + +# One or more tracking issues related to the change +issues: [21715] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/receiver/azuremonitorreceiver/scraper.go b/receiver/azuremonitorreceiver/scraper.go index ae7f868500325..c899d1a827492 100644 --- a/receiver/azuremonitorreceiver/scraper.go +++ b/receiver/azuremonitorreceiver/scraper.go @@ -15,7 +15,9 @@ package azuremonitorreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azuremonitorreceiver" import ( + "bytes" "context" + "crypto/sha256" "fmt" "strings" "sync" @@ -64,18 +66,26 @@ const ( ) type azureResource struct { - metricsByCompositeKey map[metricsCompositeKey]*azureResourceMetrics + metricsByCompositeKey map[string]*azureResourceMetrics metricsDefinitionsUpdated time.Time tags map[string]*string location string } type metricsCompositeKey struct { - dimension string - timeGrain string + dimensions []string + timeGrain string +} + +func (s metricsCompositeKey) Hash() string { + h := sha256.New() + h.Write([]byte(s.timeGrain)) + h.Write([]byte(strings.Join(s.dimensions, ","))) + return string(h.Sum(nil)) } type azureResourceMetrics struct { + compositeKey metricsCompositeKey metrics []string metricsValuesUpdated time.Time } @@ -249,7 +259,7 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour return } - s.resources[resourceID].metricsByCompositeKey = map[metricsCompositeKey]*azureResourceMetrics{} + s.resources[resourceID].metricsByCompositeKey = map[string]*azureResourceMetrics{} pager := s.clientMetricsDefinitions.NewListPager(resourceID, nil) for pager.More() { @@ -266,10 +276,14 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour compositeKey := metricsCompositeKey{timeGrain: timeGrain} if len(v.Dimensions) > 0 { + var dimensionsSlice []string for _, dimension := range v.Dimensions { - compositeKey.dimension = *dimension.Value - s.storeMetricsDefinition(resourceID, name, compositeKey) + if len(strings.TrimSpace(*dimension.Value)) > 0 { + dimensionsSlice = append(dimensionsSlice, *dimension.Value) + } } + dimensionsCompositeKey := metricsCompositeKey{timeGrain: timeGrain, dimensions: dimensionsSlice} + s.storeMetricsDefinition(resourceID, name, dimensionsCompositeKey) } else { s.storeMetricsDefinition(resourceID, name, compositeKey) } @@ -279,21 +293,22 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour } func (s *azureScraper) storeMetricsDefinition(resourceID, name string, compositeKey metricsCompositeKey) { - if _, ok := s.resources[resourceID].metricsByCompositeKey[compositeKey]; ok { - s.resources[resourceID].metricsByCompositeKey[compositeKey].metrics = append( - s.resources[resourceID].metricsByCompositeKey[compositeKey].metrics, name, + hash := compositeKey.Hash() + if _, ok := s.resources[resourceID].metricsByCompositeKey[hash]; ok { + s.resources[resourceID].metricsByCompositeKey[hash].metrics = append( + s.resources[resourceID].metricsByCompositeKey[hash].metrics, name, ) } else { - s.resources[resourceID].metricsByCompositeKey[compositeKey] = &azureResourceMetrics{metrics: []string{name}} + s.resources[resourceID].metricsByCompositeKey[hash] = &azureResourceMetrics{metrics: []string{name}, compositeKey: compositeKey} } } func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID string) { res := *s.resources[resourceID] - for compositeKey, metricsByGrain := range res.metricsByCompositeKey { + for _, metricsByGrain := range res.metricsByCompositeKey { - if time.Since(metricsByGrain.metricsValuesUpdated).Seconds() < float64(timeGrains[compositeKey.timeGrain]) { + if time.Since(metricsByGrain.metricsValuesUpdated).Seconds() < float64(timeGrains[metricsByGrain.compositeKey.timeGrain]) { continue } metricsByGrain.metricsValuesUpdated = time.Now() @@ -309,8 +324,8 @@ func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID opts := getResourceMetricsValuesRequestOptions( metricsByGrain.metrics, - compositeKey.dimension, - compositeKey.timeGrain, + metricsByGrain.compositeKey.dimensions, + metricsByGrain.compositeKey.timeGrain, start, end, ) @@ -356,7 +371,7 @@ func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID func getResourceMetricsValuesRequestOptions( metrics []string, - dimension string, + dimensions []string, timeGrain string, start int, end int, @@ -369,9 +384,17 @@ func getResourceMetricsValuesRequestOptions( Aggregation: to.Ptr(strings.Join(aggregations, ",")), } - if len(dimension) > 0 { - dimensionFilter := fmt.Sprintf("%s eq '*'", dimension) - filter.Filter = &dimensionFilter + if len(dimensions) > 0 { + var dimensionsFilter bytes.Buffer + for i, dimension := range dimensions { + dimensionsFilter.WriteString(dimension) + dimensionsFilter.WriteString(" eq '*' ") + if i != len(dimensions) { + dimensionsFilter.WriteString(" and ") + } + } + dimensionFilterString := dimensionsFilter.String() + filter.Filter = &dimensionFilterString } return filter From 065f78b68f37e631392b0c16903fae993de99557 Mon Sep 17 00:00:00 2001 From: Rajeshwari Muthupandian Date: Wed, 10 May 2023 14:34:00 -0700 Subject: [PATCH 2/6] issue #21715- adding testdata with multiple dimensions --- receiver/azuremonitorreceiver/scraper.go | 2 +- receiver/azuremonitorreceiver/scraper_test.go | 38 +++++++++++++------ .../expected_metrics/metrics_golden.yaml | 37 ++++++++++++------ .../expected_metrics/metrics_tags_golden.yaml | 37 ++++++++++++------ 4 files changed, 79 insertions(+), 35 deletions(-) diff --git a/receiver/azuremonitorreceiver/scraper.go b/receiver/azuremonitorreceiver/scraper.go index c899d1a827492..caa5371490bd5 100644 --- a/receiver/azuremonitorreceiver/scraper.go +++ b/receiver/azuremonitorreceiver/scraper.go @@ -389,7 +389,7 @@ func getResourceMetricsValuesRequestOptions( for i, dimension := range dimensions { dimensionsFilter.WriteString(dimension) dimensionsFilter.WriteString(" eq '*' ") - if i != len(dimensions) { + if i < len(dimensions)-1 { dimensionsFilter.WriteString(" and ") } } diff --git a/receiver/azuremonitorreceiver/scraper_test.go b/receiver/azuremonitorreceiver/scraper_test.go index b3baff4481492..ec8a290fae3ad 100644 --- a/receiver/azuremonitorreceiver/scraper_test.go +++ b/receiver/azuremonitorreceiver/scraper_test.go @@ -231,7 +231,6 @@ func TestAzureScraperScrape(t *testing.T) { expectedFile := filepath.Join("testdata", "expected_metrics", tt.name+".yaml") expectedMetrics, err := golden.ReadMetrics(expectedFile) require.NoError(t, err) - require.NoError(t, pmetrictest.CompareMetrics( expectedMetrics, metrics, @@ -279,8 +278,8 @@ func getResourcesMockData(tags bool) []armresources.ClientListResponse { } func getMetricsDefinitionsMockData() (map[string]int, map[string][]armmonitor.MetricDefinitionsClientListResponse) { - name1, name2, name3, name4, name5, name6, name7, timeGrain1, timeGrain2, dimension := "metric1", - "metric2", "metric3", "metric4", "metric5", "metric6", "metric7", "PT1M", "PT1H", "dimension" + name1, name2, name3, name4, name5, name6, name7, timeGrain1, timeGrain2, dimension1, dimension2 := "metric1", + "metric2", "metric3", "metric4", "metric5", "metric6", "metric7", "PT1M", "PT1H", "dimension1", "dimension2" counters := map[string]int{ "resourceId1": 0, @@ -352,7 +351,10 @@ func getMetricsDefinitionsMockData() (map[string]int, map[string][]armmonitor.Me }, Dimensions: []*armmonitor.LocalizableString{ { - Value: &dimension, + Value: &dimension1, + }, + { + Value: &dimension2, }, }, }, @@ -367,7 +369,7 @@ func getMetricsDefinitionsMockData() (map[string]int, map[string][]armmonitor.Me }, Dimensions: []*armmonitor.LocalizableString{ { - Value: &dimension, + Value: &dimension1, }, }, }, @@ -390,7 +392,7 @@ func getMetricsDefinitionsMockData() (map[string]int, map[string][]armmonitor.Me }, Dimensions: []*armmonitor.LocalizableString{ { - Value: &dimension, + Value: &dimension1, }, }, }, @@ -403,8 +405,8 @@ func getMetricsDefinitionsMockData() (map[string]int, map[string][]armmonitor.Me } func getMetricsValuesMockData() map[string]map[string]armmonitor.MetricsClientListResponse { - name1, name2, name3, name4, name5, name6, name7, dimension, dimensionValue := "metric1", "metric2", - "metric3", "metric4", "metric5", "metric6", "metric7", "dimension", "dimension value" + name1, name2, name3, name4, name5, name6, name7, dimension1, dimension2, dimensionValue := "metric1", "metric2", + "metric3", "metric4", "metric5", "metric6", "metric7", "dimension1", "dimension2", "dimension value" var unit1 armmonitor.MetricUnit = "unit1" var value1 float64 = 1 @@ -506,7 +508,7 @@ func getMetricsValuesMockData() map[string]map[string]armmonitor.MetricsClientLi }, }, }, - strings.Join([]string{name5, name6}, ","): { + name5: { Response: armmonitor.Response{ Value: []*armmonitor.Metric{ { @@ -528,7 +530,13 @@ func getMetricsValuesMockData() map[string]map[string]armmonitor.MetricsClientLi Metadatavalues: []*armmonitor.MetadataValue{ { Name: &armmonitor.LocalizableString{ - Value: &dimension, + Value: &dimension1, + }, + Value: &dimensionValue, + }, + { + Name: &armmonitor.LocalizableString{ + Value: &dimension2, }, Value: &dimensionValue, }, @@ -536,6 +544,12 @@ func getMetricsValuesMockData() map[string]map[string]armmonitor.MetricsClientLi }, }, }, + }, + }, + }, + name6: { + Response: armmonitor.Response{ + Value: []*armmonitor.Metric{ { Name: &armmonitor.LocalizableString{ Value: &name6, @@ -555,7 +569,7 @@ func getMetricsValuesMockData() map[string]map[string]armmonitor.MetricsClientLi Metadatavalues: []*armmonitor.MetadataValue{ { Name: &armmonitor.LocalizableString{ - Value: &dimension, + Value: &dimension1, }, Value: &dimensionValue, }, @@ -586,7 +600,7 @@ func getMetricsValuesMockData() map[string]map[string]armmonitor.MetricsClientLi Metadatavalues: []*armmonitor.MetadataValue{ { Name: &armmonitor.LocalizableString{ - Value: &dimension, + Value: &dimension1, }, Value: &dimensionValue, }, diff --git a/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_golden.yaml b/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_golden.yaml index 3bbeb956d812d..9e5b3b66b3198 100644 --- a/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_golden.yaml +++ b/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_golden.yaml @@ -16,7 +16,10 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 + value: + stringValue: dimension value + - key: metadata_dimension2 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -30,7 +33,10 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 + value: + stringValue: dimension value + - key: metadata_dimension2 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -44,7 +50,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -167,7 +173,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -181,7 +187,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -220,7 +226,10 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 + value: + stringValue: dimension value + - key: metadata_dimension2 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -234,7 +243,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -284,7 +293,10 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 + value: + stringValue: dimension value + - key: metadata_dimension2 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -326,7 +338,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId3 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -340,7 +352,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -393,7 +405,10 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 + value: + stringValue: dimension value + - key: metadata_dimension2 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" diff --git a/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_tags_golden.yaml b/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_tags_golden.yaml index 38ba130fd191c..c191b9baf63e3 100644 --- a/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_tags_golden.yaml +++ b/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_tags_golden.yaml @@ -16,7 +16,10 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 + value: + stringValue: dimension value + - key: metadata_dimension2 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -30,7 +33,10 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 + value: + stringValue: dimension value + - key: metadata_dimension2 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -44,7 +50,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -188,7 +194,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -202,7 +208,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -244,7 +250,10 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 + value: + stringValue: dimension value + - key: metadata_dimension2 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -258,7 +267,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -311,7 +320,10 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 + value: + stringValue: dimension value + - key: metadata_dimension2 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -359,7 +371,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId3 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -373,7 +385,7 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" @@ -432,7 +444,10 @@ resourceMetrics: - key: azuremonitor.resource_id value: stringValue: resourceId2 - - key: metadata_dimension + - key: metadata_dimension1 + value: + stringValue: dimension value + - key: metadata_dimension2 value: stringValue: dimension value startTimeUnixNano: "1681790471654481953" From 8e443b8b50a8b5bbf8ab762ac3755f9a036a6c5c Mon Sep 17 00:00:00 2001 From: Rajeshwari Muthupandian Date: Tue, 16 May 2023 16:15:06 -0700 Subject: [PATCH 3/6] issue #21715 - review comments - removing CompositeKey struct --- receiver/azuremonitorreceiver/scraper.go | 40 ++++++++++-------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/receiver/azuremonitorreceiver/scraper.go b/receiver/azuremonitorreceiver/scraper.go index caa5371490bd5..4bdf67447718c 100644 --- a/receiver/azuremonitorreceiver/scraper.go +++ b/receiver/azuremonitorreceiver/scraper.go @@ -72,20 +72,9 @@ type azureResource struct { location string } -type metricsCompositeKey struct { - dimensions []string - timeGrain string -} - -func (s metricsCompositeKey) Hash() string { - h := sha256.New() - h.Write([]byte(s.timeGrain)) - h.Write([]byte(strings.Join(s.dimensions, ","))) - return string(h.Sum(nil)) -} - type azureResourceMetrics struct { - compositeKey metricsCompositeKey + dimensions []string + timeGrain string metrics []string metricsValuesUpdated time.Time } @@ -273,7 +262,6 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour timeGrain := *v.MetricAvailabilities[0].TimeGrain name := *v.Name.Value - compositeKey := metricsCompositeKey{timeGrain: timeGrain} if len(v.Dimensions) > 0 { var dimensionsSlice []string @@ -282,24 +270,30 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour dimensionsSlice = append(dimensionsSlice, *dimension.Value) } } - dimensionsCompositeKey := metricsCompositeKey{timeGrain: timeGrain, dimensions: dimensionsSlice} - s.storeMetricsDefinition(resourceID, name, dimensionsCompositeKey) + s.storeMetricsDefinition(resourceID, name, timeGrain, dimensionsSlice) } else { - s.storeMetricsDefinition(resourceID, name, compositeKey) + s.storeMetricsDefinition(resourceID, name, timeGrain, []string{}) } } } s.resources[resourceID].metricsDefinitionsUpdated = time.Now() } -func (s *azureScraper) storeMetricsDefinition(resourceID, name string, compositeKey metricsCompositeKey) { - hash := compositeKey.Hash() +func (s *azureScraper) calculateHash(timeGrain string, dimensions []string) string { + h := sha256.New() + h.Write([]byte(timeGrain)) + h.Write([]byte(strings.Join(dimensions, ","))) + return string(h.Sum(nil)) +} + +func (s *azureScraper) storeMetricsDefinition(resourceID, name string, timeGrain string, dimensions []string) { + hash := s.calculateHash(timeGrain, dimensions) if _, ok := s.resources[resourceID].metricsByCompositeKey[hash]; ok { s.resources[resourceID].metricsByCompositeKey[hash].metrics = append( s.resources[resourceID].metricsByCompositeKey[hash].metrics, name, ) } else { - s.resources[resourceID].metricsByCompositeKey[hash] = &azureResourceMetrics{metrics: []string{name}, compositeKey: compositeKey} + s.resources[resourceID].metricsByCompositeKey[hash] = &azureResourceMetrics{metrics: []string{name}, timeGrain: timeGrain, dimensions: dimensions} } } @@ -308,7 +302,7 @@ func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID for _, metricsByGrain := range res.metricsByCompositeKey { - if time.Since(metricsByGrain.metricsValuesUpdated).Seconds() < float64(timeGrains[metricsByGrain.compositeKey.timeGrain]) { + if time.Since(metricsByGrain.metricsValuesUpdated).Seconds() < float64(timeGrains[metricsByGrain.timeGrain]) { continue } metricsByGrain.metricsValuesUpdated = time.Now() @@ -324,8 +318,8 @@ func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID opts := getResourceMetricsValuesRequestOptions( metricsByGrain.metrics, - metricsByGrain.compositeKey.dimensions, - metricsByGrain.compositeKey.timeGrain, + metricsByGrain.dimensions, + metricsByGrain.timeGrain, start, end, ) From 6a187bc88c6db751655e4f47bb662710e683c66c Mon Sep 17 00:00:00 2001 From: Rajeshwari Muthupandian Date: Thu, 18 May 2023 21:06:19 -0700 Subject: [PATCH 4/6] Revert "issue #21715 - review comments - removing CompositeKey struct" This reverts commit 8e443b8b50a8b5bbf8ab762ac3755f9a036a6c5c. --- receiver/azuremonitorreceiver/scraper.go | 40 ++++++++++++++---------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/receiver/azuremonitorreceiver/scraper.go b/receiver/azuremonitorreceiver/scraper.go index 4bdf67447718c..caa5371490bd5 100644 --- a/receiver/azuremonitorreceiver/scraper.go +++ b/receiver/azuremonitorreceiver/scraper.go @@ -72,9 +72,20 @@ type azureResource struct { location string } +type metricsCompositeKey struct { + dimensions []string + timeGrain string +} + +func (s metricsCompositeKey) Hash() string { + h := sha256.New() + h.Write([]byte(s.timeGrain)) + h.Write([]byte(strings.Join(s.dimensions, ","))) + return string(h.Sum(nil)) +} + type azureResourceMetrics struct { - dimensions []string - timeGrain string + compositeKey metricsCompositeKey metrics []string metricsValuesUpdated time.Time } @@ -262,6 +273,7 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour timeGrain := *v.MetricAvailabilities[0].TimeGrain name := *v.Name.Value + compositeKey := metricsCompositeKey{timeGrain: timeGrain} if len(v.Dimensions) > 0 { var dimensionsSlice []string @@ -270,30 +282,24 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour dimensionsSlice = append(dimensionsSlice, *dimension.Value) } } - s.storeMetricsDefinition(resourceID, name, timeGrain, dimensionsSlice) + dimensionsCompositeKey := metricsCompositeKey{timeGrain: timeGrain, dimensions: dimensionsSlice} + s.storeMetricsDefinition(resourceID, name, dimensionsCompositeKey) } else { - s.storeMetricsDefinition(resourceID, name, timeGrain, []string{}) + s.storeMetricsDefinition(resourceID, name, compositeKey) } } } s.resources[resourceID].metricsDefinitionsUpdated = time.Now() } -func (s *azureScraper) calculateHash(timeGrain string, dimensions []string) string { - h := sha256.New() - h.Write([]byte(timeGrain)) - h.Write([]byte(strings.Join(dimensions, ","))) - return string(h.Sum(nil)) -} - -func (s *azureScraper) storeMetricsDefinition(resourceID, name string, timeGrain string, dimensions []string) { - hash := s.calculateHash(timeGrain, dimensions) +func (s *azureScraper) storeMetricsDefinition(resourceID, name string, compositeKey metricsCompositeKey) { + hash := compositeKey.Hash() if _, ok := s.resources[resourceID].metricsByCompositeKey[hash]; ok { s.resources[resourceID].metricsByCompositeKey[hash].metrics = append( s.resources[resourceID].metricsByCompositeKey[hash].metrics, name, ) } else { - s.resources[resourceID].metricsByCompositeKey[hash] = &azureResourceMetrics{metrics: []string{name}, timeGrain: timeGrain, dimensions: dimensions} + s.resources[resourceID].metricsByCompositeKey[hash] = &azureResourceMetrics{metrics: []string{name}, compositeKey: compositeKey} } } @@ -302,7 +308,7 @@ func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID for _, metricsByGrain := range res.metricsByCompositeKey { - if time.Since(metricsByGrain.metricsValuesUpdated).Seconds() < float64(timeGrains[metricsByGrain.timeGrain]) { + if time.Since(metricsByGrain.metricsValuesUpdated).Seconds() < float64(timeGrains[metricsByGrain.compositeKey.timeGrain]) { continue } metricsByGrain.metricsValuesUpdated = time.Now() @@ -318,8 +324,8 @@ func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID opts := getResourceMetricsValuesRequestOptions( metricsByGrain.metrics, - metricsByGrain.dimensions, - metricsByGrain.timeGrain, + metricsByGrain.compositeKey.dimensions, + metricsByGrain.compositeKey.timeGrain, start, end, ) From 856b74b47ac3785861d5180a11a0b35db4327f3d Mon Sep 17 00:00:00 2001 From: Rajeshwari Muthupandian Date: Thu, 18 May 2023 21:14:39 -0700 Subject: [PATCH 5/6] #21715 - review comments - sort dimensions strings before hasing --- receiver/azuremonitorreceiver/scraper.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/receiver/azuremonitorreceiver/scraper.go b/receiver/azuremonitorreceiver/scraper.go index caa5371490bd5..24120367066e9 100644 --- a/receiver/azuremonitorreceiver/scraper.go +++ b/receiver/azuremonitorreceiver/scraper.go @@ -19,6 +19,7 @@ import ( "context" "crypto/sha256" "fmt" + "sort" "strings" "sync" "time" @@ -80,6 +81,7 @@ type metricsCompositeKey struct { func (s metricsCompositeKey) Hash() string { h := sha256.New() h.Write([]byte(s.timeGrain)) + sort.Strings(s.dimensions) h.Write([]byte(strings.Join(s.dimensions, ","))) return string(h.Sum(nil)) } From 7034dcbe7fb122ba1e0615799ea65d850b517a14 Mon Sep 17 00:00:00 2001 From: Rajeshwari Muthupandian Date: Fri, 19 May 2023 16:45:11 -0700 Subject: [PATCH 6/6] and use metricCompisteKey as key to map --- receiver/azuremonitorreceiver/scraper.go | 41 +++++++++--------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/receiver/azuremonitorreceiver/scraper.go b/receiver/azuremonitorreceiver/scraper.go index 24120367066e9..7bb890b340c91 100644 --- a/receiver/azuremonitorreceiver/scraper.go +++ b/receiver/azuremonitorreceiver/scraper.go @@ -17,7 +17,6 @@ package azuremonitorreceiver // import "github.com/open-telemetry/opentelemetry- import ( "bytes" "context" - "crypto/sha256" "fmt" "sort" "strings" @@ -67,27 +66,18 @@ const ( ) type azureResource struct { - metricsByCompositeKey map[string]*azureResourceMetrics + metricsByCompositeKey map[metricsCompositeKey]*azureResourceMetrics metricsDefinitionsUpdated time.Time tags map[string]*string location string } type metricsCompositeKey struct { - dimensions []string + dimensions string // comma separated sorted dimensions timeGrain string } -func (s metricsCompositeKey) Hash() string { - h := sha256.New() - h.Write([]byte(s.timeGrain)) - sort.Strings(s.dimensions) - h.Write([]byte(strings.Join(s.dimensions, ","))) - return string(h.Sum(nil)) -} - type azureResourceMetrics struct { - compositeKey metricsCompositeKey metrics []string metricsValuesUpdated time.Time } @@ -261,7 +251,7 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour return } - s.resources[resourceID].metricsByCompositeKey = map[string]*azureResourceMetrics{} + s.resources[resourceID].metricsByCompositeKey = map[metricsCompositeKey]*azureResourceMetrics{} pager := s.clientMetricsDefinitions.NewListPager(resourceID, nil) for pager.More() { @@ -284,7 +274,8 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour dimensionsSlice = append(dimensionsSlice, *dimension.Value) } } - dimensionsCompositeKey := metricsCompositeKey{timeGrain: timeGrain, dimensions: dimensionsSlice} + sort.Strings(dimensionsSlice) + dimensionsCompositeKey := metricsCompositeKey{timeGrain: timeGrain, dimensions: strings.Join(dimensionsSlice, ",")} s.storeMetricsDefinition(resourceID, name, dimensionsCompositeKey) } else { s.storeMetricsDefinition(resourceID, name, compositeKey) @@ -295,22 +286,21 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour } func (s *azureScraper) storeMetricsDefinition(resourceID, name string, compositeKey metricsCompositeKey) { - hash := compositeKey.Hash() - if _, ok := s.resources[resourceID].metricsByCompositeKey[hash]; ok { - s.resources[resourceID].metricsByCompositeKey[hash].metrics = append( - s.resources[resourceID].metricsByCompositeKey[hash].metrics, name, + if _, ok := s.resources[resourceID].metricsByCompositeKey[compositeKey]; ok { + s.resources[resourceID].metricsByCompositeKey[compositeKey].metrics = append( + s.resources[resourceID].metricsByCompositeKey[compositeKey].metrics, name, ) } else { - s.resources[resourceID].metricsByCompositeKey[hash] = &azureResourceMetrics{metrics: []string{name}, compositeKey: compositeKey} + s.resources[resourceID].metricsByCompositeKey[compositeKey] = &azureResourceMetrics{metrics: []string{name}} } } func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID string) { res := *s.resources[resourceID] - for _, metricsByGrain := range res.metricsByCompositeKey { + for compositeKey, metricsByGrain := range res.metricsByCompositeKey { - if time.Since(metricsByGrain.metricsValuesUpdated).Seconds() < float64(timeGrains[metricsByGrain.compositeKey.timeGrain]) { + if time.Since(metricsByGrain.metricsValuesUpdated).Seconds() < float64(timeGrains[compositeKey.timeGrain]) { continue } metricsByGrain.metricsValuesUpdated = time.Now() @@ -326,8 +316,8 @@ func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID opts := getResourceMetricsValuesRequestOptions( metricsByGrain.metrics, - metricsByGrain.compositeKey.dimensions, - metricsByGrain.compositeKey.timeGrain, + compositeKey.dimensions, + compositeKey.timeGrain, start, end, ) @@ -373,7 +363,7 @@ func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID func getResourceMetricsValuesRequestOptions( metrics []string, - dimensions []string, + dimensionsStr string, timeGrain string, start int, end int, @@ -386,8 +376,9 @@ func getResourceMetricsValuesRequestOptions( Aggregation: to.Ptr(strings.Join(aggregations, ",")), } - if len(dimensions) > 0 { + if len(dimensionsStr) > 0 { var dimensionsFilter bytes.Buffer + dimensions := strings.Split(dimensionsStr, ",") for i, dimension := range dimensions { dimensionsFilter.WriteString(dimension) dimensionsFilter.WriteString(" eq '*' ")