diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0e3c87698aa..483e1dbff86 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -184,6 +184,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Update README file on how to run Metricbeat on Kubernetes. {pull}33308[33308] - Add per-thread metrics to system_summary {pull}33614[33614] - Add GCP CloudSQL metadata {pull}33066[33066] +- Remove GCP Compute metadata cache {pull}33655[33655] - Add support for multiple regions in GCP {pull}32964[32964] - Add GCP Redis regions support {pull}33728[33728] diff --git a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go index 4fd84ece5ee..700a667164c 100644 --- a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go @@ -9,45 +9,35 @@ import ( "fmt" "strconv" "strings" - "time" - "google.golang.org/api/compute/v1" + compute "google.golang.org/api/compute/v1" "google.golang.org/api/option" monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" ) -const ( - cacheTTL = 30 * time.Second - initialCacheSize = 13 -) - // NewMetadataService returns the specific Metadata service for a GCP Compute resource func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ - projectID: projectID, - zone: zone, - region: region, - regions: regions, - opt: opt, - instanceCache: common.NewCache(cacheTTL, initialCacheSize), - logger: logp.NewLogger("metrics-compute"), + projectID: projectID, + zone: zone, + region: region, + regions: regions, + opt: opt, + computeInstances: make(map[uint64]*compute.Instance), + logger: logp.NewLogger("metrics-compute"), }, nil } // computeMetadata is an object to store data in between the extraction and the writing in the destination (to uncouple // reading and writing in the same method) type computeMetadata struct { - // projectID string zone string instanceID string machineType string - // ts *monitoringpb.TimeSeries - User map[string]string Metadata map[string]string Metrics interface{} @@ -55,13 +45,13 @@ type computeMetadata struct { } type metadataCollector struct { - projectID string - zone string - region string - regions []string - opt []option.ClientOption - instanceCache *common.Cache - logger *logp.Logger + projectID string + zone string + region string + regions []string + opt []option.ClientOption + computeInstances map[uint64]*compute.Instance + logger *logp.Logger } // Metadata implements googlecloud.MetadataCollector to the known set of labels from a Compute TimeSeries single point of data. @@ -120,6 +110,7 @@ func (s *metadataCollector) instanceMetadata(ctx context.Context, instanceID, zo } if instance == nil { + s.logger.Debugf("couldn't find instance %s, call Instances.AggregatedList", instanceID) return computeMetadata, nil } @@ -144,14 +135,17 @@ func (s *metadataCollector) instanceMetadata(ctx context.Context, instanceID, zo // instance returns data from an instance ID using the cache or making a request func (s *metadataCollector) instance(ctx context.Context, instanceID string) (*compute.Instance, error) { - s.refreshInstanceCache(ctx) - instanceCachedData := s.instanceCache.Get(instanceID) - if instanceCachedData != nil { - if computeInstance, ok := instanceCachedData.(*compute.Instance); ok { - return computeInstance, nil - } + s.getComputeInstances(ctx) + + instanceIdInt, _ := strconv.Atoi(instanceID) + computeInstance, ok := s.computeInstances[uint64(instanceIdInt)] + if ok { + return computeInstance, nil } + // Remake the compute instances map to avoid having stale data. + s.computeInstances = make(map[uint64]*compute.Instance) + return nil, nil } @@ -171,12 +165,13 @@ func (s *metadataCollector) instanceZone(ts *monitoringpb.TimeSeries) string { return "" } -func (s *metadataCollector) refreshInstanceCache(ctx context.Context) { - // only refresh cache if it is empty - if s.instanceCache.Size() > 0 { +func (s *metadataCollector) getComputeInstances(ctx context.Context) { + if len(s.computeInstances) > 0 { return } - s.logger.Debugf("refresh cache with Instances.AggregatedList API") + + s.logger.Debug("Compute API Instances.AggregatedList") + computeService, err := compute.NewService(ctx, s.opt...) if err != nil { s.logger.Errorf("error getting client from Compute service: %v", err) @@ -187,7 +182,7 @@ func (s *metadataCollector) refreshInstanceCache(ctx context.Context) { if err := req.Pages(ctx, func(page *compute.InstanceAggregatedList) error { for _, instancesScopedList := range page.Items { for _, instance := range instancesScopedList.Instances { - s.instanceCache.Put(strconv.Itoa(int(instance.Id)), instance) + s.computeInstances[instance.Id] = instance } } return nil diff --git a/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go index 53f1878addd..cee4484f615 100644 --- a/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go @@ -6,15 +6,12 @@ package compute import ( "testing" - "time" "github.com/golang/protobuf/ptypes/timestamp" "github.com/stretchr/testify/assert" "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/genproto/googleapis/api/monitoredres" - "google.golang.org/genproto/googleapis/monitoring/v3" - - "github.com/elastic/beats/v7/libbeat/common" + monitoring "google.golang.org/genproto/googleapis/monitoring/v3" ) var fake = &monitoring.TimeSeries{ @@ -67,8 +64,7 @@ var fake = &monitoring.TimeSeries{ } var m = &metadataCollector{ - projectID: "projectID", - instanceCache: common.NewCache(30*time.Second, 13), + projectID: "projectID", } func TestInstanceID(t *testing.T) {