From 738f021f133c265e6f63e54ba18b4be822181d69 Mon Sep 17 00:00:00 2001 From: Kyle Eckhart Date: Tue, 31 Jan 2023 14:23:25 -0500 Subject: [PATCH 1/5] Add more info about filters to docs and rename struct fields Signed-off-by: Kyle Eckhart --- README.md | 26 +++++++++++++++++++++++--- collectors/monitoring_collector.go | 8 ++++---- stackdriver_exporter.go | 12 +++++++----- utils/utils.go | 2 +- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2103dd72..82be9fd2 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ If you are still using the legacy [Access scopes][access-scopes], the `https://w | `monitoring.metrics-type-prefixes` | Yes | | Comma separated Google Stackdriver Monitoring Metric Type prefixes (see [example][metrics-prefix-example] and [available metrics][metrics-list]) | | `monitoring.metrics-interval` | No | `5m` | Metric's timestamp interval to request from the Google Stackdriver Monitoring Metrics API. Only the most recent data point is used | | `monitoring.metrics-offset` | No | `0s` | Offset (into the past) for the metric's timestamp interval to request from the Google Stackdriver Monitoring Metrics API, to handle latency in published metrics | -| `monitoring.filters` | No | | Formatted string to allow filtering on certain metrics type | +| `monitoring.filters` | No | | Additonal filters to be sent on the Monitoring API call. Add multiple filters by providing this parameter multiple times. See [monitoring.filters](#using-filters) for more info. | | `monitoring.aggregate-deltas` | No | | If enabled will treat all DELTA metrics as an in-memory counter instead of a gauge. Be sure to read [what to know about aggregating DELTA metrics](#what-to-know-about-aggregating-delta-metrics) | | `monitoring.aggregate-deltas-ttl` | No | `30m` | How long should a delta metric continue to be exported and stored after GCP stops producing it. Read [slow moving metrics](#slow-moving-metrics) to understand the problem this attempts to solve | | `monitoring.descriptor-cache-ttl` | No | `0s` | How long should the metric descriptors for a prefixed be cached for | @@ -147,13 +147,33 @@ stackdriver_exporter \ --monitoring.metrics-type-prefixes "compute.googleapis.com/instance/cpu,compute.googleapis.com/instance/disk" ``` -Using extra filters: +### Using filters +The structure for a filter is `:` + +The `targeted_metric_prefix` is used to ensure the filter is only applied to the metric_prefix(es) where it makes sense. +It does not explicitly have to match a value from `metric_prefixes` but the `targeted_metric_prefix` must be at least a prefix to one or more `metric_prefixes` + +Example: \ + metrics_prefixes = pubsub.googleapis.com/snapshot, pubsub.googleapis.com/subscription/num_undelivered_messages \ + targeted_metric_prefix options would be \ + pubsub.googleapis.com (apply to all defined prefixes) \ + pubsub.googleapis.com/snapshot (apply to only snapshot metrics) \ + pubsub.googleapis.com/subscription (apply to only subscription metrics) \ + pubsub.googleapis.com/subscription/num_undelivered_messages (apply to only the specific subscription metric) \ + +The `filter_query` will be applied to a final metrics API query when querying for metric data. You can read more about the metric API filter options in GCPs documentation https://cloud.google.com/monitoring/api/v3/filters + +The final query sent to the metrics API already includes filters for project and metric type. Each applicable `filter_query` will be appended to the query with an AND + +Full example ``` stackdriver_exporter \ --google.project-id=my-test-project \ --monitoring.metrics-type-prefixes='pubsub.googleapis.com/subscription' \ - --monitoring.filters='pubsub.googleapis.com/subscription:resource.labels.subscription_id=monitoring.regex.full_match("us-west4.*my-team-subs.*")' + --monitoring.metrics-type-prefixes='compute.googleapis.com/instance/cpu' \ + --monitoring.filters='pubsub.googleapis.com/subscription:resource.labels.subscription_id=monitoring.regex.full_match("us-west4.*my-team-subs.*")' \ + --monitoring.filters='compute.googleapis.com/instance/cpu:resource.labels.instance=monitoring.regex.full_match("us-west4.*my-team-subs.*")' ``` Using projects filter: diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index 4f9ddb30..b56c23df 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -33,8 +33,8 @@ import ( const namespace = "stackdriver" type MetricFilter struct { - Prefix string - Modifier string + TargetedMetricPrefix string + FilterQuery string } type MonitoringCollector struct { @@ -306,8 +306,8 @@ func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metri } for _, ef := range c.metricsFilters { - if strings.Contains(metricDescriptor.Type, ef.Prefix) { - filter = fmt.Sprintf("%s AND (%s)", filter, ef.Modifier) + if strings.Contains(metricDescriptor.Type, ef.TargetedMetricPrefix) { + filter = fmt.Sprintf("%s AND (%s)", filter, ef.FilterQuery) } } diff --git a/stackdriver_exporter.go b/stackdriver_exporter.go index f71c4f43..8abee410 100644 --- a/stackdriver_exporter.go +++ b/stackdriver_exporter.go @@ -110,7 +110,9 @@ var ( ).Default("false").Bool() monitoringMetricsExtraFilter = kingpin.Flag( - "monitoring.filters", "Filters. i.e: pubsub.googleapis.com/subscription:resource.labels.subscription_id=monitoring.regex.full_match(\"my-subs-prefix.*\")").Strings() + "monitoring.filters", + "Filters. i.e: pubsub.googleapis.com/subscription:resource.labels.subscription_id=monitoring.regex.full_match(\"my-subs-prefix.*\")", + ).Strings() monitoringMetricsAggregateDeltas = kingpin.Flag( "monitoring.aggregate-deltas", "If enabled will treat all DELTA metrics as an in-memory counter instead of a gauge", @@ -356,11 +358,11 @@ func main() { func parseMetricExtraFilters() []collectors.MetricFilter { var extraFilters []collectors.MetricFilter for _, ef := range *monitoringMetricsExtraFilter { - efPrefix, efModifier := utils.GetExtraFilterModifiers(ef, ":") - if efPrefix != "" { + targetedMetricPrefix, filterQuery := utils.SplitExtraFilter(ef, ":") + if targetedMetricPrefix != "" { extraFilter := collectors.MetricFilter{ - Prefix: efPrefix, - Modifier: efModifier, + TargetedMetricPrefix: targetedMetricPrefix, + FilterQuery: filterQuery, } extraFilters = append(extraFilters, extraFilter) } diff --git a/utils/utils.go b/utils/utils.go index 37018e07..58ac9eca 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -41,7 +41,7 @@ func NormalizeMetricName(metricName string) string { return strings.Join(normalizedMetricName, "_") } -func GetExtraFilterModifiers(extraFilter string, separator string) (string, string) { +func SplitExtraFilter(extraFilter string, separator string) (string, string) { mPrefix := strings.Split(extraFilter, separator) if mPrefix[0] == extraFilter { return "", "" From 83ecb65e8c7e9eb603803b1bef305cce728cd34a Mon Sep 17 00:00:00 2001 From: Kyle Eckhart Date: Sat, 4 Feb 2023 08:45:31 -0500 Subject: [PATCH 2/5] Single startup log with more context Signed-off-by: Kyle Eckhart --- stackdriver_exporter.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/stackdriver_exporter.go b/stackdriver_exporter.go index 8abee410..36ee9dfa 100644 --- a/stackdriver_exporter.go +++ b/stackdriver_exporter.go @@ -279,7 +279,14 @@ func main() { } } - level.Info(logger).Log("msg", "Starting stackdriver_exporter", "version", version.Info()) + level.Info(logger).Log( + "msg", "Starting stackdriver_exporter", + "version", version.Info(), + "build_context", version.BuildContext(), + "projects", *projectID, + "metric_prefixes", *monitoringMetricsTypePrefixes, + "extra_filters", strings.Join(*monitoringMetricsExtraFilter, ","), + ) level.Info(logger).Log("msg", "Build context", "build_context", version.BuildContext()) monitoringService, err := createMonitoringService(ctx) From 10f595cc614b8a24f7fd4ef931c67a72efe8cb9a Mon Sep 17 00:00:00 2001 From: Kyle Eckhart Date: Sat, 4 Feb 2023 08:48:02 -0500 Subject: [PATCH 3/5] ToLower extra fitler prefixes to avoid casing issues with user config Signed-off-by: Kyle Eckhart --- stackdriver_exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdriver_exporter.go b/stackdriver_exporter.go index 36ee9dfa..ca0aafac 100644 --- a/stackdriver_exporter.go +++ b/stackdriver_exporter.go @@ -368,7 +368,7 @@ func parseMetricExtraFilters() []collectors.MetricFilter { targetedMetricPrefix, filterQuery := utils.SplitExtraFilter(ef, ":") if targetedMetricPrefix != "" { extraFilter := collectors.MetricFilter{ - TargetedMetricPrefix: targetedMetricPrefix, + TargetedMetricPrefix: strings.ToLower(targetedMetricPrefix), FilterQuery: filterQuery, } extraFilters = append(extraFilters, extraFilter) From b6295146b8d47b3f7f816f50dd8dd8e7dc0ef6ed Mon Sep 17 00:00:00 2001 From: Kyle Eckhart Date: Fri, 15 Mar 2024 15:24:24 -0400 Subject: [PATCH 4/5] Switch extra filters to apply by prefix instead of contains Signed-off-by: Kyle Eckhart --- collectors/monitoring_collector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index b56c23df..821c2438 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -306,7 +306,7 @@ func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metri } for _, ef := range c.metricsFilters { - if strings.Contains(metricDescriptor.Type, ef.TargetedMetricPrefix) { + if strings.HasPrefix(metricDescriptor.Type, ef.TargetedMetricPrefix) { filter = fmt.Sprintf("%s AND (%s)", filter, ef.FilterQuery) } } From 1ec30461c519a2c116864d9bc27164287102190e Mon Sep 17 00:00:00 2001 From: Kyle Eckhart Date: Fri, 15 Mar 2024 15:24:39 -0400 Subject: [PATCH 5/5] Single startup log with more context - post rebase Signed-off-by: Kyle Eckhart --- stackdriver_exporter.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/stackdriver_exporter.go b/stackdriver_exporter.go index ca0aafac..fce94d41 100644 --- a/stackdriver_exporter.go +++ b/stackdriver_exporter.go @@ -279,16 +279,6 @@ func main() { } } - level.Info(logger).Log( - "msg", "Starting stackdriver_exporter", - "version", version.Info(), - "build_context", version.BuildContext(), - "projects", *projectID, - "metric_prefixes", *monitoringMetricsTypePrefixes, - "extra_filters", strings.Join(*monitoringMetricsExtraFilter, ","), - ) - level.Info(logger).Log("msg", "Build context", "build_context", version.BuildContext()) - monitoringService, err := createMonitoringService(ctx) if err != nil { level.Error(logger).Log("msg", "failed to create monitoring service", "err", err) @@ -298,7 +288,6 @@ func main() { var projectIDs []string if *projectsFilter != "" { - level.Info(logger).Log("msg", "Using Google Cloud Projects Filter", "projectsFilter", *projectsFilter) projectIDs, err = utils.GetProjectIDsFromFilter(ctx, *projectsFilter) if err != nil { level.Error(logger).Log("msg", "failed to get project IDs from filter", "err", err) @@ -310,7 +299,16 @@ func main() { projectIDs = append(projectIDs, strings.Split(*projectID, ",")...) } - level.Info(logger).Log("msg", "Using Google Cloud Project IDs", "projectIDs", fmt.Sprintf("%v", projectIDs)) + level.Info(logger).Log( + "msg", "Starting stackdriver_exporter", + "version", version.Info(), + "build_context", version.BuildContext(), + "projects", *projectID, + "metric_prefixes", *monitoringMetricsTypePrefixes, + "extra_filters", strings.Join(*monitoringMetricsExtraFilter, ","), + "projectIDs", fmt.Sprintf("%v", projectIDs), + "projectsFilter", *projectsFilter, + ) metricsTypePrefixes := strings.Split(*monitoringMetricsTypePrefixes, ",") metricExtraFilters := parseMetricExtraFilters()