diff --git a/README.md b/README.md index 2103dd72..82be9fd2 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ If you are still using the legacy [Access scopes][access-scopes], the `https://w | `monitoring.metrics-type-prefixes` | Yes | | Comma separated Google Stackdriver Monitoring Metric Type prefixes (see [example][metrics-prefix-example] and [available metrics][metrics-list]) | | `monitoring.metrics-interval` | No | `5m` | Metric's timestamp interval to request from the Google Stackdriver Monitoring Metrics API. Only the most recent data point is used | | `monitoring.metrics-offset` | No | `0s` | Offset (into the past) for the metric's timestamp interval to request from the Google Stackdriver Monitoring Metrics API, to handle latency in published metrics | -| `monitoring.filters` | No | | Formatted string to allow filtering on certain metrics type | +| `monitoring.filters` | No | | Additonal filters to be sent on the Monitoring API call. Add multiple filters by providing this parameter multiple times. See [monitoring.filters](#using-filters) for more info. | | `monitoring.aggregate-deltas` | No | | If enabled will treat all DELTA metrics as an in-memory counter instead of a gauge. Be sure to read [what to know about aggregating DELTA metrics](#what-to-know-about-aggregating-delta-metrics) | | `monitoring.aggregate-deltas-ttl` | No | `30m` | How long should a delta metric continue to be exported and stored after GCP stops producing it. Read [slow moving metrics](#slow-moving-metrics) to understand the problem this attempts to solve | | `monitoring.descriptor-cache-ttl` | No | `0s` | How long should the metric descriptors for a prefixed be cached for | @@ -147,13 +147,33 @@ stackdriver_exporter \ --monitoring.metrics-type-prefixes "compute.googleapis.com/instance/cpu,compute.googleapis.com/instance/disk" ``` -Using extra filters: +### Using filters +The structure for a filter is `:` + +The `targeted_metric_prefix` is used to ensure the filter is only applied to the metric_prefix(es) where it makes sense. +It does not explicitly have to match a value from `metric_prefixes` but the `targeted_metric_prefix` must be at least a prefix to one or more `metric_prefixes` + +Example: \ + metrics_prefixes = pubsub.googleapis.com/snapshot, pubsub.googleapis.com/subscription/num_undelivered_messages \ + targeted_metric_prefix options would be \ + pubsub.googleapis.com (apply to all defined prefixes) \ + pubsub.googleapis.com/snapshot (apply to only snapshot metrics) \ + pubsub.googleapis.com/subscription (apply to only subscription metrics) \ + pubsub.googleapis.com/subscription/num_undelivered_messages (apply to only the specific subscription metric) \ + +The `filter_query` will be applied to a final metrics API query when querying for metric data. You can read more about the metric API filter options in GCPs documentation https://cloud.google.com/monitoring/api/v3/filters + +The final query sent to the metrics API already includes filters for project and metric type. Each applicable `filter_query` will be appended to the query with an AND + +Full example ``` stackdriver_exporter \ --google.project-id=my-test-project \ --monitoring.metrics-type-prefixes='pubsub.googleapis.com/subscription' \ - --monitoring.filters='pubsub.googleapis.com/subscription:resource.labels.subscription_id=monitoring.regex.full_match("us-west4.*my-team-subs.*")' + --monitoring.metrics-type-prefixes='compute.googleapis.com/instance/cpu' \ + --monitoring.filters='pubsub.googleapis.com/subscription:resource.labels.subscription_id=monitoring.regex.full_match("us-west4.*my-team-subs.*")' \ + --monitoring.filters='compute.googleapis.com/instance/cpu:resource.labels.instance=monitoring.regex.full_match("us-west4.*my-team-subs.*")' ``` Using projects filter: diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index 4f9ddb30..821c2438 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -33,8 +33,8 @@ import ( const namespace = "stackdriver" type MetricFilter struct { - Prefix string - Modifier string + TargetedMetricPrefix string + FilterQuery string } type MonitoringCollector struct { @@ -306,8 +306,8 @@ func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metri } for _, ef := range c.metricsFilters { - if strings.Contains(metricDescriptor.Type, ef.Prefix) { - filter = fmt.Sprintf("%s AND (%s)", filter, ef.Modifier) + if strings.HasPrefix(metricDescriptor.Type, ef.TargetedMetricPrefix) { + filter = fmt.Sprintf("%s AND (%s)", filter, ef.FilterQuery) } } diff --git a/stackdriver_exporter.go b/stackdriver_exporter.go index f71c4f43..fce94d41 100644 --- a/stackdriver_exporter.go +++ b/stackdriver_exporter.go @@ -110,7 +110,9 @@ var ( ).Default("false").Bool() monitoringMetricsExtraFilter = kingpin.Flag( - "monitoring.filters", "Filters. i.e: pubsub.googleapis.com/subscription:resource.labels.subscription_id=monitoring.regex.full_match(\"my-subs-prefix.*\")").Strings() + "monitoring.filters", + "Filters. i.e: pubsub.googleapis.com/subscription:resource.labels.subscription_id=monitoring.regex.full_match(\"my-subs-prefix.*\")", + ).Strings() monitoringMetricsAggregateDeltas = kingpin.Flag( "monitoring.aggregate-deltas", "If enabled will treat all DELTA metrics as an in-memory counter instead of a gauge", @@ -277,9 +279,6 @@ func main() { } } - level.Info(logger).Log("msg", "Starting stackdriver_exporter", "version", version.Info()) - level.Info(logger).Log("msg", "Build context", "build_context", version.BuildContext()) - monitoringService, err := createMonitoringService(ctx) if err != nil { level.Error(logger).Log("msg", "failed to create monitoring service", "err", err) @@ -289,7 +288,6 @@ func main() { var projectIDs []string if *projectsFilter != "" { - level.Info(logger).Log("msg", "Using Google Cloud Projects Filter", "projectsFilter", *projectsFilter) projectIDs, err = utils.GetProjectIDsFromFilter(ctx, *projectsFilter) if err != nil { level.Error(logger).Log("msg", "failed to get project IDs from filter", "err", err) @@ -301,7 +299,16 @@ func main() { projectIDs = append(projectIDs, strings.Split(*projectID, ",")...) } - level.Info(logger).Log("msg", "Using Google Cloud Project IDs", "projectIDs", fmt.Sprintf("%v", projectIDs)) + level.Info(logger).Log( + "msg", "Starting stackdriver_exporter", + "version", version.Info(), + "build_context", version.BuildContext(), + "projects", *projectID, + "metric_prefixes", *monitoringMetricsTypePrefixes, + "extra_filters", strings.Join(*monitoringMetricsExtraFilter, ","), + "projectIDs", fmt.Sprintf("%v", projectIDs), + "projectsFilter", *projectsFilter, + ) metricsTypePrefixes := strings.Split(*monitoringMetricsTypePrefixes, ",") metricExtraFilters := parseMetricExtraFilters() @@ -356,11 +363,11 @@ func main() { func parseMetricExtraFilters() []collectors.MetricFilter { var extraFilters []collectors.MetricFilter for _, ef := range *monitoringMetricsExtraFilter { - efPrefix, efModifier := utils.GetExtraFilterModifiers(ef, ":") - if efPrefix != "" { + targetedMetricPrefix, filterQuery := utils.SplitExtraFilter(ef, ":") + if targetedMetricPrefix != "" { extraFilter := collectors.MetricFilter{ - Prefix: efPrefix, - Modifier: efModifier, + TargetedMetricPrefix: strings.ToLower(targetedMetricPrefix), + FilterQuery: filterQuery, } extraFilters = append(extraFilters, extraFilter) } diff --git a/utils/utils.go b/utils/utils.go index 37018e07..58ac9eca 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -41,7 +41,7 @@ func NormalizeMetricName(metricName string) string { return strings.Join(normalizedMetricName, "_") } -func GetExtraFilterModifiers(extraFilter string, separator string) (string, string) { +func SplitExtraFilter(extraFilter string, separator string) (string, string) { mPrefix := strings.Split(extraFilter, separator) if mPrefix[0] == extraFilter { return "", ""