feat(api-usage-monitoring-filter): use regex for realms to track (zal…

…ando#949)

Signed-off-by: Maxim Tschumak <maxim.tschumak@gmail.com>

cmd/skipper/main.go

@@ -73,7 +73,7 @@ const (
 	defaultApiUsageMonitoringRealmKeys                    = ""
 	defaultApiUsageMonitoringClientKeys                   = "sub"
 	defaultApiUsageMonitoringDefaultClientTrackingPattern = ""
-	defaultApiUsageMonitoringRealms                       = "services"
+	defaultApiUsageMonitoringRealmsTrackingPattern        = "services"
 
 	// generic:
 	addressUsage                         = "network address that skipper should listen on"
@@ -170,7 +170,7 @@ const (
 	apiUsageMonitoringRealmKeysUsage                    = "name of the property in the JWT payload that contains the authority realm"
 	apiUsageMonitoringClientKeysUsage                   = "comma separated list of names of the properties in the JWT body that contains the client ID"
 	apiUsageMonitoringDefaultClientTrackingPatternUsage = "*Deprecated*: set `client_tracking_pattern` directly on filter"
-	apiUsageMonitoringRealmsUsage                       = "comma separated list of realms to be monitored (defaults to 'services')"
+	apiUsageMonitoringRealmsTrackingPatternUsage        = "regular expression used for matching monitored realms (defaults is 'services')"
 
 	// connections, timeouts:
 	idleConnsPerHostUsage             = "maximum idle connections per backend host"
@@ -312,7 +312,7 @@ var (
 	apiUsageMonitoringRealmKeys                    string
 	apiUsageMonitoringClientKeys                   string
 	apiUsageMonitoringDefaultClientTrackingPattern string
-	apiUsageMonitoringRealms                       string
+	apiUsageMonitoringRealmsTrackingPattern        string
 
 	// connections, timeouts:
 	idleConnsPerHost             int
@@ -452,7 +452,7 @@ func init() {
 	flag.StringVar(&apiUsageMonitoringRealmKeys, "api-usage-monitoring-realm-keys", defaultApiUsageMonitoringRealmKeys, apiUsageMonitoringRealmKeysUsage)
 	flag.StringVar(&apiUsageMonitoringClientKeys, "api-usage-monitoring-client-keys", defaultApiUsageMonitoringClientKeys, apiUsageMonitoringClientKeysUsage)
 	flag.StringVar(&apiUsageMonitoringDefaultClientTrackingPattern, "api-usage-monitoring-default-client-tracking-pattern", defaultApiUsageMonitoringDefaultClientTrackingPattern, apiUsageMonitoringDefaultClientTrackingPatternUsage)
-	flag.StringVar(&apiUsageMonitoringRealms, "api-usage-monitoring-realms", defaultApiUsageMonitoringRealms, apiUsageMonitoringRealmsUsage)
+	flag.StringVar(&apiUsageMonitoringRealmsTrackingPattern, "api-usage-monitoring-realms-tracking-pattern", defaultApiUsageMonitoringRealmsTrackingPattern, apiUsageMonitoringRealmsTrackingPatternUsage)
 
 	// connections, timeouts:
 	flag.IntVar(&idleConnsPerHost, "idle-conns-num", proxy.DefaultIdleConnsPerHost, idleConnsPerHostUsage)
@@ -657,7 +657,7 @@ func main() {
 		ApiUsageMonitoringRealmKeys:                    apiUsageMonitoringRealmKeys,
 		ApiUsageMonitoringClientKeys:                   apiUsageMonitoringClientKeys,
 		ApiUsageMonitoringDefaultClientTrackingPattern: apiUsageMonitoringDefaultClientTrackingPattern,
-		ApiUsageMonitoringRealms:                       apiUsageMonitoringRealms,
+		ApiUsageMonitoringRealmsTrackingPattern:        apiUsageMonitoringRealmsTrackingPattern,
 
 		// Auth:
 		OAuthUrl:                       oauthURL,

docs/reference/filters.md

@@ -1050,15 +1050,15 @@ For the client based metrics, additional flags need to be specified.
 |--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `api-usage-monitoring-realm-keys`                      | Name of the property in the JWT JSON body that contains the name of the _realm_.                                                                                                                                         |
 | `api-usage-monitoring-client-keys`                     | Name of the property in the JWT JSON body that contains the name of the _client_.                                                                                                                                        |
-| `api-usage-monitoring-realms`                          | List of _realms_ to be monitored. Defaults to 'services'.                                                                                                                                                                |
+| `api-usage-monitoring-realms-tracking-pattern`         | RegEx of _realms_ to be monitored. Defaults to 'services'.                                                                                                                                                                |
 
 NOTE: Make sure to activate the metrics flavour proper to your environment using the `metrics-flavour`
 flag in order to get those metrics.
 
 Example:
 
 ```bash
-skipper -metrics-flavour prometheus -enable-api-usage-monitoring -api-usage-monitoring-realm-keys="realm" -api-usage-monitoring-client-keys="managed-id" api-usage-monitoring-realms="services,users"
+skipper -metrics-flavour prometheus -enable-api-usage-monitoring -api-usage-monitoring-realm-keys="realm" -api-usage-monitoring-client-keys="managed-id" api-usage-monitoring-realms-tracking-pattern="services,users"
 ```
 
 The structure of the metrics is all of those elements, separated by `.` dots:
@@ -1227,12 +1227,13 @@ Here is the _Prometheus_ query to obtain it.
 histogram_quantile(0.5, sum(rate(skipper_custom_duration_seconds_bucket{key="apiUsageMonitoring.custom.my-app.orders-api.POST.foo/orders.*.*.latency"}[60s])) by (le, key))
 ```
 
-NOTE: Non configured paths will be tracked with `<unknown>` application ID, API ID
+NOTE: Non configured paths will be tracked with `{unknown}` application ID, API ID
 and path template.
 
 However, if all `application_id`s of your configuration refer to the same application,
 the filter assume that also non configured paths will be directed to this application.
 E.g.:
 
 ```
-apiUsageMonitoring.custom.my-app.<unknown>.GET.<unknown>.*.*.http_count
+apiUsageMonitoring.custom.my-app.{unknown}.GET.{no-match}.*.*.http_count
+```

filters/apiusagemonitoring/doc.go

@@ -36,7 +36,7 @@ Command line example for executing locally:
 		-enable-api-usage-monitoring \
 		-api-usage-monitoring-realm-keys="https://identity.zalando.com/realm" \
 		-api-usage-monitoring-client-keys="https://identity.zalando.com/managed-id,sub" \
-		-api-usage-monitoring-realms="services" \
+		-api-usage-monitoring-realmsTrackingPattern-tracking-pattern="services" \
 		-metrics-flavour prometheus \
 		-histogram-metric-buckets=".01,.025,.05,.075,.1,.2,.3,.4,.5,.75,1,2,3,4,5,7,10,15,20,30,60,120,300,600" \
 		-application-log-level=DEBUG

filters/apiusagemonitoring/filter.go

@@ -120,8 +120,8 @@ func (f *apiUsageMonitoringFilter) getRealmClientKey(r *http.Request, path *path
 		return unknownUnknown
 	}
 
-	// realm is not one of the realms to be tracked ==> realm.{all}
-	if !contains(path.ClientTracking.RealmsToTrack, realm) {
+	// realm is not one of the realmsTrackingPattern to be tracked ==> realm.{all}
+	if !path.ClientTracking.RealmsTrackingMatcher.MatchString(realm) {
 		return realm + ".{all}"
 	}
 
@@ -140,15 +140,6 @@ func (f *apiUsageMonitoringFilter) getRealmClientKey(r *http.Request, path *path
 	return realm + "." + client
 }
 
-func contains(strings []string, s string) bool {
-	for _, v := range strings {
-		if s == v {
-			return true
-		}
-	}
-	return false
-}
-
 // resolvePath tries to match the request's path with one of the configured path template.
 func (f *apiUsageMonitoringFilter) resolvePath(req *http.Request) *pathInfo {
 	for _, p := range f.Paths {

filters/apiusagemonitoring/filter_clientmetrics_test.go

@@ -15,7 +15,7 @@ type clientMetricsTest struct {
 
 	expectedEndpointMetricPrefix string
 	expectedClientMetricPrefix   string
-	realms                       string
+	realmsTrackingPattern        string
 }
 
 var (
@@ -38,6 +38,7 @@ func Test_Filter_ClientMetrics_ClientTrackingPatternDoesNotCompile(t *testing.T)
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:                 "realm",
 		clientKeyName:                "client",
+		realmsTrackingPattern:        "service",
 		clientTrackingPattern:        s("(["),
 		header:                       headerUsersJoe,
 		expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.",
@@ -50,11 +51,11 @@ func Test_Filter_ClientMetrics_NoMatchingPath_RealmIsKnown(t *testing.T) {
 		realmKeyName:                 "realm",
 		clientKeyName:                "client",
 		clientTrackingPattern:        s(".*"),
+		realmsTrackingPattern:        "services",
 		header:                       headerUsersJoe,
 		url:                          "https://www.example.com/non/configured/path/template",
 		expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.{unknown}.GET.{no-match}.*.*.",
 		expectedClientMetricPrefix:   "apiUsageMonitoring.custom.my_app.{unknown}.*.*.users.{all}.",
-		realms:                       "services",
 	})
 }
 
@@ -78,7 +79,7 @@ func Test_Filter_ClientMetrics_MatchAll(t *testing.T) {
 		header:                       headerUsersJoe,
 		expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.",
 		expectedClientMetricPrefix:   "apiUsageMonitoring.custom.my_app.my_api.*.*.users.joe.",
-		realms:                       "users",
+		realmsTrackingPattern:        "users",
 	})
 }
 
@@ -97,15 +98,15 @@ func Test_Filter_ClientMetrics_MatchOneOfClientKeyName(t *testing.T) {
 		},
 		expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.",
 		expectedClientMetricPrefix:   "apiUsageMonitoring.custom.my_app.my_api.*.*.services.payments.",
-		realms:                       "services",
+		realmsTrackingPattern:        "services",
 	})
 }
 
 func Test_Filter_ClientMetrics_MatchOneOfClientKeyName_UseFirstMatching(t *testing.T) {
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:          "realm",
 		clientKeyName:         "client,sub",
-		realms:                "services",
+		realmsTrackingPattern: "services",
 		clientTrackingPattern: s(".*"),
 		header: http.Header{
 			authorizationHeaderName: {
@@ -125,7 +126,7 @@ func Test_Filter_ClientMetrics_Realm1User1(t *testing.T) {
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:                 "realm",
 		clientKeyName:                "client",
-		realms:                       "users",
+		realmsTrackingPattern:        "users",
 		clientTrackingPattern:        clientTrackingPatternJustSomeUsers,
 		header:                       headerUsersJoe,
 		expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.",
@@ -137,7 +138,7 @@ func Test_Filter_ClientMetrics_Realm1User0(t *testing.T) {
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:          "realm",
 		clientKeyName:         "client",
-		realms:                "users",
+		realmsTrackingPattern: "users",
 		clientTrackingPattern: clientTrackingPatternJustSomeUsers,
 		header: http.Header{
 			authorizationHeaderName: {
@@ -156,7 +157,7 @@ func Test_Filter_ClientMetrics_Realm0User1(t *testing.T) {
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:          "realm",
 		clientKeyName:         "client",
-		realms:                "services",
+		realmsTrackingPattern: "services",
 		clientTrackingPattern: clientTrackingPatternJustSomeUsers,
 		header: http.Header{
 			authorizationHeaderName: {
@@ -175,7 +176,7 @@ func Test_Filter_ClientMetrics_Realm0User0(t *testing.T) {
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:          "realm",
 		clientKeyName:         "client",
-		realms:                "services",
+		realmsTrackingPattern: "services",
 		clientTrackingPattern: clientTrackingPatternJustSomeUsers,
 		header: http.Header{
 			authorizationHeaderName: {
@@ -194,7 +195,7 @@ func Test_Filter_ClientMetrics_AuthDoesNotHaveBearerPrefix(t *testing.T) {
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:          "realm",
 		clientKeyName:         "client",
-		realms:                "services",
+		realmsTrackingPattern: "services",
 		clientTrackingPattern: clientTrackingPatternJustSomeUsers,
 		header: http.Header{
 			authorizationHeaderName: {
@@ -290,7 +291,7 @@ func Test_Filter_ClientMetrics_JWTBodyHasNoClient_ShouldTrackRealm(t *testing.T)
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:          "realm",
 		clientKeyName:         "client",
-		realms:                "users",
+		realmsTrackingPattern: "users",
 		clientTrackingPattern: clientTrackingPatternJustSomeUsers,
 		header: http.Header{
 			authorizationHeaderName: {
@@ -331,7 +332,7 @@ func Test_Filter_ClientMetrics_DefaultClientTrackingPattern_NoClientTrackingPatt
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:          "realm",
 		clientKeyName:         "client",
-		realms:                "services",
+		realmsTrackingPattern: "services",
 		clientTrackingPattern: nil, // no client tracking in route's filter configuration
 		header: http.Header{
 			authorizationHeaderName: {
@@ -350,7 +351,7 @@ func Test_Filter_ClientMetrics_DefaultClientTrackingPattern_NoClientTrackingPatt
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:          "realm",
 		clientKeyName:         "client",
-		realms:                "services",
+		realmsTrackingPattern: "services",
 		clientTrackingPattern: nil, // no client tracking in route's filter configuration
 		header: http.Header{
 			authorizationHeaderName: {
@@ -369,7 +370,7 @@ func Test_Filter_ClientMetrics_EmptyClientTrackingPatternInRouteFilterJSON(t *te
 	testClientMetrics(t, clientMetricsTest{
 		realmKeyName:          "realm",
 		clientKeyName:         "client",
-		realms:                "services",
+		realmsTrackingPattern: "services",
 		clientTrackingPattern: s(""), // no client tracking in route's filter configuration
 		header: http.Header{
 			authorizationHeaderName: {

filters/apiusagemonitoring/pathinfo.go

@@ -91,5 +91,5 @@ var (
 
 type clientTrackingInfo struct {
 	ClientTrackingMatcher *regexp.Regexp
-	RealmsToTrack         []string
+	RealmsTrackingMatcher *regexp.Regexp
 }

filters/apiusagemonitoring/spec.go

@@ -32,7 +32,7 @@ func NewApiUsageMonitoring(
 	enabled bool,
 	realmKeys string,
 	clientKeys string,
-	realms string,
+	realmsTrackingPattern string,
 ) filters.Spec {
 	if !enabled {
 		log.Debugf("filter %q is not enabled. spec returns `noop` filters.", Name)
@@ -55,20 +55,22 @@ func NewApiUsageMonitoring(
 			clientKeyList = append(clientKeyList, strippedKey)
 		}
 	}
-	// parse comma separated list of realms to monitor
-	var realmValues []string
-	for _, key := range strings.Split(realms, ",") {
-		strippedKey := strings.TrimSpace(key)
-		if strippedKey != "" {
-			realmValues = append(realmValues, strippedKey)
-		}
+	// compile realms regex
+	realmsTrackingMatcher, err := regexp.Compile(realmsTrackingPattern)
+	if err != nil {
+		log.Errorf(
+			"api-usage-monitoring-realmsTrackingPattern-tracking-pattern (global config) ignored: error compiling regular expression %q: %v",
+			realmsTrackingPattern, err)
+		realmsTrackingMatcher = regexp.MustCompile("services")
+		log.Warn("defaulting to 'services' as api-usage-monitoring-realmsTrackingPattern-tracking-pattern (global config)")
 	}
 
 	// Create the filter Spec
 	var unknownPathClientTracking *clientTrackingInfo = nil // client metrics feature is disabled
 	if realmKeys != "" {
 		unknownPathClientTracking = &clientTrackingInfo{
-			ClientTrackingMatcher: nil, // do not match anything (track `realm.<unknown>`)
+			ClientTrackingMatcher: nil, // do not match anything (track `realm.{unknown}`)
+			RealmsTrackingMatcher: realmsTrackingMatcher,
 		}
 	}
 	unknownPath := newPathInfo(
@@ -78,11 +80,12 @@ func NewApiUsageMonitoring(
 		noMatchPlaceholder,
 		unknownPathClientTracking,
 	)
+
 	spec := &apiUsageMonitoringSpec{
-		realmKeys:   realmKeyList,
-		clientKeys:  clientKeyList,
-		unknownPath: unknownPath,
-		realms:      realmValues,
+		realmKeys:             realmKeyList,
+		clientKeys:            clientKeyList,
+		unknownPath:           unknownPath,
+		realmsTrackingMatcher: realmsTrackingMatcher,
 	}
 	log.Debugf("created filter spec: %+v", spec)
 	return spec
@@ -97,10 +100,10 @@ type apiConfig struct {
 }
 
 type apiUsageMonitoringSpec struct {
-	realmKeys   []string
-	clientKeys  []string
-	realms      []string
-	unknownPath *pathInfo
+	realmKeys             []string
+	clientKeys            []string
+	realmsTrackingMatcher *regexp.Regexp
+	unknownPath           *pathInfo
 }
 
 func (s *apiUsageMonitoringSpec) Name() string {
@@ -124,7 +127,7 @@ func (s *apiUsageMonitoringSpec) CreateFilter(args []interface{}) (filter filter
 }
 
 func (s *apiUsageMonitoringSpec) parseJsonConfiguration(args []interface{}) []*apiConfig {
-	apis := make([]*apiConfig, 0)
+	apis := make([]*apiConfig, 0, len(args))
 	for i, a := range args {
 		rawJsonConfiguration, ok := a.(string)
 		if !ok {
@@ -170,6 +173,7 @@ func (s *apiUsageMonitoringSpec) buildPathInfoListFromConfiguration(apis []*apiC
 	var paths []*pathInfo
 	existingPathTemplates := make(map[string]*pathInfo)
 	existingRegEx := make(map[string]*pathInfo)
+
 	for apiIndex, api := range apis {
 
 		applicationId := api.ApplicationId
@@ -189,7 +193,7 @@ func (s *apiUsageMonitoringSpec) buildPathInfoListFromConfiguration(apis []*apiC
 			continue
 		}
 
-		clientTrackingInfo := s.buildClientTrackingInfo(apiIndex, api, s.realms)
+		clientTrackingInfo := s.buildClientTrackingInfo(apiIndex, api, s.realmsTrackingMatcher)
 
 		for templateIndex, template := range api.PathTemplates {
 
@@ -246,7 +250,7 @@ func (s *apiUsageMonitoringSpec) buildPathInfoListFromConfiguration(apis []*apiC
 	return paths
 }
 
-func (s *apiUsageMonitoringSpec) buildClientTrackingInfo(apiIndex int, api *apiConfig, realmsToTrack []string) *clientTrackingInfo {
+func (s *apiUsageMonitoringSpec) buildClientTrackingInfo(apiIndex int, api *apiConfig, realmsTrackingMatcher *regexp.Regexp) *clientTrackingInfo {
 	if len(s.realmKeys) == 0 {
 		log.Infof(
 			`args[%d]: skipper wide configuration "api-usage-monitoring-realm-keys" not provided, not tracking client metrics`,
@@ -276,7 +280,7 @@ func (s *apiUsageMonitoringSpec) buildClientTrackingInfo(apiIndex int, api *apiC
 
 	return &clientTrackingInfo{
 		ClientTrackingMatcher: clientTrackingMatcher,
-		RealmsToTrack:         realmsToTrack,
+		RealmsTrackingMatcher: realmsTrackingMatcher,
 	}
 }
 

filters/apiusagemonitoring/testutils_test.go

@@ -190,7 +190,7 @@ func testClientMetrics(t *testing.T, testCase clientMetricsTest) {
 				t.FailNow()
 			}
 			args := []interface{}{string(js)}
-			spec := NewApiUsageMonitoring(true, testCase.realmKeyName, testCase.clientKeyName, testCase.realms)
+			spec := NewApiUsageMonitoring(true, testCase.realmKeyName, testCase.clientKeyName, testCase.realmsTrackingPattern)
 			return spec.CreateFilter(args)
 		},
 	}