diff --git a/cmd/skipper/main.go b/cmd/skipper/main.go index 317c800975..d78a00537c 100644 --- a/cmd/skipper/main.go +++ b/cmd/skipper/main.go @@ -73,6 +73,7 @@ const ( defaultApiUsageMonitoringRealmKeys = "" defaultApiUsageMonitoringClientKeys = "sub" defaultApiUsageMonitoringDefaultClientTrackingPattern = "" + defaultApiUsageMonitoringRealms = "services" // generic: addressUsage = "network address that skipper should listen on" @@ -168,7 +169,8 @@ const ( apiUsageMonitoringEnableUsage = "enables the apiUsageMonitoring filter" apiUsageMonitoringRealmKeysUsage = "name of the property in the JWT payload that contains the authority realm" apiUsageMonitoringClientKeysUsage = "comma separated list of names of the properties in the JWT body that contains the client ID" - apiUsageMonitoringDefaultClientTrackingPatternUsage = "regular expression to default to when API usage monitoring filter configuration does not provide `client_tracking_pattern`" + apiUsageMonitoringDefaultClientTrackingPatternUsage = "*Deprecated*: set `client_tracking_pattern` directly on filter" + apiUsageMonitoringRealmsUsage = "comma separated list of realms to be monitored (defaults to 'services')" // connections, timeouts: idleConnsPerHostUsage = "maximum idle connections per backend host" @@ -310,6 +312,7 @@ var ( apiUsageMonitoringRealmKeys string apiUsageMonitoringClientKeys string apiUsageMonitoringDefaultClientTrackingPattern string + apiUsageMonitoringRealms string // connections, timeouts: idleConnsPerHost int @@ -449,6 +452,7 @@ func init() { flag.StringVar(&apiUsageMonitoringRealmKeys, "api-usage-monitoring-realm-keys", defaultApiUsageMonitoringRealmKeys, apiUsageMonitoringRealmKeysUsage) flag.StringVar(&apiUsageMonitoringClientKeys, "api-usage-monitoring-client-keys", defaultApiUsageMonitoringClientKeys, apiUsageMonitoringClientKeysUsage) flag.StringVar(&apiUsageMonitoringDefaultClientTrackingPattern, "api-usage-monitoring-default-client-tracking-pattern", defaultApiUsageMonitoringDefaultClientTrackingPattern, apiUsageMonitoringDefaultClientTrackingPatternUsage) + flag.StringVar(&apiUsageMonitoringRealms, "api-usage-monitoring-realms", defaultApiUsageMonitoringRealms, apiUsageMonitoringRealmsUsage) // connections, timeouts: flag.IntVar(&idleConnsPerHost, "idle-conns-num", proxy.DefaultIdleConnsPerHost, idleConnsPerHostUsage) @@ -650,9 +654,10 @@ func main() { // API Monitoring: ApiUsageMonitoringEnable: apiUsageMonitoringEnable, - ApiUsageMonitoringRealmKey: apiUsageMonitoringRealmKeys, - ApiUsageMonitoringClientIdKeyName: apiUsageMonitoringClientKeys, + ApiUsageMonitoringRealmKeys: apiUsageMonitoringRealmKeys, + ApiUsageMonitoringClientKeys: apiUsageMonitoringClientKeys, ApiUsageMonitoringDefaultClientTrackingPattern: apiUsageMonitoringDefaultClientTrackingPattern, + ApiUsageMonitoringRealms: apiUsageMonitoringRealms, // Auth: OAuthUrl: oauthURL, diff --git a/docs/reference/filters.md b/docs/reference/filters.md index 9a1183a3a3..9a4f39837a 100644 --- a/docs/reference/filters.md +++ b/docs/reference/filters.md @@ -1050,7 +1050,7 @@ For the client based metrics, additional flags need to be specified. |--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `api-usage-monitoring-realm-keys` | Name of the property in the JWT JSON body that contains the name of the _realm_. | | `api-usage-monitoring-client-keys` | Name of the property in the JWT JSON body that contains the name of the _client_. | -| `api-usage-monitoring-default-client-tracking-pattern` | Optional. Default is empty. Allows to deploy Skipper with a default client tracking pattern. When `apiUsageMonitoring` configuration does not specify one, this default is used instead of disabling the client metrics. | +| `api-usage-monitoring-realms` | List of _realms_ to be monitored. Defaults to 'services'. | NOTE: Make sure to activate the metrics flavour proper to your environment using the `metrics-flavour` flag in order to get those metrics. @@ -1058,7 +1058,7 @@ flag in order to get those metrics. Example: ```bash -skipper -metrics-flavour prometheus -enable-api-usage-monitoring -api-usage-monitoring-realm-keys="realm" -api-usage-monitoring-client-keys="managed-id" +skipper -metrics-flavour prometheus -enable-api-usage-monitoring -api-usage-monitoring-realm-keys="realm" -api-usage-monitoring-client-keys="managed-id" api-usage-monitoring-realms="services,users" ``` The structure of the metrics is all of those elements, separated by `.` dots: @@ -1085,7 +1085,7 @@ Example: ``` + Realm | -apiUsageMonitoring.custom.orders-backend.orders-api.GET.foo/orders/:order-id.*.*.http_count +apiUsageMonitoring.custom.orders-backend.orders-api.GET.foo/orders/{order-id}.*.*.http_count | | | + Metric Name + Client @@ -1155,6 +1155,7 @@ api-usage-monitoring-configuration: path_templates: description: Endpoints to be monitored. type: array + minLength: 1 items: type: string description: > @@ -1163,23 +1164,19 @@ api-usage-monitoring-configuration: example: /orders/{order-id} client_tracking_pattern: description: > - The pattern that the combination `realm.client` must match in order for the client - based metrics to be tracked, in form of a regular expression. + The pattern that matches client id in form of a regular expression. - By default (if undefined), it is set to `services\\..*`. + By default (if undefined), it is set to `.*`. An empty string disables the client metrics completely. - - IMPORTANT: Avoid patterns that would match too many different values like `.*` or `users\\..*`. Too - many different metric keys would badly affect the performances of the metric systems (e.g.: Prometheus). type: string examples: all_services: - summary: All services are tracked (clients of the realm `services`). - value: "services\\..*" + summary: All services are tracked (for all activated realms). + value: ".*" just_some_services: - summary: Only services `orders` and `shipment` are tracked. - value: "services\\.(orders|shipment)" + summary: Only services `orders-service` and `shipment-service` are tracked. + value: "(orders\-service|shipment\-service)" ``` Configuration Example: @@ -1194,7 +1191,7 @@ apiUsageMonitoring(` "foo/orders/:order-id", "foo/orders/:order-id/order_item/{order-item-id}" ], - "client_tracking_pattern": "users\\.(joe|sabine)" + "client_tracking_pattern": "(joe|sabine)" }`,`{ "application_id": "my-app", "api_id": "customers-api", @@ -1209,13 +1206,13 @@ apiUsageMonitoring(` Based on the previous configuration, here is an example of a counter metric. ``` -apiUsageMonitoring.custom.my-app.orders-api.GET.foo/orders/:order-id.*.*.http_count +apiUsageMonitoring.custom.my-app.orders-api.GET.foo/orders/{order-id}.*.*.http_count ``` Here is the _Prometheus_ query to obtain it. ``` -sum(rate(skipper_custom_total{key="apiUsageMonitoring.custom.my-app.orders-api.GET.foo/orders/:order-id.*.*.http_count"}[60s])) by (key) +sum(rate(skipper_custom_total{key="apiUsageMonitoring.custom.my-app.orders-api.GET.foo/orders/{order-id}.*.*.http_count"}[60s])) by (key) ``` Here is an example of a histogram metric. @@ -1228,19 +1225,4 @@ Here is the _Prometheus_ query to obtain it. ``` histogram_quantile(0.5, sum(rate(skipper_custom_duration_seconds_bucket{key="apiUsageMonitoring.custom.my-app.orders-api.POST.foo/orders.*.*.latency"}[60s])) by (le, key)) -``` - -NOTE: Non configured paths will be tracked with `` application ID, API ID -and path template. - -``` -apiUsageMonitoring.custom...GET..*.*.http_count -``` - -However, if all `application_id`s of your configuration refer to the same application, -the filter assume that also non configured paths will be directed to this application. -E.g.: - -``` -apiUsageMonitoring.custom.my-app..GET..*.*.http_count -``` +``` \ No newline at end of file diff --git a/filters/apiusagemonitoring/doc.go b/filters/apiusagemonitoring/doc.go index aca8ed8ac2..931ea42ed4 100644 --- a/filters/apiusagemonitoring/doc.go +++ b/filters/apiusagemonitoring/doc.go @@ -35,8 +35,8 @@ Command line example for executing locally: -routes-file "$HOME/temp/test.eskip" \ -enable-api-usage-monitoring \ -api-usage-monitoring-realm-keys="https://identity.zalando.com/realm" \ - -api-usage-monitoring-client-keys="https://identity.zalando.com/managed-id" \ - -api-usage-monitoring-default-client-tracking-pattern="services\\..*" \ + -api-usage-monitoring-client-keys="https://identity.zalando.com/managed-id,sub" \ + -api-usage-monitoring-realms="services" \ -metrics-flavour prometheus \ -histogram-metric-buckets=".01,.025,.05,.075,.1,.2,.3,.4,.5,.75,1,2,3,4,5,7,10,15,20,30,60,120,300,600" \ -application-log-level=DEBUG diff --git a/filters/apiusagemonitoring/filter.go b/filters/apiusagemonitoring/filter.go index a3483ce1c3..fcce387c51 100644 --- a/filters/apiusagemonitoring/filter.go +++ b/filters/apiusagemonitoring/filter.go @@ -104,45 +104,49 @@ func (f *apiUsageMonitoringFilter) getClientMetricsNames(realmClientKey string, return prefixes } -// getRealmClientKey generates the proper . part of the -// client metrics name. -func (f *apiUsageMonitoringFilter) getRealmClientKey(r *http.Request, path *pathInfo) string { - const ( - unknownRealmClient = unknownElementPlaceholder + "." + unknownElementPlaceholder - unknownClientAfterKnownRealm = "." + unknownElementPlaceholder - ) +const format = "%s.%s" - // no JWT ==> . +// getRealmClientKey generates the proper . part of the client metrics name. +func (f *apiUsageMonitoringFilter) getRealmClientKey(r *http.Request, path *pathInfo) string { + // no JWT ==> {unknown}.{unknown} jwt := parseJwtBody(r) if jwt == nil { - return unknownRealmClient + return fmt.Sprintf(format, unknown, unknown) } - // no realm in JWT ==> . + // no realm in JWT ==> {unknown}.{unknown} realm, ok := jwt.getOneOfString(f.Spec.realmKeys) if !ok { - return unknownRealmClient + return fmt.Sprintf(format, unknown, unknown) } - // no matcher ==> realm. - if path.ClientTracking.ClientTrackingMatcher == nil { - return realm + unknownClientAfterKnownRealm + // realm is not one of the realms to be tracked ==> realm.{all} + if !contains(path.ClientTracking.RealmsToTrack, realm) { + return fmt.Sprintf(format, realm, "{all}") } - // no client in JWT ==> realm. + // no client in JWT ==> realm.{unknown} client, ok := jwt.getOneOfString(f.Spec.clientKeys) if !ok { - return realm + unknownClientAfterKnownRealm + return fmt.Sprintf(format, realm, unknown) } - // if `realm.client` does not match ==> realm. - realmAndClient := realm + "." + client - if !path.ClientTracking.ClientTrackingMatcher.MatchString(realmAndClient) { - return realm + unknownClientAfterKnownRealm + // if client does not match ==> realm.{no-match} + if !path.ClientTracking.ClientTrackingMatcher.MatchString(client) { + return fmt.Sprintf(format, realm, noMatch) } - // all matched ==> realm.client - return realmAndClient + // all matched ==> realm.client + return fmt.Sprintf(format, realm, client) +} + +func contains(strings []string, s string) bool { + for _, v := range strings { + if s == v { + return true + } + } + return false } // resolvePath tries to match the request's path with one of the configured path template. @@ -163,7 +167,7 @@ func getEndpointMetricsNames(req *http.Request, path *pathInfo) *endpointMetricN methodIndex, ok := methodToIndex[method] if !ok { methodIndex = methodIndexUnknown - method = unknownElementPlaceholder + method = unknown } if p := path.metricPrefixesPerMethod[methodIndex]; p != nil { diff --git a/filters/apiusagemonitoring/filter_clientmetrics_test.go b/filters/apiusagemonitoring/filter_clientmetrics_test.go index 43e3b6948c..cc2fb38449 100644 --- a/filters/apiusagemonitoring/filter_clientmetrics_test.go +++ b/filters/apiusagemonitoring/filter_clientmetrics_test.go @@ -15,11 +15,11 @@ type clientMetricsTest struct { expectedEndpointMetricPrefix string expectedClientMetricPrefix string - defaultClientTrackingPattern string + realms string } var ( - clientTrackingPatternJustSomeUsers = s(`users\.(joe|sabine)`) + clientTrackingPatternJustSomeUsers = s(`(joe|sabine)`) headerUsersJoe = http.Header{ authorizationHeaderName: { "Bearer " + buildFakeJwtWithBody(map[string]interface{}{ @@ -52,8 +52,9 @@ func Test_Filter_ClientMetrics_NoMatchingPath_RealmIsKnown(t *testing.T) { clientTrackingPattern: s(".*"), header: headerUsersJoe, url: "https://www.example.com/non/configured/path/template", - expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app..GET.{no-match}.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app..*.*.users..", + expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.{unknown}.GET.{no-match}.*.*.", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.{unknown}.*.*.users.{all}.", + realms: "services", }) } @@ -64,8 +65,8 @@ func Test_Filter_ClientMetrics_NoMatchingPath_RealmIsUnknown(t *testing.T) { clientTrackingPattern: s(".*"), header: headerUsersJoe, url: "https://www.example.com/non/configured/path/template", - expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app..GET.{no-match}.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app..*.*...", + expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.{unknown}.GET.{no-match}.*.*.", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.{unknown}.*.*.{unknown}.{unknown}.", }) } @@ -77,6 +78,7 @@ func Test_Filter_ClientMetrics_MatchAll(t *testing.T) { header: headerUsersJoe, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.users.joe.", + realms: "users", }) } @@ -95,6 +97,7 @@ func Test_Filter_ClientMetrics_MatchOneOfClientKeyName(t *testing.T) { }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.services.payments.", + realms: "services", }) } @@ -102,6 +105,7 @@ func Test_Filter_ClientMetrics_MatchOneOfClientKeyName_UseFirstMatching(t *testi testClientMetrics(t, clientMetricsTest{ realmKeyName: "realm", clientKeyName: "client,sub", + realms: "services", clientTrackingPattern: s(".*"), header: http.Header{ authorizationHeaderName: { @@ -121,6 +125,7 @@ func Test_Filter_ClientMetrics_Realm1User1(t *testing.T) { testClientMetrics(t, clientMetricsTest{ realmKeyName: "realm", clientKeyName: "client", + realms: "users", clientTrackingPattern: clientTrackingPatternJustSomeUsers, header: headerUsersJoe, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", @@ -132,6 +137,7 @@ func Test_Filter_ClientMetrics_Realm1User0(t *testing.T) { testClientMetrics(t, clientMetricsTest{ realmKeyName: "realm", clientKeyName: "client", + realms: "users", clientTrackingPattern: clientTrackingPatternJustSomeUsers, header: http.Header{ authorizationHeaderName: { @@ -142,7 +148,7 @@ func Test_Filter_ClientMetrics_Realm1User0(t *testing.T) { }, }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.users..", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.users.{no-match}.", }) } @@ -150,6 +156,7 @@ func Test_Filter_ClientMetrics_Realm0User1(t *testing.T) { testClientMetrics(t, clientMetricsTest{ realmKeyName: "realm", clientKeyName: "client", + realms: "services", clientTrackingPattern: clientTrackingPatternJustSomeUsers, header: http.Header{ authorizationHeaderName: { @@ -160,7 +167,7 @@ func Test_Filter_ClientMetrics_Realm0User1(t *testing.T) { }, }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.nobodies..", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.nobodies.{all}.", }) } @@ -168,6 +175,7 @@ func Test_Filter_ClientMetrics_Realm0User0(t *testing.T) { testClientMetrics(t, clientMetricsTest{ realmKeyName: "realm", clientKeyName: "client", + realms: "services", clientTrackingPattern: clientTrackingPatternJustSomeUsers, header: http.Header{ authorizationHeaderName: { @@ -178,7 +186,7 @@ func Test_Filter_ClientMetrics_Realm0User0(t *testing.T) { }, }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.nobodies..", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.nobodies.{all}.", }) } @@ -186,6 +194,7 @@ func Test_Filter_ClientMetrics_AuthDoesNotHaveBearerPrefix(t *testing.T) { testClientMetrics(t, clientMetricsTest{ realmKeyName: "realm", clientKeyName: "client", + realms: "services", clientTrackingPattern: clientTrackingPatternJustSomeUsers, header: http.Header{ authorizationHeaderName: { @@ -196,7 +205,7 @@ func Test_Filter_ClientMetrics_AuthDoesNotHaveBearerPrefix(t *testing.T) { }, }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*...", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.{unknown}.{unknown}.", }) } @@ -205,11 +214,11 @@ func Test_Filter_ClientMetrics_NoAuthHeader(t *testing.T) { realmKeyName: "realm", clientKeyName: "client", clientTrackingPattern: clientTrackingPatternJustSomeUsers, - header: http.Header{ + header: http.Header{ /* no Authorization header */ }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*...", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.{unknown}.{unknown}.", }) } @@ -224,7 +233,7 @@ func Test_Filter_ClientMetrics_JWTIsNot3DotSeparatedString(t *testing.T) { }, }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*...", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.{unknown}.{unknown}.", }) } @@ -239,7 +248,7 @@ func Test_Filter_ClientMetrics_JWTIsNotBase64Encoded(t *testing.T) { }, }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*...", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.{unknown}.{unknown}.", }) } @@ -255,7 +264,7 @@ func Test_Filter_ClientMetrics_JWTBodyIsNoJSON(t *testing.T) { }, }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*...", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.{unknown}.{unknown}.", }) } @@ -273,7 +282,7 @@ func Test_Filter_ClientMetrics_JWTBodyHasNoRealm(t *testing.T) { }, }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*...", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.{unknown}.{unknown}.", }) } @@ -281,6 +290,7 @@ func Test_Filter_ClientMetrics_JWTBodyHasNoClient_ShouldTrackRealm(t *testing.T) testClientMetrics(t, clientMetricsTest{ realmKeyName: "realm", clientKeyName: "client", + realms: "users", clientTrackingPattern: clientTrackingPatternJustSomeUsers, header: http.Header{ authorizationHeaderName: { @@ -291,7 +301,7 @@ func Test_Filter_ClientMetrics_JWTBodyHasNoClient_ShouldTrackRealm(t *testing.T) }, }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.users..", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.users.{unknown}.", }) } @@ -319,10 +329,10 @@ func Test_Filter_ClientMetrics_NoFlagClientKeyName(t *testing.T) { func Test_Filter_ClientMetrics_DefaultClientTrackingPattern_NoClientTrackingPatternInRouteFilterJSON_User(t *testing.T) { testClientMetrics(t, clientMetricsTest{ - realmKeyName: "realm", - clientKeyName: "client", - defaultClientTrackingPattern: "services\\..*", - clientTrackingPattern: nil, // no client tracking in route's filter configuration + realmKeyName: "realm", + clientKeyName: "client", + realms: "services", + clientTrackingPattern: nil, // no client tracking in route's filter configuration header: http.Header{ authorizationHeaderName: { "Bearer " + buildFakeJwtWithBody(map[string]interface{}{ @@ -332,16 +342,16 @@ func Test_Filter_ClientMetrics_DefaultClientTrackingPattern_NoClientTrackingPatt }, }, expectedEndpointMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.GET.foo/orders.*.*.", - expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.users..", + expectedClientMetricPrefix: "apiUsageMonitoring.custom.my_app.my_api.*.*.users.{all}.", }) } func Test_Filter_ClientMetrics_DefaultClientTrackingPattern_NoClientTrackingPatternInRouteFilterJSON_Service(t *testing.T) { testClientMetrics(t, clientMetricsTest{ - realmKeyName: "realm", - clientKeyName: "client", - defaultClientTrackingPattern: "services\\..*", - clientTrackingPattern: nil, // no client tracking in route's filter configuration + realmKeyName: "realm", + clientKeyName: "client", + realms: "services", + clientTrackingPattern: nil, // no client tracking in route's filter configuration header: http.Header{ authorizationHeaderName: { "Bearer " + buildFakeJwtWithBody(map[string]interface{}{ @@ -357,10 +367,10 @@ func Test_Filter_ClientMetrics_DefaultClientTrackingPattern_NoClientTrackingPatt func Test_Filter_ClientMetrics_EmptyClientTrackingPatternInRouteFilterJSON(t *testing.T) { testClientMetrics(t, clientMetricsTest{ - realmKeyName: "realm", - clientKeyName: "client", - defaultClientTrackingPattern: "services\\..*", - clientTrackingPattern: s(""), // no client tracking in route's filter configuration + realmKeyName: "realm", + clientKeyName: "client", + realms: "services", + clientTrackingPattern: s(""), // no client tracking in route's filter configuration header: http.Header{ authorizationHeaderName: { "Bearer " + buildFakeJwtWithBody(map[string]interface{}{ diff --git a/filters/apiusagemonitoring/filter_endpointmetrics_test.go b/filters/apiusagemonitoring/filter_endpointmetrics_test.go index bf0b6f9f98..95605c1cb6 100644 --- a/filters/apiusagemonitoring/filter_endpointmetrics_test.go +++ b/filters/apiusagemonitoring/filter_endpointmetrics_test.go @@ -20,7 +20,7 @@ func Test_Filter_NoPathTemplate(t *testing.T) { "https://www.example.org/a/b/c", 299, func(t *testing.T, pass int, m *metricstest.MockMetrics) { - pre := "apiUsageMonitoring.custom.my_app..GET.{no-match}.*.*." + pre := "apiUsageMonitoring.custom.my_app.{unknown}.GET.{no-match}.*.*." // no path matching: tracked as unknown m.WithCounters(func(counters map[string]int64) { assert.Equal(t, @@ -218,7 +218,7 @@ func Test_Filter_NonConfiguredPathTrackedUnderNoMatch(t *testing.T) { "https://www.example.org/lapin/malin", 200, func(t *testing.T, pass int, m *metricstest.MockMetrics) { - pre := "apiUsageMonitoring.custom.my_app..GET.{no-match}.*.*." + pre := "apiUsageMonitoring.custom.my_app.{unknown}.GET.{no-match}.*.*." m.WithCounters(func(counters map[string]int64) { assert.Equal(t, map[string]int64{ @@ -248,8 +248,8 @@ func Test_Filter_AllHttpMethodsAreSupported(t *testing.T) { {http.MethodConnect, "CONNECT"}, {http.MethodOptions, "OPTIONS"}, {http.MethodTrace, "TRACE"}, - {"", ""}, - {"foo", ""}, + {"", "{unknown}"}, + {"foo", "{unknown}"}, } { t.Run(testCase.method, func(t *testing.T) { testWithFilter( @@ -260,7 +260,7 @@ func Test_Filter_AllHttpMethodsAreSupported(t *testing.T) { 200, func(t *testing.T, pass int, m *metricstest.MockMetrics) { pre := fmt.Sprintf( - "apiUsageMonitoring.custom.my_app..%s.{no-match}.*.*.", + "apiUsageMonitoring.custom.my_app.{unknown}.%s.{no-match}.*.*.", testCase.expectedMethodInMetric) m.WithCounters(func(counters map[string]int64) { assert.Equal(t, diff --git a/filters/apiusagemonitoring/pathinfo.go b/filters/apiusagemonitoring/pathinfo.go index 42e7f023e1..16cd6aa923 100644 --- a/filters/apiusagemonitoring/pathinfo.go +++ b/filters/apiusagemonitoring/pathinfo.go @@ -69,8 +69,8 @@ const ( methodIndexOptions // OPTIONS methodIndexTrace // TRACE - methodIndexUnknown // Value when the HTTP Method is not in the known list - methodIndexLength // Gives the constant size of the `metricPrefixesPerMethod` array. + methodIndexUnknown // Value when the HTTP Method is not in the known list + methodIndexLength // Gives the constant size of the `metricPrefixesPerMethod` array. ) var ( @@ -89,4 +89,5 @@ var ( type clientTrackingInfo struct { ClientTrackingMatcher *regexp.Regexp + RealmsToTrack []string } diff --git a/filters/apiusagemonitoring/spec.go b/filters/apiusagemonitoring/spec.go index 95cbc0f522..21d18d0b8a 100644 --- a/filters/apiusagemonitoring/spec.go +++ b/filters/apiusagemonitoring/spec.go @@ -13,8 +13,8 @@ import ( const ( Name = "apiUsageMonitoring" - unknownElementPlaceholder = "" - noMatchPath = "{no-match}" + unknown = "{unknown}" + noMatch = "{no-match}" regexUrlPathPart = `.+` regexOptionalSlashes = `\/*` @@ -32,14 +32,14 @@ func NewApiUsageMonitoring( enabled bool, realmKeys string, clientKeys string, - defaultClientTrackingPattern string, + realms string, ) filters.Spec { if !enabled { - log.Debugf("Filter %q is not enabled. Spec returns `noop` filters.", Name) + log.Debugf("filter %q is not enabled. spec returns `noop` filters.", Name) return &noopSpec{&noopFilter{}} } - // Parse realm keys comma separated list + // parse realm keys comma separated list var realmKeyList []string for _, key := range strings.Split(realmKeys, ",") { strippedKey := strings.TrimSpace(key) @@ -47,7 +47,7 @@ func NewApiUsageMonitoring( realmKeyList = append(realmKeyList, strippedKey) } } - // Parse client keys comma separated list + // parse client keys comma separated list var clientKeyList []string for _, key := range strings.Split(clientKeys, ",") { strippedKey := strings.TrimSpace(key) @@ -55,6 +55,14 @@ func NewApiUsageMonitoring( clientKeyList = append(clientKeyList, strippedKey) } } + // parse comma separated list of realms to monitor + var realmValues []string + for _, key := range strings.Split(realms, ",") { + strippedKey := strings.TrimSpace(key) + if strippedKey != "" { + realmValues = append(realmValues, strippedKey) + } + } // Create the filter Spec var unknownPathClientTracking *clientTrackingInfo = nil // client metrics feature is disabled @@ -64,18 +72,18 @@ func NewApiUsageMonitoring( } } unknownPath := newPathInfo( - unknownElementPlaceholder, - unknownElementPlaceholder, - noMatchPath, + unknown, + unknown, + noMatch, unknownPathClientTracking, ) spec := &apiUsageMonitoringSpec{ - realmKeys: realmKeyList, - clientKeys: clientKeyList, - unknownPath: unknownPath, - defaultClientTrackingPattern: defaultClientTrackingPattern, + realmKeys: realmKeyList, + clientKeys: clientKeyList, + unknownPath: unknownPath, + realms: realmValues, } - log.Debugf("Created filter spec: %+v", spec) + log.Debugf("created filter spec: %+v", spec) return spec } @@ -88,10 +96,10 @@ type apiConfig struct { } type apiUsageMonitoringSpec struct { - realmKeys []string - clientKeys []string - unknownPath *pathInfo - defaultClientTrackingPattern string + realmKeys []string + clientKeys []string + realms []string + unknownPath *pathInfo } func (s *apiUsageMonitoringSpec) Name() string { @@ -106,12 +114,10 @@ func (s *apiUsageMonitoringSpec) CreateFilter(args []interface{}) (filter filter return nil, fmt.Errorf("no valid configurations") } - unknownPath := s.buildUnknownPathInfo(apis) - filter = &apiUsageMonitoringFilter{ Spec: s, Paths: paths, - UnknownPath: unknownPath, + UnknownPath: s.buildUnknownPathInfo(paths), } return } @@ -125,7 +131,7 @@ func (s *apiUsageMonitoringSpec) parseJsonConfiguration(args []interface{}) []*a continue } config := &apiConfig{ - ClientTrackingPattern: s.defaultClientTrackingPattern, + ClientTrackingPattern: ".*", // track all clients per default } decoder := json.NewDecoder(strings.NewReader(rawJsonConfiguration)) decoder.DisallowUnknownFields() @@ -139,13 +145,13 @@ func (s *apiUsageMonitoringSpec) parseJsonConfiguration(args []interface{}) []*a return apis } -func (s *apiUsageMonitoringSpec) buildUnknownPathInfo(apis []*apiConfig) *pathInfo { +func (s *apiUsageMonitoringSpec) buildUnknownPathInfo(paths []*pathInfo) *pathInfo { var applicationId *string - for _, api := range apis { - if applicationId != nil && *applicationId != api.ApplicationId { + for _, path := range paths { + if applicationId != nil && *applicationId != path.ApplicationId { return s.unknownPath } - applicationId = &api.ApplicationId + applicationId = &path.ApplicationId } if applicationId != nil && *applicationId != "" { @@ -181,7 +187,7 @@ func (s *apiUsageMonitoringSpec) buildPathInfoListFromConfiguration(apis []*apiC continue } - clientTrackingInfo := s.buildClientTrackingInfo(apiIndex, api) + clientTrackingInfo := s.buildClientTrackingInfo(apiIndex, api, s.realms) for templateIndex, template := range api.PathTemplates { @@ -238,7 +244,7 @@ func (s *apiUsageMonitoringSpec) buildPathInfoListFromConfiguration(apis []*apiC return paths } -func (s *apiUsageMonitoringSpec) buildClientTrackingInfo(apiIndex int, api *apiConfig) *clientTrackingInfo { +func (s *apiUsageMonitoringSpec) buildClientTrackingInfo(apiIndex int, api *apiConfig, realmsToTrack []string) *clientTrackingInfo { if len(s.realmKeys) == 0 { log.Infof( `args[%d]: skipper wide configuration "api-usage-monitoring-realm-keys" not provided, not tracking client metrics`, @@ -268,6 +274,7 @@ func (s *apiUsageMonitoringSpec) buildClientTrackingInfo(apiIndex int, api *apiC return &clientTrackingInfo{ ClientTrackingMatcher: clientTrackingMatcher, + RealmsToTrack: realmsToTrack, } } diff --git a/filters/apiusagemonitoring/spec_test.go b/filters/apiusagemonitoring/spec_test.go index 77b1115fe1..8c9bfff8ee 100644 --- a/filters/apiusagemonitoring/spec_test.go +++ b/filters/apiusagemonitoring/spec_test.go @@ -45,7 +45,7 @@ func unknownPath(applicationId string) pathMatcher { return pathMatcher{ PathTemplate: "{no-match}", ApplicationId: applicationId, - ApiId: "", + ApiId: "{unknown}", Matcher: nil, } } @@ -391,7 +391,7 @@ func Test_CreateFilter_FullConfigWithApisWithoutPaths(t *testing.T) { Matcher: matcher("^\\/*foo\\/orders\\/*$"), }, }) - assertPath(t, filter.UnknownPath, unknownPath("")) + assertPath(t, filter.UnknownPath, unknownPath("my_order_app")) }) } diff --git a/filters/apiusagemonitoring/testutils_test.go b/filters/apiusagemonitoring/testutils_test.go index fc953214bc..0e3701ce79 100644 --- a/filters/apiusagemonitoring/testutils_test.go +++ b/filters/apiusagemonitoring/testutils_test.go @@ -190,7 +190,7 @@ func testClientMetrics(t *testing.T, testCase clientMetricsTest) { t.FailNow() } args := []interface{}{string(js)} - spec := NewApiUsageMonitoring(true, testCase.realmKeyName, testCase.clientKeyName, testCase.defaultClientTrackingPattern) + spec := NewApiUsageMonitoring(true, testCase.realmKeyName, testCase.clientKeyName, testCase.realms) return spec.CreateFilter(args) }, } diff --git a/skipper.go b/skipper.go index 891c0574ce..8656357472 100644 --- a/skipper.go +++ b/skipper.go @@ -492,9 +492,10 @@ type Options struct { // API Monitoring feature is active (feature toggle) ApiUsageMonitoringEnable bool - ApiUsageMonitoringRealmKey string - ApiUsageMonitoringClientIdKeyName string + ApiUsageMonitoringRealmKeys string + ApiUsageMonitoringClientKeys string ApiUsageMonitoringDefaultClientTrackingPattern string + ApiUsageMonitoringRealms string // WebhookTimeout sets timeout duration while calling a custom webhook auth service WebhookTimeout time.Duration @@ -761,9 +762,9 @@ func Run(o Options) error { auth.NewOAuthOidcAllClaims(o.OIDCSecretsFile), apiusagemonitoring.NewApiUsageMonitoring( o.ApiUsageMonitoringEnable, - o.ApiUsageMonitoringRealmKey, - o.ApiUsageMonitoringClientIdKeyName, - o.ApiUsageMonitoringDefaultClientTrackingPattern, + o.ApiUsageMonitoringRealmKeys, + o.ApiUsageMonitoringClientKeys, + o.ApiUsageMonitoringRealms, ), )