From 72b025915124d127a9e22aa43dcb438ea469343e Mon Sep 17 00:00:00 2001 From: Giancarlo Rubio Date: Wed, 13 Sep 2017 08:35:05 +0200 Subject: [PATCH 1/2] support apdex metrics --- .gitignore | 3 ++ controllers/nginx/configuration.md | 1 + controllers/nginx/pkg/config/config.go | 41 +++++++++++------- controllers/nginx/pkg/metric/collector/vts.go | 18 ++++---- .../rootfs/etc/nginx/template/nginx.tmpl | 6 ++- controllers/nginx/test/data/config.json | 4 +- .../ingress/annotations/vtsfilterkey/main.go | 42 +++++++++++++++++++ core/pkg/ingress/controller/annotations.go | 2 + core/pkg/ingress/types.go | 4 ++ 9 files changed, 95 insertions(+), 26 deletions(-) create mode 100644 core/pkg/ingress/annotations/vtsfilterkey/main.go diff --git a/.gitignore b/.gitignore index 562090a3071f..32947c45be60 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ .project .settings/** +# visual studio code +.vscode/* + # Emacs save files *~ \#*\# diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md index 07c2285b05ae..315d0e0a97bb 100644 --- a/controllers/nginx/configuration.md +++ b/controllers/nginx/configuration.md @@ -537,6 +537,7 @@ The following table shows the options, the default value and a description. |variables-hash-bucket-size|64| |variables-hash-max-size|2048| |vts-status-zone-size|10m| +|vts-default-filter-key|$geoip_country_code country::*| |whitelist-source-range|permit all| |worker-processes|number of CPUs| |limit-conn-zone-variable|$binary_remote_addr| diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go index 4bb53b0db7e5..68d6af068e66 100644 --- a/controllers/nginx/pkg/config/config.go +++ b/controllers/nginx/pkg/config/config.go @@ -146,8 +146,21 @@ type Configuration struct { // By default this is disabled EnableVtsStatus bool `json:"enable-vts-status,omitempty"` + // Vts config on http level + // Description: Sets parameters for a shared memory zone that will keep states for various keys. The cache is shared between all worker processe + // https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_zone + // Default value is 10m VtsStatusZoneSize string `json:"vts-status-zone-size,omitempty"` + // Vts config on http level + // Description: Enables the keys by user defined variable. The key is a key string to calculate traffic. + // The name is a group string to calculate traffic. The key and name can contain variables such as $host, + // $server_name. The name's group belongs to filterZones if specified. The key's group belongs to serverZones + // if not specified second argument name. The example with geoip module is as follows: + // https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_filter_by_set_key + // Default value is $geoip_country_code country::* + VtsDefaultFilterKey string `json:"vts-default-filter-key,omitempty"` + // RetryNonIdempotent since 1.9.13 NGINX will not retry non-idempotent requests (POST, LOCK, PATCH) // in case of an error. The previous behavior can be restored using the value true RetryNonIdempotent bool `json:"retry-non-idempotent"` @@ -414,21 +427,21 @@ func NewDefault() Configuration { UseHTTP2: true, ProxyStreamTimeout: "600s", Backend: defaults.Backend{ - ProxyBodySize: bodySize, - ProxyConnectTimeout: 5, - ProxyReadTimeout: 60, - ProxySendTimeout: 60, - ProxyBufferSize: "4k", - ProxyCookieDomain: "off", - ProxyCookiePath: "off", - ProxyNextUpstream: "error timeout invalid_header http_502 http_503 http_504", + ProxyBodySize: bodySize, + ProxyConnectTimeout: 5, + ProxyReadTimeout: 60, + ProxySendTimeout: 60, + ProxyBufferSize: "4k", + ProxyCookieDomain: "off", + ProxyCookiePath: "off", + ProxyNextUpstream: "error timeout invalid_header http_502 http_503 http_504", ProxyRequestBuffering: "on", - SSLRedirect: true, - CustomHTTPErrors: []int{}, - WhitelistSourceRange: []string{}, - SkipAccessLogURLs: []string{}, - LimitRate: 0, - LimitRateAfter: 0, + SSLRedirect: true, + CustomHTTPErrors: []int{}, + WhitelistSourceRange: []string{}, + SkipAccessLogURLs: []string{}, + LimitRate: 0, + LimitRateAfter: 0, }, UpstreamKeepaliveConnections: 32, LimitConnZoneVariable: defaultLimitConnZoneVariable, diff --git a/controllers/nginx/pkg/metric/collector/vts.go b/controllers/nginx/pkg/metric/collector/vts.go index f57cc4dd1760..a0ed03eb7c85 100644 --- a/controllers/nginx/pkg/metric/collector/vts.go +++ b/controllers/nginx/pkg/metric/collector/vts.go @@ -96,17 +96,17 @@ func NewNGINXVTSCollector(watchNamespace, ingressClass string, port int, path st filterZoneBytes: prometheus.NewDesc( prometheus.BuildFQName(ns, "", "filterzone_bytes_total"), "Nginx bytes count", - []string{"ingress_class", "namespace", "server_zone", "country", "direction"}, nil), + []string{"ingress_class", "namespace", "server_zone", "key", "direction"}, nil), filterZoneResponses: prometheus.NewDesc( prometheus.BuildFQName(ns, "", "filterzone_responses_total"), "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", - []string{"ingress_class", "namespace", "server_zone", "country", "status_code"}, nil), + []string{"ingress_class", "namespace", "server_zone", "key", "status_code"}, nil), filterZoneCache: prometheus.NewDesc( prometheus.BuildFQName(ns, "", "filterzone_cache_total"), "Nginx cache count", - []string{"ingress_class", "namespace", "server_zone", "country", "type"}, nil), + []string{"ingress_class", "namespace", "server_zone", "key", "type"}, nil), upstreamBackup: prometheus.NewDesc( prometheus.BuildFQName(ns, "", "upstream_backup"), @@ -246,15 +246,15 @@ func (p vtsCollector) scrapeVts(ch chan<- prometheus.Metric) { prometheus.CounterValue, zone.OutBytes, p.ingressClass, p.watchNamespace, name, "out") } - for serverZone, countries := range nginxMetrics.FilterZones { - for country, zone := range countries { - reflectMetrics(&zone.Responses, p.data.filterZoneResponses, ch, p.ingressClass, p.watchNamespace, serverZone, country) - reflectMetrics(&zone.Cache, p.data.filterZoneCache, ch, p.ingressClass, p.watchNamespace, serverZone, country) + for serverZone, keys := range nginxMetrics.FilterZones { + for name, zone := range keys { + reflectMetrics(&zone.Responses, p.data.filterZoneResponses, ch, p.ingressClass, p.watchNamespace, serverZone, name) + reflectMetrics(&zone.Cache, p.data.filterZoneCache, ch, p.ingressClass, p.watchNamespace, serverZone, name) ch <- prometheus.MustNewConstMetric(p.data.filterZoneBytes, - prometheus.CounterValue, zone.InBytes, p.ingressClass, p.watchNamespace, serverZone, country, "in") + prometheus.CounterValue, zone.InBytes, p.ingressClass, p.watchNamespace, serverZone, name, "in") ch <- prometheus.MustNewConstMetric(p.data.filterZoneBytes, - prometheus.CounterValue, zone.OutBytes, p.ingressClass, p.watchNamespace, serverZone, country, "out") + prometheus.CounterValue, zone.OutBytes, p.ingressClass, p.watchNamespace, serverZone, name, "out") } } } diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index 5349f1fc1d20..a878d2c5c69e 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -46,7 +46,7 @@ http { {{ if $cfg.EnableVtsStatus }} vhost_traffic_status_zone shared:vhost_traffic_status:{{ $cfg.VtsStatusZoneSize }}; - vhost_traffic_status_filter_by_set_key $geoip_country_code country::*; + vhost_traffic_status_filter_by_set_key {{ if not $cfg.VtsDefaultFilterKey }}$geoip_country_code country::*{{ else }} {{ $cfg.VtsDefaultFilterKey }}{{ end }}; {{ end }} sendfile on; @@ -578,7 +578,6 @@ stream { more_set_headers "Strict-Transport-Security: max-age={{ $all.Cfg.HSTSMaxAge }}{{ if $all.Cfg.HSTSIncludeSubdomains }}; includeSubDomains{{ end }};{{ if $all.Cfg.HSTSPreload }} preload{{ end }}"; {{ end }} - {{ if $all.Cfg.EnableVtsStatus }}vhost_traffic_status_filter_by_set_key $geoip_country_code country::$server_name;{{ end }} {{ if not (empty $server.CertificateAuth.CAFileName) }} # PEM sha: {{ $server.CertificateAuth.PemSHA }} @@ -630,6 +629,9 @@ stream { {{ end }} location {{ $path }} { + + {{ if $all.Cfg.EnableVtsStatus }}{{ if $location.VtsFilterKey }} vhost_traffic_status_filter_by_set_key {{ $location.VtsFilterKey }};{{ end }}{{ end }} + set $proxy_upstream_name "{{ buildUpstreamName $server.Hostname $all.Backends $location }}"; {{ $ing := (getIngressInformation $location.Ingress $path) }} diff --git a/controllers/nginx/test/data/config.json b/controllers/nginx/test/data/config.json index c339b7759a68..1ac608b78991 100644 --- a/controllers/nginx/test/data/config.json +++ b/controllers/nginx/test/data/config.json @@ -110,7 +110,9 @@ "keyFilename": "", "caFilename": "", "pemSha": "" - } + }, + "vtsDefaultFilterKey": "$uri $server_name" + }, { "path": "/", "isDefBackend": true, diff --git a/core/pkg/ingress/annotations/vtsfilterkey/main.go b/core/pkg/ingress/annotations/vtsfilterkey/main.go new file mode 100644 index 000000000000..401e9b7e89f6 --- /dev/null +++ b/core/pkg/ingress/annotations/vtsfilterkey/main.go @@ -0,0 +1,42 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtsfilterkey + +import ( + extensions "k8s.io/api/extensions/v1beta1" + + "k8s.io/ingress/core/pkg/ingress/annotations/parser" +) + +const ( + annotation = "ingress.kubernetes.io/vts-filter-key" +) + +type vtsFilterKey struct { +} + +// NewParser creates a new vts filter key annotation parser +func NewParser() parser.IngressAnnotation { + return vtsFilterKey{} +} + +// Parse parses the annotations contained in the ingress rule +// used to indicate if the location/s contains a fragment of +// configuration to be included inside the paths of the rules +func (a vtsFilterKey) Parse(ing *extensions.Ingress) (interface{}, error) { + return parser.GetStringAnnotation(annotation, ing) +} diff --git a/core/pkg/ingress/controller/annotations.go b/core/pkg/ingress/controller/annotations.go index af2037dc7550..8fc9525802e9 100644 --- a/core/pkg/ingress/controller/annotations.go +++ b/core/pkg/ingress/controller/annotations.go @@ -40,6 +40,7 @@ import ( "k8s.io/ingress/core/pkg/ingress/annotations/snippet" "k8s.io/ingress/core/pkg/ingress/annotations/sslpassthrough" "k8s.io/ingress/core/pkg/ingress/annotations/upstreamvhost" + "k8s.io/ingress/core/pkg/ingress/annotations/vtsfilterkey" "k8s.io/ingress/core/pkg/ingress/errors" "k8s.io/ingress/core/pkg/ingress/resolver" ) @@ -80,6 +81,7 @@ func newAnnotationExtractor(cfg extractorConfig) annotationExtractor { "ClientBodyBufferSize": clientbodybuffersize.NewParser(), "DefaultBackend": defaultbackend.NewParser(cfg), "UpstreamVhost": upstreamvhost.NewParser(), + "VtsFilterKey": vtsfilterkey.NewParser(), }, } } diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go index 0cb76a719a31..f2e57524f1c4 100644 --- a/core/pkg/ingress/types.go +++ b/core/pkg/ingress/types.go @@ -313,6 +313,10 @@ type Location struct { // UsePortInRedirects indicates if redirects must specify the port // +optional UsePortInRedirects bool `json:"usePortInRedirects"` + // VtsFilterKey contains the vts filter key on the location level + // https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_filter_by_set_key + // +optional + VtsFilterKey string `json:"vtsFilterKey,omitempty"` // ConfigurationSnippet contains additional configuration for the backend // to be considered in the configuration of the location ConfigurationSnippet string `json:"configurationSnippet"` From e37ea14e7eb4efaae065213be550c6e467c15502 Mon Sep 17 00:00:00 2001 From: Giancarlo Rubio Date: Fri, 15 Sep 2017 10:13:04 +0200 Subject: [PATCH 2/2] Set default value for vtsDefaultFilterKey --- controllers/nginx/pkg/config/config.go | 1 + controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go index 68d6af068e66..d4121e35b027 100644 --- a/controllers/nginx/pkg/config/config.go +++ b/controllers/nginx/pkg/config/config.go @@ -422,6 +422,7 @@ func NewDefault() Configuration { WorkerShutdownTimeout: "10s", LoadBalanceAlgorithm: defaultLoadBalancerAlgorithm, VtsStatusZoneSize: "10m", + VtsDefaultFilterKey: "$geoip_country_code country::*", VariablesHashBucketSize: 64, VariablesHashMaxSize: 2048, UseHTTP2: true, diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index a878d2c5c69e..cfa721acdbb1 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -46,7 +46,7 @@ http { {{ if $cfg.EnableVtsStatus }} vhost_traffic_status_zone shared:vhost_traffic_status:{{ $cfg.VtsStatusZoneSize }}; - vhost_traffic_status_filter_by_set_key {{ if not $cfg.VtsDefaultFilterKey }}$geoip_country_code country::*{{ else }} {{ $cfg.VtsDefaultFilterKey }}{{ end }}; + vhost_traffic_status_filter_by_set_key {{ $cfg.VtsDefaultFilterKey }}; {{ end }} sendfile on;